Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update closure library: automatically generate the list of required files 

Also updated parseProvidesRequires to use a bufio.Scanner

http://camlistore.org/issue/149

Change-Id: I13df7f03e3482b77e71687adc2b71b8dd9eeb0db
This commit is contained in:
mpl 2013-07-18 01:10:50 +02:00
parent 847c583183
commit 6c4dc821fa
3 changed files with 243 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
pkg/misc/closure/gendeps.go
View File

@ -96,15 +96,9 @@ func parseProvidesRequires(fi os.FileInfo, path string, f io.Reader) (provides,
return ci.provides, ci.requires, nil
}
br := bufio.NewReader(f)
for {
l, err := br.ReadString('\n')
if err == io.EOF {
break
}
if err != nil {
return nil, nil, err
}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
l := scanner.Text()
if !strings.HasPrefix(l, "goog.") {
continue
}
@ -117,6 +111,9 @@ func parseProvidesRequires(fi os.FileInfo, path string, f io.Reader) (provides,
}
}
}
if err := scanner.Err(); err != nil {
return nil, nil, err
}
depCache[path] = depCacheItem{provides: provides, requires: requires, modTime: mt}
return provides, requires, nil
}
@ -136,3 +133,82 @@ func (s jsList) String() string {
buf.WriteByte(']')
return buf.String()
}
// Example of a match:
// goog.addDependency('asserts/asserts.js', ['goog.asserts', 'goog.asserts.AssertionError'], ['goog.debug.Error', 'goog.string']);
// So with m := depsRx.FindStringSubmatch,
// the provider: m[1] == "asserts/asserts.js"
// the provided namespaces: m[2] == "'goog.asserts', 'goog.asserts.AssertionError'"
// the required namespaces: m[5] == "'goog.debug.Error', 'goog.string'"
var depsRx = regexp.MustCompile(`^goog.addDependency\(['"]([^/]+[a-zA-Z0-9\-\_/\.]*\.js)['"], \[((['"][\w\.]+['"])+(, ['"][\w\.]+['"])*)\], \[((['"][\w\.]+['"])+(, ['"][\w\.]+['"])*)?\]\);`)
// ParseDeps reads closure namespace dependency lines and
// returns a map giving the js file provider for each namespace,
// and a map giving the namespace dependencies for each namespace.
func ParseDeps(r io.Reader) (providedBy map[string]string, requires map[string][]string, err error) {
providedBy = make(map[string]string)
requires = make(map[string][]string)
scanner := bufio.NewScanner(r)
for scanner.Scan() {
l := scanner.Text()
if strings.HasPrefix(l, "//") {
continue
}
if l == "" {
continue
}
m := depsRx.FindStringSubmatch(l)
if m == nil {
return nil, nil, fmt.Errorf("Invalid line in deps: %q", l)
}
jsfile := m[1]
provides := strings.Split(m[2], ", ")
var required []string
if m[5] != "" {
required = strings.Split(
strings.Replace(strings.Replace(m[5], "'", "", -1), `"`, "", -1), ", ")
}
for _, v := range provides {
namespace := strings.Trim(v, `'"`)
if otherjs, ok := providedBy[namespace]; ok {
return nil, nil, fmt.Errorf("Name %v is provided by both %v and %v", namespace, jsfile, otherjs)
}
providedBy[namespace] = jsfile
if _, ok := requires[namespace]; ok {
return nil, nil, fmt.Errorf("Name %v has two sets of dependencies")
}
if required != nil {
requires[namespace] = required
}
}
}
if err := scanner.Err(); err != nil {
return nil, nil, err
}
return providedBy, requires, nil
}
// DeepParseDeps reads closure namespace dependency lines and
// returns a map giving all the required js files for each namespace.
func DeepParseDeps(r io.Reader) (map[string][]string, error) {
providedBy, requires, err := ParseDeps(r)
if err != nil {
return nil, err
}
filesDeps := make(map[string][]string)
var deeperDeps func(namespace string) []string
deeperDeps = func(namespace string) []string {
if jsdeps, ok := filesDeps[namespace]; ok {
return jsdeps
}
jsfiles := []string{providedBy[namespace]}
for _, dep := range requires[namespace] {
jsfiles = append(jsfiles, deeperDeps(dep)...)
}
return jsfiles
}
for namespace, _ := range providedBy {
filesDeps[namespace] = deeperDeps(namespace)
}
return filesDeps, nil
}

79
pkg/misc/closure/gendeps_test.go Normal file
View File

@ -0,0 +1,79 @@
/*
Copyright 2013 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package closure
import (
"reflect"
"strings"
"testing"
)
var testdata = `
goog.addDependency('asserts/asserts.js', ['goog.asserts', 'goog.asserts.AssertionError'], ['goog.debug.Error', 'goog.string']);
goog.addDependency('debug/error.js', ['goog.debug.Error'], []);
goog.addDependency('string/string.js', ['goog.string', 'goog.string.Unicode'], []);
`
type parsedDeps struct {
providedBy map[string]string
requires map[string][]string
}
var parsedWant = parsedDeps{
providedBy: map[string]string{
"goog.asserts": "asserts/asserts.js",
"goog.asserts.AssertionError": "asserts/asserts.js",
"goog.debug.Error": "debug/error.js",
"goog.string": "string/string.js",
"goog.string.Unicode": "string/string.js",
},
requires: map[string][]string{
"goog.asserts": []string{"goog.debug.Error", "goog.string"},
"goog.asserts.AssertionError": []string{"goog.debug.Error", "goog.string"},
},
}
var deepParsedWant = map[string][]string{
"goog.asserts": []string{"asserts/asserts.js", "debug/error.js", "string/string.js"},
"goog.asserts.AssertionError": []string{"asserts/asserts.js", "debug/error.js", "string/string.js"},
"goog.debug.Error": []string{"debug/error.js"},
"goog.string": []string{"string/string.js"},
"goog.string.Unicode": []string{"string/string.js"},
}
func TestParseDeps(t *testing.T) {
providedBy, requires, err := ParseDeps(strings.NewReader(testdata))
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(parsedWant.providedBy, providedBy) {
t.Fatalf("Failed to parse closure deps: wanted %v, got %v", parsedWant.providedBy, providedBy)
}
if !reflect.DeepEqual(parsedWant.requires, requires) {
t.Fatalf("Failed to parse closure deps: wanted %v, got %v", parsedWant.requires, requires)
}
}
func TestDeepParseDeps(t *testing.T) {
deps, err := DeepParseDeps(strings.NewReader(testdata))
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(deepParsedWant, deps) {
t.Fatalf("Failed to parse closure deps: wanted %v, got %v", deepParsedWant, deps)
}
}

204
third_party/closure/updatelibrary.go vendored
View File

@ -20,15 +20,19 @@ limitations under the License.
package main
import (
"bytes"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"camlistore.org/pkg/misc/closure"
"camlistore.org/pkg/osutil"
)
@ -37,129 +41,6 @@ const (
gitHash = "1389e13"
)
// fileList is the list of resources from the closure library that
// are required by the ui pages of Camlistore. It was generated
// from the error messages given in the javascript console.
// TODO(mpl): Better way to do that generation.
// See http://camlistore.org/issue/149
var fileList = []string{
"AUTHORS",
"LICENSE",
"README",
"closure/goog/a11y/aria/announcer.js",
"closure/goog/a11y/aria/aria.js",
"closure/goog/array/array.js",
"closure/goog/asserts/asserts.js",
"closure/goog/base.js",
"closure/goog/css/common.css",
"closure/goog/css/toolbar.css",
"closure/goog/debug/debug.js",
"closure/goog/debug/entrypointregistry.js",
"closure/goog/debug/errorhandler.js",
"closure/goog/debug/errorhandlerweakdep.js",
"closure/goog/debug/error.js",
"closure/goog/debug/logbuffer.js",
"closure/goog/debug/logger.js",
"closure/goog/debug/logrecord.js",
"closure/goog/debug/tracer.js",
"closure/goog/deps.js",
"closure/goog/dom/a11y.js",
"closure/goog/dom/browserfeature.js",
"closure/goog/dom/classes.js",
"closure/goog/dom/dom.js",
"closure/goog/dom/tagname.js",
"closure/goog/dom/vendor.js",
"closure/goog/disposable/disposable.js",
"closure/goog/disposable/idisposable.js",
"closure/goog/events/browserevent.js",
"closure/goog/events/browserfeature.js",
"closure/goog/events/eventhandler.js",
"closure/goog/events/event.js",
"closure/goog/events/events.js",
"closure/goog/events/eventtarget.js",
"closure/goog/events/eventtype.js",
"closure/goog/events/eventwrapper.js",
"closure/goog/events/filedrophandler.js",
"closure/goog/events/keycodes.js",
"closure/goog/events/keyhandler.js",
"closure/goog/events/listenable.js",
"closure/goog/events/listener.js",
"closure/goog/fx/transition.js",
"closure/goog/iter/iter.js",
"closure/goog/json/json.js",
"closure/goog/math/box.js",
"closure/goog/math/coordinate.js",
"closure/goog/math/math.js",
"closure/goog/math/rect.js",
"closure/goog/math/size.js",
"closure/goog/net/errorcode.js",
"closure/goog/net/eventtype.js",
"closure/goog/net/httpstatus.js",
"closure/goog/net/wrapperxmlhttpfactory.js",
"closure/goog/net/xhrio.js",
"closure/goog/net/xmlhttpfactory.js",
"closure/goog/net/xmlhttp.js",
"closure/goog/object/object.js",
"closure/goog/positioning/abstractposition.js",
"closure/goog/positioning/anchoredposition.js",
"closure/goog/positioning/anchoredviewportposition.js",
"closure/goog/positioning/clientposition.js",
"closure/goog/positioning/menuanchoredposition.js",
"closure/goog/positioning/positioning.js",
"closure/goog/positioning/viewportclientposition.js",
"closure/goog/reflect/reflect.js",
"closure/goog/string/string.js",
"closure/goog/structs/collection.js",
"closure/goog/structs/map.js",
"closure/goog/structs/set.js",
"closure/goog/structs/simplepool.js",
"closure/goog/structs/structs.js",
"closure/goog/style/bidi.js",
"closure/goog/style/style.js",
"closure/goog/timer/timer.js",
"closure/goog/ui/button.js",
"closure/goog/ui/buttonrenderer.js",
"closure/goog/ui/buttonside.js",
"closure/goog/ui/component.js",
"closure/goog/ui/container.js",
"closure/goog/ui/containerrenderer.js",
"closure/goog/ui/controlcontent.js",
"closure/goog/ui/control.js",
"closure/goog/ui/controlrenderer.js",
"closure/goog/ui/cssnames.js",
"closure/goog/ui/custombuttonrenderer.js",
"closure/goog/ui/decorate.js",
"closure/goog/ui/idgenerator.js",
"closure/goog/ui/menubutton.js",
"closure/goog/ui/menubuttonrenderer.js",
"closure/goog/ui/menuheader.js",
"closure/goog/ui/menuheaderrenderer.js",
"closure/goog/ui/menuitem.js",
"closure/goog/ui/menuitemrenderer.js",
"closure/goog/ui/menu.js",
"closure/goog/ui/menurenderer.js",
"closure/goog/ui/menuseparator.js",
"closure/goog/ui/menuseparatorrenderer.js",
"closure/goog/ui/nativebuttonrenderer.js",
"closure/goog/ui/popupbase.js",
"closure/goog/ui/popupmenu.js",
"closure/goog/ui/registry.js",
"closure/goog/ui/separator.js",
"closure/goog/ui/textarea.js",
"closure/goog/ui/textarearenderer.js",
"closure/goog/ui/toolbarbutton.js",
"closure/goog/ui/toolbarbuttonrenderer.js",
"closure/goog/ui/toolbar.js",
"closure/goog/ui/toolbarmenubutton.js",
"closure/goog/ui/toolbarmenubuttonrenderer.js",
"closure/goog/ui/toolbarrenderer.js",
"closure/goog/ui/toolbarseparatorrenderer.js",
"closure/goog/uri/uri.js",
"closure/goog/uri/utils.js",
"closure/goog/useragent/product.js",
"closure/goog/useragent/useragent.js",
}
var (
currentRevCmd = newCmd("git", "rev-parse", "--short", "HEAD")
gitFetchCmd = newCmd("git", "fetch")
@ -178,6 +59,70 @@ func init() {
flag.BoolVar(&verbose, "verbose", false, "verbose output")
}
// fileList parses deps.js from the closure repo, as well as the similar
// dependencies generated for the UI js files, and compiles the list of
// js files from the closure lib required for the UI.
func fileList() ([]string, error) {
camliRootPath, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
}
uiDir := filepath.Join(camliRootPath, "server", "camlistored", "ui")
closureDepsFile := filepath.Join(closureGitDir, "closure", "goog", "deps.js")
f, err := os.Open(closureDepsFile)
if err != nil {
return nil, err
}
defer f.Close()
allClosureDeps, err := closure.DeepParseDeps(f)
if err != nil {
return nil, err
}
uiDeps, err := closure.GenDeps(http.Dir(uiDir))
if err != nil {
return nil, err
}
_, requ, err := closure.ParseDeps(bytes.NewReader(uiDeps))
if err != nil {
return nil, err
}
nameDone := make(map[string]bool)
jsfilesDone := make(map[string]bool)
for _, deps := range requ {
for _, dep := range deps {
if _, ok := nameDone[dep]; ok {
continue
}
jsfiles := allClosureDeps[dep]
for _, filename := range jsfiles {
if _, ok := jsfilesDone[filename]; ok {
continue
}
jsfilesDone[filename] = true
}
nameDone[dep] = true
}
}
jsfiles := []string{
"AUTHORS",
"LICENSE",
"README",
filepath.Join("closure", "goog", "base.js"),
filepath.Join("closure", "goog", "css", "common.css"),
filepath.Join("closure", "goog", "css", "toolbar.css"),
filepath.Join("closure", "goog", "deps.js"),
}
prefix := filepath.Join("closure", "goog")
for k, _ := range jsfilesDone {
jsfiles = append(jsfiles, filepath.Join(prefix, k))
}
sort.Strings(jsfiles)
return jsfiles, nil
}
type command struct {
program string
args []string
@ -204,11 +149,20 @@ func (c *command) run() []byte {
func resetAndCheckout() {
gitResetCmd.run()
// we need deps.js to build the list of files, so we get it first
args := gitCheckoutCmd.args
args = append(args, fileList...)
args = append(args, filepath.Join("closure", "goog", "deps.js"))
depsCheckoutCmd := newCmd(gitCheckoutCmd.program, args...)
depsCheckoutCmd.run()
files, err := fileList()
if err != nil {
log.Fatalf("Could not generate files list: %v", err)
}
args = gitCheckoutCmd.args
args = append(args, files...)
partialCheckoutCmd := newCmd(gitCheckoutCmd.program, args...)
if verbose {
log.Printf("%v", partialCheckoutCmd)
fmt.Printf("%v\n", partialCheckoutCmd)
}
partialCheckoutCmd.run()
}