From 6c4dc821fa240d8ad1c24c3397a5eea2eb0a86aa Mon Sep 17 00:00:00 2001 From: mpl Date: Thu, 18 Jul 2013 01:10:50 +0200 Subject: [PATCH] update closure library: automatically generate the list of required files Also updated parseProvidesRequires to use a bufio.Scanner http://camlistore.org/issue/149 Change-Id: I13df7f03e3482b77e71687adc2b71b8dd9eeb0db --- pkg/misc/closure/gendeps.go | 94 ++++++++++-- pkg/misc/closure/gendeps_test.go | 79 +++++++++++ third_party/closure/updatelibrary.go | 204 +++++++++++---------------- 3 files changed, 243 insertions(+), 134 deletions(-) create mode 100644 pkg/misc/closure/gendeps_test.go diff --git a/pkg/misc/closure/gendeps.go b/pkg/misc/closure/gendeps.go index 0a8e53e95..8b3624dcf 100644 --- a/pkg/misc/closure/gendeps.go +++ b/pkg/misc/closure/gendeps.go @@ -96,15 +96,9 @@ func parseProvidesRequires(fi os.FileInfo, path string, f io.Reader) (provides, return ci.provides, ci.requires, nil } - br := bufio.NewReader(f) - for { - l, err := br.ReadString('\n') - if err == io.EOF { - break - } - if err != nil { - return nil, nil, err - } + scanner := bufio.NewScanner(f) + for scanner.Scan() { + l := scanner.Text() if !strings.HasPrefix(l, "goog.") { continue } @@ -117,6 +111,9 @@ func parseProvidesRequires(fi os.FileInfo, path string, f io.Reader) (provides, } } } + if err := scanner.Err(); err != nil { + return nil, nil, err + } depCache[path] = depCacheItem{provides: provides, requires: requires, modTime: mt} return provides, requires, nil } @@ -136,3 +133,82 @@ func (s jsList) String() string { buf.WriteByte(']') return buf.String() } + +// Example of a match: +// goog.addDependency('asserts/asserts.js', ['goog.asserts', 'goog.asserts.AssertionError'], ['goog.debug.Error', 'goog.string']); +// So with m := depsRx.FindStringSubmatch, +// the provider: m[1] == "asserts/asserts.js" +// the provided namespaces: m[2] == "'goog.asserts', 'goog.asserts.AssertionError'" +// the required namespaces: m[5] == "'goog.debug.Error', 'goog.string'" +var depsRx = regexp.MustCompile(`^goog.addDependency\(['"]([^/]+[a-zA-Z0-9\-\_/\.]*\.js)['"], \[((['"][\w\.]+['"])+(, ['"][\w\.]+['"])*)\], \[((['"][\w\.]+['"])+(, ['"][\w\.]+['"])*)?\]\);`) + +// ParseDeps reads closure namespace dependency lines and +// returns a map giving the js file provider for each namespace, +// and a map giving the namespace dependencies for each namespace. +func ParseDeps(r io.Reader) (providedBy map[string]string, requires map[string][]string, err error) { + providedBy = make(map[string]string) + requires = make(map[string][]string) + scanner := bufio.NewScanner(r) + for scanner.Scan() { + l := scanner.Text() + if strings.HasPrefix(l, "//") { + continue + } + if l == "" { + continue + } + m := depsRx.FindStringSubmatch(l) + if m == nil { + return nil, nil, fmt.Errorf("Invalid line in deps: %q", l) + } + jsfile := m[1] + provides := strings.Split(m[2], ", ") + var required []string + if m[5] != "" { + required = strings.Split( + strings.Replace(strings.Replace(m[5], "'", "", -1), `"`, "", -1), ", ") + } + for _, v := range provides { + namespace := strings.Trim(v, `'"`) + if otherjs, ok := providedBy[namespace]; ok { + return nil, nil, fmt.Errorf("Name %v is provided by both %v and %v", namespace, jsfile, otherjs) + } + providedBy[namespace] = jsfile + if _, ok := requires[namespace]; ok { + return nil, nil, fmt.Errorf("Name %v has two sets of dependencies") + } + if required != nil { + requires[namespace] = required + } + } + } + if err := scanner.Err(); err != nil { + return nil, nil, err + } + return providedBy, requires, nil +} + +// DeepParseDeps reads closure namespace dependency lines and +// returns a map giving all the required js files for each namespace. +func DeepParseDeps(r io.Reader) (map[string][]string, error) { + providedBy, requires, err := ParseDeps(r) + if err != nil { + return nil, err + } + filesDeps := make(map[string][]string) + var deeperDeps func(namespace string) []string + deeperDeps = func(namespace string) []string { + if jsdeps, ok := filesDeps[namespace]; ok { + return jsdeps + } + jsfiles := []string{providedBy[namespace]} + for _, dep := range requires[namespace] { + jsfiles = append(jsfiles, deeperDeps(dep)...) + } + return jsfiles + } + for namespace, _ := range providedBy { + filesDeps[namespace] = deeperDeps(namespace) + } + return filesDeps, nil +} diff --git a/pkg/misc/closure/gendeps_test.go b/pkg/misc/closure/gendeps_test.go new file mode 100644 index 000000000..6326c84aa --- /dev/null +++ b/pkg/misc/closure/gendeps_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2013 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package closure + +import ( + "reflect" + "strings" + "testing" +) + +var testdata = ` +goog.addDependency('asserts/asserts.js', ['goog.asserts', 'goog.asserts.AssertionError'], ['goog.debug.Error', 'goog.string']); +goog.addDependency('debug/error.js', ['goog.debug.Error'], []); +goog.addDependency('string/string.js', ['goog.string', 'goog.string.Unicode'], []); +` + +type parsedDeps struct { + providedBy map[string]string + requires map[string][]string +} + +var parsedWant = parsedDeps{ + providedBy: map[string]string{ + "goog.asserts": "asserts/asserts.js", + "goog.asserts.AssertionError": "asserts/asserts.js", + "goog.debug.Error": "debug/error.js", + "goog.string": "string/string.js", + "goog.string.Unicode": "string/string.js", + }, + requires: map[string][]string{ + "goog.asserts": []string{"goog.debug.Error", "goog.string"}, + "goog.asserts.AssertionError": []string{"goog.debug.Error", "goog.string"}, + }, +} + +var deepParsedWant = map[string][]string{ + "goog.asserts": []string{"asserts/asserts.js", "debug/error.js", "string/string.js"}, + "goog.asserts.AssertionError": []string{"asserts/asserts.js", "debug/error.js", "string/string.js"}, + "goog.debug.Error": []string{"debug/error.js"}, + "goog.string": []string{"string/string.js"}, + "goog.string.Unicode": []string{"string/string.js"}, +} + +func TestParseDeps(t *testing.T) { + providedBy, requires, err := ParseDeps(strings.NewReader(testdata)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(parsedWant.providedBy, providedBy) { + t.Fatalf("Failed to parse closure deps: wanted %v, got %v", parsedWant.providedBy, providedBy) + } + if !reflect.DeepEqual(parsedWant.requires, requires) { + t.Fatalf("Failed to parse closure deps: wanted %v, got %v", parsedWant.requires, requires) + } +} + +func TestDeepParseDeps(t *testing.T) { + deps, err := DeepParseDeps(strings.NewReader(testdata)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(deepParsedWant, deps) { + t.Fatalf("Failed to parse closure deps: wanted %v, got %v", deepParsedWant, deps) + } +} diff --git a/third_party/closure/updatelibrary.go b/third_party/closure/updatelibrary.go index 3aeaed093..5f543b1e7 100644 --- a/third_party/closure/updatelibrary.go +++ b/third_party/closure/updatelibrary.go @@ -20,15 +20,19 @@ limitations under the License. package main import ( + "bytes" "flag" "fmt" "io" "log" + "net/http" "os" "os/exec" "path/filepath" + "sort" "strings" + "camlistore.org/pkg/misc/closure" "camlistore.org/pkg/osutil" ) @@ -37,129 +41,6 @@ const ( gitHash = "1389e13" ) -// fileList is the list of resources from the closure library that -// are required by the ui pages of Camlistore. It was generated -// from the error messages given in the javascript console. -// TODO(mpl): Better way to do that generation. -// See http://camlistore.org/issue/149 -var fileList = []string{ - "AUTHORS", - "LICENSE", - "README", - "closure/goog/a11y/aria/announcer.js", - "closure/goog/a11y/aria/aria.js", - "closure/goog/array/array.js", - "closure/goog/asserts/asserts.js", - "closure/goog/base.js", - "closure/goog/css/common.css", - "closure/goog/css/toolbar.css", - "closure/goog/debug/debug.js", - "closure/goog/debug/entrypointregistry.js", - "closure/goog/debug/errorhandler.js", - "closure/goog/debug/errorhandlerweakdep.js", - "closure/goog/debug/error.js", - "closure/goog/debug/logbuffer.js", - "closure/goog/debug/logger.js", - "closure/goog/debug/logrecord.js", - "closure/goog/debug/tracer.js", - "closure/goog/deps.js", - "closure/goog/dom/a11y.js", - "closure/goog/dom/browserfeature.js", - "closure/goog/dom/classes.js", - "closure/goog/dom/dom.js", - "closure/goog/dom/tagname.js", - "closure/goog/dom/vendor.js", - "closure/goog/disposable/disposable.js", - "closure/goog/disposable/idisposable.js", - "closure/goog/events/browserevent.js", - "closure/goog/events/browserfeature.js", - "closure/goog/events/eventhandler.js", - "closure/goog/events/event.js", - "closure/goog/events/events.js", - "closure/goog/events/eventtarget.js", - "closure/goog/events/eventtype.js", - "closure/goog/events/eventwrapper.js", - "closure/goog/events/filedrophandler.js", - "closure/goog/events/keycodes.js", - "closure/goog/events/keyhandler.js", - "closure/goog/events/listenable.js", - "closure/goog/events/listener.js", - "closure/goog/fx/transition.js", - "closure/goog/iter/iter.js", - "closure/goog/json/json.js", - "closure/goog/math/box.js", - "closure/goog/math/coordinate.js", - "closure/goog/math/math.js", - "closure/goog/math/rect.js", - "closure/goog/math/size.js", - "closure/goog/net/errorcode.js", - "closure/goog/net/eventtype.js", - "closure/goog/net/httpstatus.js", - "closure/goog/net/wrapperxmlhttpfactory.js", - "closure/goog/net/xhrio.js", - "closure/goog/net/xmlhttpfactory.js", - "closure/goog/net/xmlhttp.js", - "closure/goog/object/object.js", - "closure/goog/positioning/abstractposition.js", - "closure/goog/positioning/anchoredposition.js", - "closure/goog/positioning/anchoredviewportposition.js", - "closure/goog/positioning/clientposition.js", - "closure/goog/positioning/menuanchoredposition.js", - "closure/goog/positioning/positioning.js", - "closure/goog/positioning/viewportclientposition.js", - "closure/goog/reflect/reflect.js", - "closure/goog/string/string.js", - "closure/goog/structs/collection.js", - "closure/goog/structs/map.js", - "closure/goog/structs/set.js", - "closure/goog/structs/simplepool.js", - "closure/goog/structs/structs.js", - "closure/goog/style/bidi.js", - "closure/goog/style/style.js", - "closure/goog/timer/timer.js", - "closure/goog/ui/button.js", - "closure/goog/ui/buttonrenderer.js", - "closure/goog/ui/buttonside.js", - "closure/goog/ui/component.js", - "closure/goog/ui/container.js", - "closure/goog/ui/containerrenderer.js", - "closure/goog/ui/controlcontent.js", - "closure/goog/ui/control.js", - "closure/goog/ui/controlrenderer.js", - "closure/goog/ui/cssnames.js", - "closure/goog/ui/custombuttonrenderer.js", - "closure/goog/ui/decorate.js", - "closure/goog/ui/idgenerator.js", - "closure/goog/ui/menubutton.js", - "closure/goog/ui/menubuttonrenderer.js", - "closure/goog/ui/menuheader.js", - "closure/goog/ui/menuheaderrenderer.js", - "closure/goog/ui/menuitem.js", - "closure/goog/ui/menuitemrenderer.js", - "closure/goog/ui/menu.js", - "closure/goog/ui/menurenderer.js", - "closure/goog/ui/menuseparator.js", - "closure/goog/ui/menuseparatorrenderer.js", - "closure/goog/ui/nativebuttonrenderer.js", - "closure/goog/ui/popupbase.js", - "closure/goog/ui/popupmenu.js", - "closure/goog/ui/registry.js", - "closure/goog/ui/separator.js", - "closure/goog/ui/textarea.js", - "closure/goog/ui/textarearenderer.js", - "closure/goog/ui/toolbarbutton.js", - "closure/goog/ui/toolbarbuttonrenderer.js", - "closure/goog/ui/toolbar.js", - "closure/goog/ui/toolbarmenubutton.js", - "closure/goog/ui/toolbarmenubuttonrenderer.js", - "closure/goog/ui/toolbarrenderer.js", - "closure/goog/ui/toolbarseparatorrenderer.js", - "closure/goog/uri/uri.js", - "closure/goog/uri/utils.js", - "closure/goog/useragent/product.js", - "closure/goog/useragent/useragent.js", -} - var ( currentRevCmd = newCmd("git", "rev-parse", "--short", "HEAD") gitFetchCmd = newCmd("git", "fetch") @@ -178,6 +59,70 @@ func init() { flag.BoolVar(&verbose, "verbose", false, "verbose output") } +// fileList parses deps.js from the closure repo, as well as the similar +// dependencies generated for the UI js files, and compiles the list of +// js files from the closure lib required for the UI. +func fileList() ([]string, error) { + camliRootPath, err := osutil.GoPackagePath("camlistore.org") + if err != nil { + log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).") + } + uiDir := filepath.Join(camliRootPath, "server", "camlistored", "ui") + closureDepsFile := filepath.Join(closureGitDir, "closure", "goog", "deps.js") + + f, err := os.Open(closureDepsFile) + if err != nil { + return nil, err + } + defer f.Close() + allClosureDeps, err := closure.DeepParseDeps(f) + if err != nil { + return nil, err + } + + uiDeps, err := closure.GenDeps(http.Dir(uiDir)) + if err != nil { + return nil, err + } + _, requ, err := closure.ParseDeps(bytes.NewReader(uiDeps)) + if err != nil { + return nil, err + } + + nameDone := make(map[string]bool) + jsfilesDone := make(map[string]bool) + for _, deps := range requ { + for _, dep := range deps { + if _, ok := nameDone[dep]; ok { + continue + } + jsfiles := allClosureDeps[dep] + for _, filename := range jsfiles { + if _, ok := jsfilesDone[filename]; ok { + continue + } + jsfilesDone[filename] = true + } + nameDone[dep] = true + } + } + jsfiles := []string{ + "AUTHORS", + "LICENSE", + "README", + filepath.Join("closure", "goog", "base.js"), + filepath.Join("closure", "goog", "css", "common.css"), + filepath.Join("closure", "goog", "css", "toolbar.css"), + filepath.Join("closure", "goog", "deps.js"), + } + prefix := filepath.Join("closure", "goog") + for k, _ := range jsfilesDone { + jsfiles = append(jsfiles, filepath.Join(prefix, k)) + } + sort.Strings(jsfiles) + return jsfiles, nil +} + type command struct { program string args []string @@ -204,11 +149,20 @@ func (c *command) run() []byte { func resetAndCheckout() { gitResetCmd.run() + // we need deps.js to build the list of files, so we get it first args := gitCheckoutCmd.args - args = append(args, fileList...) + args = append(args, filepath.Join("closure", "goog", "deps.js")) + depsCheckoutCmd := newCmd(gitCheckoutCmd.program, args...) + depsCheckoutCmd.run() + files, err := fileList() + if err != nil { + log.Fatalf("Could not generate files list: %v", err) + } + args = gitCheckoutCmd.args + args = append(args, files...) partialCheckoutCmd := newCmd(gitCheckoutCmd.program, args...) if verbose { - log.Printf("%v", partialCheckoutCmd) + fmt.Printf("%v\n", partialCheckoutCmd) } partialCheckoutCmd.run() }