Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ui: serve Closure from embedded zip data when available 

Change-Id: I9bb6bb4f13f69b293fd98441d82068c0677ffbd5
This commit is contained in:
Brad Fitzpatrick 2013-06-18 23:14:36 -07:00
parent 8071a49516
commit 10d000d900
6 changed files with 168 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
TODO
View File

@ -4,8 +4,7 @@ There are two TODO lists. This file (good for airplanes) and the online bug trac
Offline list:
-- add camliRoot & closureRoot to pkg/genconfig high-level config &
the docs
-- add sourceRoot to pkg/genconfig high-level config & the docs
-- website: remove the "Installation" heading for /cmd/*, since
they're misleading and people should use "go run make.go" in the
@ -14,24 +13,6 @@ Offline list:
-- website: add godoc for /server/camlistored (also without a "go get"
line)
-- make fileembed & zembed only happen as part of "make.go", but
remove the z*.go files from git (and make them all .gitignore'd).
Users' options will be either a) use make.go to get a fully static
binary (with all of Closure zembed-ed), or b) use "go install" and
get a binary that requires configuration to tell it where files on
disk are. On start-up, if the UI is configured, the UI should fail
and say "You didn't configure the path to the UI & Closure
directories". make.go could further have a flag.Bool to do the
zembed, defaulting to true.
Despite the full Closure source being 10 MB, it's only 2 MB if we
compress it and embed a zip or .tar.gz in the static binary:
$ find tmp/closure-lib/closure/ -name '*.js' | xargs cat | wc
327098 1214807 10449092
$ find tmp/closure-lib/closure/ -name '*.js' | xargs cat | gzip -c | wc
7436 58545 1932707
-- tests for all cmd/* stuff, perhaps as part of some integration
tests.

3
config/dev-server-config.json
View File

@ -52,8 +52,7 @@
"/ui/": {
"handler": "ui",
"handlerArgs": {
"camliRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""],
"closureRoot": ["_env", "${CAMLI_DEV_CLOSURE_ROOT}", ""],
"sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""],
"jsonSignRoot": "/sighelper/",
"cache": "/cache/",
"scaledImage": "lrucache",

34
dev-server
View File

@ -7,10 +7,9 @@ require "$Bin/misc/devlib.pl";
require "$Bin/misc/get_closure.pl";
sub usage {
die "Usage: dev-server [--wipe] [--mongo|--mysql|--postgres] [--tls] <portnumber>" .
"[--all] [--nobuild] [--staticres] [--offline] [--KBps] <number>" .
"[--latency_ms] <number> [--fast] [--verbose] [--hostname] <name> [--compile-js]" .
" -- [other_blobserver_opts]";
die "Usage: dev-server [OPTS] [<portnumber>] -- [other_blobserver_opts]\n\nWhere OPTS include:\n [--wipe] [--mongo|--mysql|--postgres] [--tls] [--fullclosure] " .
"[--all] [--nobuild] [--staticres] [--offline] [--KBps=<number>] " .
"[--latency_ms=<number>] [--fast] [--verbose] [--hostname=<name>]\n";
}
chdir $Bin or die;
@ -22,7 +21,7 @@ my $opt_fast; # shortcut to disable throttling
my $opt_all; # listen on all interfaces
my $opt_hostname; # hostname to advertise, else `hostname` is used
my $opt_nobuild;
my $opt_offline; # don't use the network ("airplane mode")
my $opt_fullclosure; # make all of closure available
my $opt_staticres; # use static resources, not those on disk
my $opt_tls;
my $opt_wipe;
@ -44,12 +43,12 @@ GetOptions("wipe" => \$opt_wipe,
"postgres" => \$opt_postgres,
"mysql" => \$opt_mysql,
"staticres" => \$opt_staticres,
"offline" => \$opt_offline,
"KBps=i" => \$opt_KBps,
"latency_ms=i" => \$opt_latency_ms,
"fast" => \$opt_fast,
"verbose" => \$opt_verbose,
"hostname=s" => \$opt_hostname,
"fullclosure" => \$opt_fullclosure,
)
or usage();
@ -156,6 +155,7 @@ $ENV{CAMLI_TLS} = "false";
if ($opt_tls) {
$ENV{CAMLI_TLS} = "true";
}
$ENV{CAMLI_DEV_CAMLI_ROOT} = $Bin;
$ENV{CAMLI_BASEURL} = $base;
$ENV{CAMLI_AUTH} = "userpass:camlistore:pass$port:+localhost";
$ENV{CAMLI_ADVERTISED_PASSWORD} = "pass$port"; # public password
@ -180,22 +180,14 @@ if ($opt_wipe && -d $templatedir) {
# To use resources from disk, instead of the copies linked into the
# binary:
unless ($opt_staticres) {
unless ($opt_offline) {
if (-e "$Bin/tmp/closure-lib/.svn") {
system("rm", "-rf", "$Bin/tmp/closure-lib") and die "Failed to remove the svn checkout of the closure-lib.\n";
}
get_closure_lib();
get_closure_compiler();
chdir $Bin or die;
if ($opt_fullclosure) {
if (-e "$Bin/tmp/closure-lib/.svn") {
system("rm", "-rf", "$Bin/tmp/closure-lib") and die "Failed to remove the svn checkout of the closure-lib.\n";
}
$ENV{CAMLI_DEV_CAMLI_ROOT} = $Bin;
$ENV{CAMLI_DEV_CLOSURE_ROOT} = "$Bin/tmp/closure-lib";
# TODO: delete CAMLI_DEV_UI_FILES here and in
# server/ui/fileembed_normal.go, once ui.go is updated to use
# camliRoot. This is just here transitionally:
$ENV{CAMLI_DEV_UI_FILES} = "$Bin/server/camlistored/ui";
get_closure_lib();
get_closure_compiler();
chdir $Bin or die;
$ENV{CAMLI_DEV_CLOSURE_DIR} = "$Bin/tmp/closure-lib/closure";
}
print "Starting dev server on $base/ui/ with password \"pass$port\"\n";

12
make.go
View File

@ -37,6 +37,7 @@ import (
"path/filepath"
"runtime"
"strings"
"time"
)
var (
@ -383,6 +384,7 @@ func embedClosure(closureDir, embedFile string) error {
zipdest = io.MultiWriter(zipdest, f)
defer f.Close()
}
var modTime time.Time
w := zip.NewWriter(zipdest)
err := filepath.Walk(closureDir, func(path string, fi os.FileInfo, err error) error {
if err != nil {
@ -395,6 +397,9 @@ func embedClosure(closureDir, embedFile string) error {
if fi.IsDir() {
return nil
}
if mt := fi.ModTime(); mt.After(modTime) {
modTime = mt
}
b, err := ioutil.ReadFile(path)
if err != nil {
return err
@ -417,9 +422,12 @@ func embedClosure(closureDir, embedFile string) error {
// then embed it as a quoted string
var qb bytes.Buffer
fmt.Fprint(&qb, "package closure\n\n")
fmt.Fprint(&qb, "func init() {\n\tZipData = ")
fmt.Fprint(&qb, "import \"time\"\n\n")
fmt.Fprint(&qb, "func init() {\n")
fmt.Fprintf(&qb, "\tZipModTime = time.Unix(%d, 0)\n", modTime.Unix())
fmt.Fprint(&qb, "\tZipData = ")
quote(&qb, zipbuf.Bytes())
fmt.Fprint(&qb, "\n}")
fmt.Fprint(&qb, "\n}\n")
// and write to a .go file
// TODO(mpl): do not regenerate the whole zip file if the modtime

50
pkg/server/ui.go
View File

@ -36,6 +36,7 @@ import (
"camlistore.org/pkg/jsonsign/signhandler"
"camlistore.org/pkg/misc/closure"
uistatic "camlistore.org/server/camlistored/ui"
closurestatic "camlistore.org/server/camlistored/ui/closure"
)
var (
@ -74,12 +75,12 @@ type UIHandler struct {
Cache blobserver.Storage // or nil
sc ScaledImage // cache for scaled images, optional
// camliRoot optionally specifies the path to root of Camlistore's
// sourceRoot optionally specifies the path to root of Camlistore's
// source. If empty, the UI files must be compiled in to the
// binary (with go run make.go). This comes from the "camliRoot"
// binary (with go run make.go). This comes from the "sourceRoot"
// ui handler config option.
// TODO: not yet implemented.
camliRoot string
sourceRoot string
// closureHandler serves the Closure JS files.
closureHandler http.Handler
@ -93,9 +94,8 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er
ui := &UIHandler{
prefix: ld.MyPrefix(),
JSONSignRoot: conf.OptionalString("jsonSignRoot", ""),
camliRoot: conf.OptionalString("camliRoot", ""),
sourceRoot: conf.OptionalString("sourceRoot", ""),
}
closureRoot := conf.OptionalString("closureRoot", "")
pubRoots := conf.OptionalList("publishRoots")
cachePrefix := conf.OptionalString("cache", "")
scType := conf.OptionalString("scaledImage", "")
@ -155,9 +155,9 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er
}
}
ui.closureHandler, err = ui.makeClosureHandler(closureRoot)
ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot)
if err != nil {
return nil, fmt.Errorf(`Invalid "closureRoot" value of %q: %v"`, closureRoot, err)
return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err)
}
rootPrefix, _, err := ld.FindHandlerByType("root")
@ -176,19 +176,31 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er
// makeClosureHandler returns a handler to serve Closure files.
// root is either:
// 1) empty: use the Closure files compiled in to the binar (if available), else redirect to the Internet.
// 2) a URL prefix: base of Closure to redirect to
// 3) a path on disk to serve files from
// 1) empty: use the Closure files compiled in to the binary (if
// available), else redirect to the Internet.
// 2) a URL prefix: base of Camlistore to get Closure to redirect to
// 3) a path on disk to the root of camlistore's source (which
// contains the necessary subset of Closure files)
func (ui *UIHandler) makeClosureHandler(root string) (http.Handler, error) {
// In development mode, serve the Closure files from disk directly.
// dev-server environment variable takes precendence:
if d := os.Getenv("CAMLI_DEV_CLOSURE_DIR"); d != "" {
log.Printf("ui: serving Closure from dev-server's $CAMLI_DEV_CLOSURE_DIR: %v", d)
return http.FileServer(http.Dir(d)), nil
}
if root == "" {
// TODO: see if they're compiled in, and serve from that.
// But for now, assume a redirector to their current location
// on the web.
return closureBaseURL, nil
fs, err := closurestatic.FileSystem()
if err == os.ErrNotExist {
log.Printf("ui: no configured setting or embedded resources; serving Closure via %v", closureBaseURL)
return closureBaseURL, nil
}
if err != nil {
return nil, fmt.Errorf("error loading embedded Closure zip file: %v", err)
}
log.Printf("ui: serving Closure from embedded resources")
return http.FileServer(fs), nil
}
if strings.HasPrefix(root, "http") {
log.Printf("ui: serving Closure using redirects to %v", root)
return closureRedirector(root), nil
}
fi, err := os.Stat(root)
@ -198,11 +210,13 @@ func (ui *UIHandler) makeClosureHandler(root string) (http.Handler, error) {
if !fi.IsDir() {
return nil, errors.New("not a directory")
}
_, err = os.Stat(filepath.Join(root, "closure", "goog", "base.js"))
closureRoot := filepath.Join(root, "third_party", "closure", "lib", "closure")
_, err = os.Stat(filepath.Join(closureRoot, "goog", "base.js"))
if err != nil {
return nil, fmt.Errorf("directory doesn't contain closure/goog/base.js; wrong directory?")
}
return http.FileServer(http.Dir(filepath.Join(root, "closure"))), nil
log.Printf("ui: serving Closure from disk: %v", closureRoot)
return http.FileServer(http.Dir(closureRoot)), nil
}
const closureBaseURL closureRedirector = "https://closure-library.googlecode.com/git"

113
server/camlistored/ui/closure/closure.go
View File

@ -17,9 +17,20 @@ limitations under the License.
package closure
import (
"bytes"
"archive/zip"
"sync"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"strings"
"time"
)
// ZipData is either the empty string (when compiling with "go get",
@ -27,10 +38,108 @@ import (
// of the Closure library (when using make.go, which puts an extra
// file in this package containing an init function to set ZipData).
var ZipData string
var ZipModTime time.Time
func Zip() (*zip.Reader, error) {
func FileSystem() (http.FileSystem, error) {
if ZipData == "" {
return nil, os.ErrNotExist
}
return nil, errors.New("TODO: implement")
zr, err := zip.NewReader(strings.NewReader(ZipData), int64(len(ZipData)))
if err != nil {
return nil, err
}
m := make(map[string]*fileInfo)
for _, zf := range zr.File {
if !strings.HasPrefix(zf.Name, "closure/") {
continue
}
fi, err := newFileInfo(zf)
if err != nil {
return nil, fmt.Errorf("Error reading zip file %q: %v", zf.Name, err)
}
m[strings.TrimPrefix(zf.Name, "closure")] = fi
}
return &fs{zr, m}, nil
}
type fs struct {
zr *zip.Reader
m map[string]*fileInfo // keyed by what Open gets. see Open's comment.
}
var nopCloser = ioutil.NopCloser(nil)
// Open is called with names like "/goog/base.js", but the zip contains Files named like "closure/goog/base.js".
func (s *fs) Open(name string) (http.File, error) {
fi, ok := s.m[name]
if !ok {
return nil, os.ErrNotExist
}
return &file{fileInfo: fi}, nil
}
// a file is an http.File, wrapping a *fileInfo with a lazily-constructed SectionReader.
type file struct {
*fileInfo
once sync.Once // for making the SectionReader
sr *io.SectionReader
}
func (f *file) Read(p []byte) (n int, err error) {
f.once.Do(f.initReader)
return f.sr.Read(p)
}
func (f *file) Seek(offset int64, whence int) (ret int64, err error) {
f.once.Do(f.initReader)
return f.sr.Seek(offset, whence)
}
func (f *file) initReader() {
f.sr = io.NewSectionReader(f.fileInfo.ra, 0, f.Size())
}
func newFileInfo(zf *zip.File) (*fileInfo, error) {
rc, err := zf.Open()
if err != nil {
return nil, err
}
all, err := ioutil.ReadAll(rc)
if err != nil {
return nil, err
}
rc.Close()
return &fileInfo{
fullName: zf.Name,
regdata: all,
Closer: nopCloser,
ra: bytes.NewReader(all),
}, nil
}
type fileInfo struct {
fullName string
regdata []byte // non-nil if regular file
ra io.ReaderAt // over regdata
io.Closer
}
func (f *fileInfo) IsDir() bool { return f.regdata == nil }
func (f *fileInfo) Size() int64 { return int64(len(f.regdata)) }
func (f *fileInfo) ModTime() time.Time { return ZipModTime }
func (f *fileInfo) Name() string { return path.Base(f.fullName) }
func (f *fileInfo) Stat() (os.FileInfo, error) { return f, nil }
func (f *fileInfo) Sys() interface{} { return nil }
func (f *fileInfo) Readdir(count int) ([]os.FileInfo, error) {
// TODO: implement.
return nil, errors.New("TODO")
}
func (f *fileInfo) Mode() os.FileMode {
if f.IsDir() {
return 0755 | os.ModeDir
}
return 0644
}