diff --git a/app/publisher/main.go b/app/publisher/main.go index b198f52f0..65e75cc8c 100644 --- a/app/publisher/main.go +++ b/app/publisher/main.go @@ -940,7 +940,7 @@ func (pr *publishRequest) serveFileDownload(des *search.DescribedBlob) { Cache: pr.ph.cache, ForceMIME: mimeType, } - dh.ServeHTTP(pr.rw, pr.req, fileref) + dh.ServeFile(pr.rw, pr.req, fileref) } // Given a described blob, optionally follows a camliContent and diff --git a/make.go b/make.go index 10f05fbcf..a838bc0c9 100644 --- a/make.go +++ b/make.go @@ -457,8 +457,12 @@ func genPublisherJS(gopherjsBin string) error { // modtime of the existing gopherjs.js if there was no reason to. output := filepath.Join(buildSrcDir, filepath.FromSlash(publisherJS)) tmpOutput := output + ".new" - // TODO(mpl): maybe not with -m when building for devcam. - args := []string{"build", "--tags", "nocgo", "-m", "-o", tmpOutput, "camlistore.org/app/publisher/js"} + args := []string{"build", "--tags", "nocgo"} + if *embedResources { + // when embedding for "production", use -m to minify the javascript output + args = append(args, "-m") + } + args = append(args, "-o", tmpOutput, "camlistore.org/app/publisher/js") cmd := exec.Command(gopherjsBin, args...) cmd.Env = append(cleanGoEnv(), "GOPATH="+buildGoPath, @@ -530,8 +534,12 @@ func genWebUIJS(gopherjsBin string) error { // modtime of the existing goui.js if there was no reason to. output := filepath.Join(buildSrcDir, filepath.FromSlash(gopherjsUI)) tmpOutput := output + ".new" - // TODO(mpl): maybe not with -m when building for devcam. - args := []string{"build", "--tags", "nocgo", "-m", "-o", tmpOutput, "camlistore.org/server/camlistored/ui/goui"} + args := []string{"build", "--tags", "nocgo"} + if *embedResources { + // when embedding for "production", use -m to minify the javascript output + args = append(args, "-m") + } + args = append(args, "-o", tmpOutput, "camlistore.org/server/camlistored/ui/goui") cmd := exec.Command(gopherjsBin, args...) cmd.Env = append(cleanGoEnv(), "GOPATH="+buildGoPath, diff --git a/pkg/server/download.go b/pkg/server/download.go index 15bcad16f..fe13569f7 100644 --- a/pkg/server/download.go +++ b/pkg/server/download.go @@ -17,16 +17,19 @@ limitations under the License. package server import ( + "archive/zip" "fmt" "io" "log" "net/http" "os" + "regexp" "strings" "time" "camlistore.org/pkg/blob" "camlistore.org/pkg/blobserver" + "camlistore.org/pkg/httputil" "camlistore.org/pkg/magic" "camlistore.org/pkg/schema" "camlistore.org/pkg/search" @@ -34,9 +37,20 @@ import ( "golang.org/x/net/context" ) -const oneYear = 365 * 86400 * time.Second +const ( + oneYear = 365 * 86400 * time.Second + downloadTimeLayout = "20060102150405" +) -var debugPack = strings.Contains(os.Getenv("CAMLI_DEBUG_X"), "packserve") +var ( + debugPack = strings.Contains(os.Getenv("CAMLI_DEBUG_X"), "packserve") + + // Download URL suffix: + // $1: blobref (checked in download handler) + // $2: TODO. optional "/filename" to be sent as recommended download name, + // if sane looking + downloadPattern = regexp.MustCompile(`^download/([^/]+)(/.*)?$`) +) type DownloadHandler struct { Fetcher blob.Fetcher @@ -55,19 +69,20 @@ func (dh *DownloadHandler) blobSource() blob.Fetcher { } type fileInfo struct { - mime string - name string - size int64 - rs io.ReadSeeker - close func() error // release the rs - whyNot string // for testing, why fileInfoPacked failed. + mime string + name string + size int64 + modtime time.Time + rs io.ReadSeeker + close func() error // release the rs + whyNot string // for testing, why fileInfoPacked failed. } -func (dh *DownloadHandler) fileInfo(req *http.Request, file blob.Ref) (fi fileInfo, packed bool, err error) { +func (dh *DownloadHandler) fileInfo(r *http.Request, file blob.Ref) (fi fileInfo, packed bool, err error) { ctx := context.TODO() // Fast path for blobpacked. - fi, ok := fileInfoPacked(ctx, dh.Search, dh.Fetcher, req, file) + fi, ok := fileInfoPacked(ctx, dh.Search, dh.Fetcher, r, file) if debugPack { log.Printf("download.go: fileInfoPacked: ok=%v, %+v", ok, fi) } @@ -86,16 +101,17 @@ func (dh *DownloadHandler) fileInfo(req *http.Request, file blob.Ref) (fi fileIn mime = "application/octet-stream" } return fileInfo{ - mime: mime, - name: fr.FileName(), - size: fr.Size(), - rs: fr, - close: fr.Close, + mime: mime, + name: fr.FileName(), + size: fr.Size(), + modtime: fr.ModTime(), + rs: fr, + close: fr.Close, }, false, nil } // Fast path for blobpacked. -func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, req *http.Request, file blob.Ref) (packFileInfo fileInfo, ok bool) { +func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, r *http.Request, file blob.Ref) (packFileInfo fileInfo, ok bool) { if sh == nil { return fileInfo{whyNot: "no search"}, false } @@ -103,7 +119,7 @@ func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, r if !ok { return fileInfo{whyNot: "fetcher type"}, false } - if req != nil && req.Header.Get("Range") != "" { + if r != nil && r.Header.Get("Range") != "" { // TODO: not handled yet. Maybe not even important, // considering rarity. return fileInfo{whyNot: "range header"}, false @@ -135,34 +151,70 @@ func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, r log.Printf("ui: fileInfoPacked: skipping fast path due to error from WholeRefFetcher (%T): %v", src, err) return fileInfo{whyNot: "WholeRefFetcher error"}, false } + modtime := fi.ModTime + if modtime.IsAnyZero() { + modtime = fi.Time + } return fileInfo{ - mime: fi.MIMEType, - name: fi.FileName, - size: fi.Size, - rs: readerutil.NewFakeSeeker(rc, fi.Size-offset), - close: rc.Close, + mime: fi.MIMEType, + name: fi.FileName, + size: fi.Size, + modtime: modtime.Time(), + rs: readerutil.NewFakeSeeker(rc, fi.Size-offset), + close: rc.Close, }, true } -func (dh *DownloadHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, file blob.Ref) { - if req.Method != "GET" && req.Method != "HEAD" { - http.Error(rw, "Invalid download method", http.StatusBadRequest) - return - } - if req.Header.Get("If-Modified-Since") != "" { - // Immutable, so any copy's a good copy. - rw.WriteHeader(http.StatusNotModified) +// ServeHTTP answers the following queries: +// +// POST: +// ?files=sha1-foo,sha1-bar,sha1-baz +// Creates a zip archive of the provided files and serves it in the response. +// +// GET: +// / +// Serves the file described by the requested file schema blobref. +func (dh *DownloadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" { + dh.serveZip(w, r) return } - fi, packed, err := dh.fileInfo(req, file) + suffix := httputil.PathSuffix(r) + m := downloadPattern.FindStringSubmatch(suffix) + if m == nil { + httputil.ErrorRouting(w, r) + return + } + file, ok := blob.Parse(m[1]) + if !ok { + http.Error(w, "Invalid blobref", http.StatusBadRequest) + return + } + // TODO(mpl): make use of m[2] (the optional filename). + dh.ServeFile(w, r, file) +} + +func (dh *DownloadHandler) ServeFile(w http.ResponseWriter, r *http.Request, file blob.Ref) { + if r.Method != "GET" && r.Method != "HEAD" { + http.Error(w, "Invalid download method", http.StatusBadRequest) + return + } + + if r.Header.Get("If-Modified-Since") != "" { + // Immutable, so any copy's a good copy. + w.WriteHeader(http.StatusNotModified) + return + } + + fi, packed, err := dh.fileInfo(r, file) if err != nil { - http.Error(rw, "Can't serve file: "+err.Error(), http.StatusInternalServerError) + http.Error(w, "Can't serve file: "+err.Error(), http.StatusInternalServerError) return } defer fi.close() - h := rw.Header() + h := w.Header() h.Set("Content-Length", fmt.Sprint(fi.size)) h.Set("Expires", time.Now().Add(oneYear).Format(http.TimeFormat)) h.Set("Content-Type", fi.mime) @@ -179,11 +231,11 @@ func (dh *DownloadHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, if fileName == "" { fileName = "file-" + file.String() + ".dat" } - rw.Header().Set("Content-Disposition", "attachment; filename="+fileName) + w.Header().Set("Content-Disposition", "attachment; filename="+fileName) } - if req.Method == "HEAD" && req.FormValue("verifycontents") != "" { - vbr, ok := blob.Parse(req.FormValue("verifycontents")) + if r.Method == "HEAD" && r.FormValue("verifycontents") != "" { + vbr, ok := blob.Parse(r.FormValue("verifycontents")) if !ok { return } @@ -193,10 +245,119 @@ func (dh *DownloadHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, } io.Copy(hash, fi.rs) // ignore errors, caught later if vbr.HashMatches(hash) { - rw.Header().Set("X-Camli-Contents", vbr.String()) + w.Header().Set("X-Camli-Contents", vbr.String()) } return } - http.ServeContent(rw, req, "", time.Now(), fi.rs) + http.ServeContent(w, r, "", time.Now(), fi.rs) +} + +// statFiles stats the given refs and returns an error if any one of them is not +// found. +// It is the responsibility of the caller to check that dh.blobSource() is a +// blobserver.BlobStatter. +func (dh *DownloadHandler) statFiles(refs []blob.Ref) error { + statter, _ := dh.blobSource().(blobserver.BlobStatter) + statted := make(map[blob.Ref]bool) + ch := make(chan (blob.SizedRef)) + errc := make(chan (error)) + go func() { + err := statter.StatBlobs(ch, refs) + close(ch) + errc <- err + + }() + for sbr := range ch { + statted[sbr.Ref] = true + } + if err := <-errc; err != nil { + log.Printf("Error statting blob files for download archive: %v", err) + return fmt.Errorf("error looking for files") + } + for _, v := range refs { + if _, ok := statted[v]; !ok { + return fmt.Errorf("%q was not found", v) + } + } + return nil +} + +// serveZip creates a zip archive from the files provided as +// ?files=sha1-foo,sha1-bar,... and serves it as the response. +func (dh *DownloadHandler) serveZip(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "Invalid download method", http.StatusBadRequest) + return + } + + filesValue := r.FormValue("files") + if filesValue == "" { + http.Error(w, "No files blobRefs specified", http.StatusBadRequest) + return + } + files := strings.Split(filesValue, ",") + + var refs []blob.Ref + for _, file := range files { + br, ok := blob.Parse(file) + if !ok { + http.Error(w, fmt.Sprintf("%q is not a valid blobRef", file), http.StatusBadRequest) + return + } + refs = append(refs, br) + } + + // We check as many things as we can before writing the zip, because + // once we start sending a response we can't http.Error anymore. + // TODO(mpl): instead of just statting, read the files (from the + // blobSource, which should be Cache then Fetcher), and write them to the + // Cache. + _, ok := dh.blobSource().(blobserver.BlobStatter) + if ok { + if err := dh.statFiles(refs); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + + // TODO(mpl): do not zip if only one file is requested? + h := w.Header() + h.Set("Content-Type", "application/zip") + zipName := "camli-download-" + time.Now().Format(downloadTimeLayout) + ".zip" + h.Set("Content-Disposition", "attachment; filename="+zipName) + zw := zip.NewWriter(w) + + zipFile := func(br blob.Ref) error { + fi, _, err := dh.fileInfo(r, br) + if err != nil { + return err + } + defer fi.close() + zh := &zip.FileHeader{ + Name: fi.name, + Method: zip.Store, + } + zh.SetModTime(fi.modtime.UTC()) + zfh, err := zw.CreateHeader(zh) + if err != nil { + return err + } + _, err = io.Copy(zfh, fi.rs) + if err != nil { + return err + } + return nil + } + for _, br := range refs { + if err := zipFile(br); err != nil { + log.Printf("error zipping %v: %v", br, err) + // http.Error is of no use since we've already started sending a response + panic(http.ErrAbortHandler) + } + } + if err := zw.Close(); err != nil { + log.Printf("error closing zip stream: %v", err) + panic(http.ErrAbortHandler) + } } diff --git a/pkg/server/share.go b/pkg/server/share.go index 5dd479cb6..f43fa668e 100644 --- a/pkg/server/share.go +++ b/pkg/server/share.go @@ -240,7 +240,7 @@ func handleGetViaSharing(rw http.ResponseWriter, req *http.Request, Fetcher: fetcher, // TODO(aa): It would be nice to specify a local cache here, as the UI handler does. } - dh.ServeHTTP(rw, req, blobRef) + dh.ServeFile(rw, req, blobRef) } else { gethandler.ServeBlobRef(rw, req, blobRef, fetcher) } diff --git a/pkg/server/ui.go b/pkg/server/ui.go index 354a2ef18..1c93a2e4d 100644 --- a/pkg/server/ui.go +++ b/pkg/server/ui.go @@ -53,15 +53,8 @@ import ( ) var ( - staticFilePattern = regexp.MustCompile(`^([a-zA-Z0-9\-\_\.]+\.(html|js|css|png|jpg|gif|svg))$`) - identOrDotPattern = regexp.MustCompile(`^[a-zA-Z\_]+(\.[a-zA-Z\_]+)*$`) - - // Download URL suffix: - // $1: blobref (checked in download handler) - // $2: optional "/filename" to be sent as recommended download name, - // if sane looking - downloadPattern = regexp.MustCompile(`^download/([^/]+)(/.*)?$`) - + staticFilePattern = regexp.MustCompile(`^([a-zA-Z0-9\-\_\.]+\.(html|js|css|png|jpg|gif|svg))$`) + identOrDotPattern = regexp.MustCompile(`^[a-zA-Z\_]+(\.[a-zA-Z\_]+)*$`) thumbnailPattern = regexp.MustCompile(`^thumbnail/([^/]+)(/.*)?$`) treePattern = regexp.MustCompile(`^tree/([^/]+)(/.*)?$`) closurePattern = regexp.MustCompile(`^closure/(([^/]+)(/.*)?)$`) @@ -504,22 +497,9 @@ func (ui *UIHandler) discovery() *camtypes.UIDiscovery { return uiDisco } -func (ui *UIHandler) serveDownload(rw http.ResponseWriter, req *http.Request) { +func (ui *UIHandler) serveDownload(w http.ResponseWriter, r *http.Request) { if ui.root.Storage == nil { - http.Error(rw, "No BlobRoot configured", 500) - return - } - - suffix := httputil.PathSuffix(req) - m := downloadPattern.FindStringSubmatch(suffix) - if m == nil { - httputil.ErrorRouting(rw, req) - return - } - - fbr, ok := blob.Parse(m[1]) - if !ok { - http.Error(rw, "Invalid blobref", 400) + http.Error(w, "No BlobRoot configured", 500) return } @@ -528,7 +508,7 @@ func (ui *UIHandler) serveDownload(rw http.ResponseWriter, req *http.Request) { Search: ui.search, Cache: ui.Cache, } - dh.ServeHTTP(rw, req, fbr) + dh.ServeHTTP(w, r) } func (ui *UIHandler) serveThumbnail(rw http.ResponseWriter, req *http.Request) { diff --git a/server/camlistored/ui/goui/downloadbutton/downloadbutton.go b/server/camlistored/ui/goui/downloadbutton/downloadbutton.go new file mode 100644 index 000000000..3f6cbb8a2 --- /dev/null +++ b/server/camlistored/ui/goui/downloadbutton/downloadbutton.go @@ -0,0 +1,149 @@ +/* +Copyright 2017 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package downloadbutton provides a Button element that is used in the sidebar of +// the web UI, to download as a zip file all selected files. +package downloadbutton + +import ( + "fmt" + "strings" + + "camlistore.org/pkg/blob" + + "github.com/myitcv/gopherjs/react" + "honnef.co/go/js/dom" +) + +// New returns the button element. It should be used as the entry point, to +// create the needed React element. +// +// key is the id for when the button is in a list, see +// https://facebook.github.io/react/docs/lists-and-keys.html +// +// config is the web UI config that was fetched from the server. +// +// getSelection returns the list of files (blobRefs) selected for downloading. +func New(key string, config map[string]string, getSelection func() []string) react.Element { + if config == nil { + fmt.Println("Nil config for DownloadItemsBtn") + return nil + } + downloadHelper, ok := config["downloadHelper"] + if !ok { + fmt.Println("No downloadHelper in config for DownloadItemsBtn") + return nil + } + if getSelection == nil { + fmt.Println("Nil getSelection for DownloadItemsBtn") + return nil + } + if key == "" { + // A key is only needed in the context of a list, which is why + // it is up to the caller to choose it. Just creating it here for + // the sake of consistency. + key = "downloadItemsButton" + } + props := DownloadItemsBtnProps{ + Key: key, + GetSelection: getSelection, + Config: config, + downloadHelper: downloadHelper, + } + return DownloadItemsBtn(props).Render() +} + +// DownloadItemsBtnDef is the definition for the button, that Renders as a React +// Button. +type DownloadItemsBtnDef struct { + react.ComponentDef +} + +type DownloadItemsBtnProps struct { + // Key is the id for when the button is in a list, see + // https://facebook.github.io/react/docs/lists-and-keys.html + Key string + // GetSelection returns the list of files (blobRefs) selected + // for downloading. + GetSelection func() []string + // Config is the web UI config that was fetched from the server. + Config map[string]string + downloadHelper string +} + +func (p *DownloadItemsBtnDef) Props() DownloadItemsBtnProps { + uprops := p.ComponentDef.Props() + return uprops.(DownloadItemsBtnProps) +} + +func DownloadItemsBtn(p DownloadItemsBtnProps) *DownloadItemsBtnDef { + res := &DownloadItemsBtnDef{} + + react.BlessElement(res, p) + + return res +} + +func (d *DownloadItemsBtnDef) Render() react.Element { + return react.Button( + react.ButtonProps(func(bp *react.ButtonPropsDef) { + bp.OnClick = d.handleDownloadSelection + bp.Key = d.Props().Key + }), + react.S("Download"), + ) +} + +func (d *DownloadItemsBtnDef) handleDownloadSelection(*react.SyntheticMouseEvent) { + // Note: there's a "memleak", as in: until the selection is cleared and + // another one is started, this button stays allocated. It is of no + // consequence in this case as we don't allocate a lot for this element (in + // previous experiments where the zip archive was in memory, the leak was + // definitely noticeable then), but it is something to keep in mind for + // future elements. + go func() { + if err := d.downloadSelection(); err != nil { + dom.GetWindow().Alert(fmt.Sprintf("%v", err)) + } + }() +} + +func (d *DownloadItemsBtnDef) downloadSelection() error { + selection := d.Props().GetSelection() + downloadPrefix := d.Props().downloadHelper + fileRefs := []string{} + for _, file := range selection { + ref, ok := blob.Parse(file) + if !ok { + return fmt.Errorf("Cannot download %q, not a valid blobRef\n", file) + } + fileRefs = append(fileRefs, ref.String()) + } + + el := dom.GetWindow().Document().CreateElement("input") + input := el.(*dom.HTMLInputElement) + input.Type = "text" + input.Name = "files" + input.Value = strings.Join(fileRefs, ",") + + el = dom.GetWindow().Document().CreateElement("form") + form := el.(*dom.HTMLFormElement) + form.Action = downloadPrefix + form.Method = "POST" + form.AppendChild(input) + form.Submit() + return nil +} diff --git a/server/camlistored/ui/goui/main.go b/server/camlistored/ui/goui/main.go index 317d1ed7c..4fe58a2cd 100644 --- a/server/camlistored/ui/goui/main.go +++ b/server/camlistored/ui/goui/main.go @@ -20,12 +20,14 @@ package main import ( "camlistore.org/server/camlistored/ui/goui/aboutdialog" + "camlistore.org/server/camlistored/ui/goui/downloadbutton" "github.com/gopherjs/gopherjs/js" ) func main() { js.Global.Set("goreact", map[string]interface{}{ - "AboutMenuItem": aboutdialog.New, + "AboutMenuItem": aboutdialog.New, + "DownloadItemsBtn": downloadbutton.New, }) } diff --git a/server/camlistored/ui/index.js b/server/camlistored/ui/index.js index 9550040cb..dcc4c6fa4 100644 --- a/server/camlistored/ui/index.js +++ b/server/camlistored/ui/index.js @@ -1056,6 +1056,27 @@ cam.IndexPage = React.createClass({ ); }, + getDownloadSelectionItem_: function() { + return goreact.DownloadItemsBtn('donwloadBtnSidebar', + this.props.config, + // TODO(mpl): I'm doing the selection business in javascript for now, + // since we already have the search session results handy. + // It shouldn't be any problem to move it to Go later. + function() { + var selection = goog.object.getKeys(this.state.selection); + var files = []; + selection.forEach(function(br) { + var meta = this.childSearchSession_.getResolvedMeta(br); + if (!meta || !meta.file || !meta.file.fileName) { + // TODO(mpl): only do direct files for now. maybe recurse later. + return; + } + files.push(meta.blobRef); + }.bind(this)) + return files; + }.bind(this)); + }, + getSidebar_: function(selectedAspect) { if (selectedAspect) { if (selectedAspect.fragment == 'search' || selectedAspect.fragment == 'contents') { @@ -1082,6 +1103,7 @@ cam.IndexPage = React.createClass({ this.getRemoveSelectionFromSetItem_(), this.getDeleteSelectionItem_(), this.getViewOriginalSelectionItem_(), + this.getDownloadSelectionItem_(), ].filter(goog.functions.identity), selectedItems: this.state.selection });