Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge "pkg/server: add files "zipper" to DownloadHandler"

This commit is contained in:
Mathieu Lonjaret 2017-03-17 13:36:40 +00:00 committed by Gerrit Code Review
parent 640fd08296 71090f7c80
commit 75424def01
8 changed files with 392 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/publisher/main.go
View File

@ -940,7 +940,7 @@ func (pr *publishRequest) serveFileDownload(des *search.DescribedBlob) {
Cache: pr.ph.cache,
ForceMIME: mimeType,
}
dh.ServeHTTP(pr.rw, pr.req, fileref)
dh.ServeFile(pr.rw, pr.req, fileref)
}
// Given a described blob, optionally follows a camliContent and

16
make.go
View File

@ -457,8 +457,12 @@ func genPublisherJS(gopherjsBin string) error {
// modtime of the existing gopherjs.js if there was no reason to.
output := filepath.Join(buildSrcDir, filepath.FromSlash(publisherJS))
tmpOutput := output + ".new"
// TODO(mpl): maybe not with -m when building for devcam.
args := []string{"build", "--tags", "nocgo", "-m", "-o", tmpOutput, "camlistore.org/app/publisher/js"}
args := []string{"build", "--tags", "nocgo"}
if *embedResources {
// when embedding for "production", use -m to minify the javascript output
args = append(args, "-m")
}
args = append(args, "-o", tmpOutput, "camlistore.org/app/publisher/js")
cmd := exec.Command(gopherjsBin, args...)
cmd.Env = append(cleanGoEnv(),
"GOPATH="+buildGoPath,
@ -530,8 +534,12 @@ func genWebUIJS(gopherjsBin string) error {
// modtime of the existing goui.js if there was no reason to.
output := filepath.Join(buildSrcDir, filepath.FromSlash(gopherjsUI))
tmpOutput := output + ".new"
// TODO(mpl): maybe not with -m when building for devcam.
args := []string{"build", "--tags", "nocgo", "-m", "-o", tmpOutput, "camlistore.org/server/camlistored/ui/goui"}
args := []string{"build", "--tags", "nocgo"}
if *embedResources {
// when embedding for "production", use -m to minify the javascript output
args = append(args, "-m")
}
args = append(args, "-o", tmpOutput, "camlistore.org/server/camlistored/ui/goui")
cmd := exec.Command(gopherjsBin, args...)
cmd.Env = append(cleanGoEnv(),
"GOPATH="+buildGoPath,

237
pkg/server/download.go
View File

@ -17,16 +17,19 @@ limitations under the License.
package server
import (
"archive/zip"
"fmt"
"io"
"log"
"net/http"
"os"
"regexp"
"strings"
"time"
"camlistore.org/pkg/blob"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/magic"
"camlistore.org/pkg/schema"
"camlistore.org/pkg/search"
@ -34,9 +37,20 @@ import (
"golang.org/x/net/context"
)
const oneYear = 365 * 86400 * time.Second
const (
oneYear = 365 * 86400 * time.Second
downloadTimeLayout = "20060102150405"
)
var debugPack = strings.Contains(os.Getenv("CAMLI_DEBUG_X"), "packserve")
var (
debugPack = strings.Contains(os.Getenv("CAMLI_DEBUG_X"), "packserve")
// Download URL suffix:
// $1: blobref (checked in download handler)
// $2: TODO. optional "/filename" to be sent as recommended download name,
// if sane looking
downloadPattern = regexp.MustCompile(`^download/([^/]+)(/.*)?$`)
)
type DownloadHandler struct {
Fetcher blob.Fetcher
@ -55,19 +69,20 @@ func (dh *DownloadHandler) blobSource() blob.Fetcher {
}
type fileInfo struct {
mime string
name string
size int64
rs io.ReadSeeker
close func() error // release the rs
whyNot string // for testing, why fileInfoPacked failed.
mime string
name string
size int64
modtime time.Time
rs io.ReadSeeker
close func() error // release the rs
whyNot string // for testing, why fileInfoPacked failed.
}
func (dh *DownloadHandler) fileInfo(req *http.Request, file blob.Ref) (fi fileInfo, packed bool, err error) {
func (dh *DownloadHandler) fileInfo(r *http.Request, file blob.Ref) (fi fileInfo, packed bool, err error) {
ctx := context.TODO()
// Fast path for blobpacked.
fi, ok := fileInfoPacked(ctx, dh.Search, dh.Fetcher, req, file)
fi, ok := fileInfoPacked(ctx, dh.Search, dh.Fetcher, r, file)
if debugPack {
log.Printf("download.go: fileInfoPacked: ok=%v, %+v", ok, fi)
}
@ -86,16 +101,17 @@ func (dh *DownloadHandler) fileInfo(req *http.Request, file blob.Ref) (fi fileIn
mime = "application/octet-stream"
}
return fileInfo{
mime: mime,
name: fr.FileName(),
size: fr.Size(),
rs: fr,
close: fr.Close,
mime: mime,
name: fr.FileName(),
size: fr.Size(),
modtime: fr.ModTime(),
rs: fr,
close: fr.Close,
}, false, nil
}
// Fast path for blobpacked.
func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, req *http.Request, file blob.Ref) (packFileInfo fileInfo, ok bool) {
func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, r *http.Request, file blob.Ref) (packFileInfo fileInfo, ok bool) {
if sh == nil {
return fileInfo{whyNot: "no search"}, false
}
@ -103,7 +119,7 @@ func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, r
if !ok {
return fileInfo{whyNot: "fetcher type"}, false
}
if req != nil && req.Header.Get("Range") != "" {
if r != nil && r.Header.Get("Range") != "" {
// TODO: not handled yet. Maybe not even important,
// considering rarity.
return fileInfo{whyNot: "range header"}, false
@ -135,34 +151,70 @@ func fileInfoPacked(ctx context.Context, sh *search.Handler, src blob.Fetcher, r
log.Printf("ui: fileInfoPacked: skipping fast path due to error from WholeRefFetcher (%T): %v", src, err)
return fileInfo{whyNot: "WholeRefFetcher error"}, false
}
modtime := fi.ModTime
if modtime.IsAnyZero() {
modtime = fi.Time
}
return fileInfo{
mime: fi.MIMEType,
name: fi.FileName,
size: fi.Size,
rs: readerutil.NewFakeSeeker(rc, fi.Size-offset),
close: rc.Close,
mime: fi.MIMEType,
name: fi.FileName,
size: fi.Size,
modtime: modtime.Time(),
rs: readerutil.NewFakeSeeker(rc, fi.Size-offset),
close: rc.Close,
}, true
}
func (dh *DownloadHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, file blob.Ref) {
if req.Method != "GET" && req.Method != "HEAD" {
http.Error(rw, "Invalid download method", http.StatusBadRequest)
return
}
if req.Header.Get("If-Modified-Since") != "" {
// Immutable, so any copy's a good copy.
rw.WriteHeader(http.StatusNotModified)
// ServeHTTP answers the following queries:
//
// POST:
// ?files=sha1-foo,sha1-bar,sha1-baz
// Creates a zip archive of the provided files and serves it in the response.
//
// GET:
// /<file-schema-blobref>
// Serves the file described by the requested file schema blobref.
func (dh *DownloadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" {
dh.serveZip(w, r)
return
}
fi, packed, err := dh.fileInfo(req, file)
suffix := httputil.PathSuffix(r)
m := downloadPattern.FindStringSubmatch(suffix)
if m == nil {
httputil.ErrorRouting(w, r)
return
}
file, ok := blob.Parse(m[1])
if !ok {
http.Error(w, "Invalid blobref", http.StatusBadRequest)
return
}
// TODO(mpl): make use of m[2] (the optional filename).
dh.ServeFile(w, r, file)
}
func (dh *DownloadHandler) ServeFile(w http.ResponseWriter, r *http.Request, file blob.Ref) {
if r.Method != "GET" && r.Method != "HEAD" {
http.Error(w, "Invalid download method", http.StatusBadRequest)
return
}
if r.Header.Get("If-Modified-Since") != "" {
// Immutable, so any copy's a good copy.
w.WriteHeader(http.StatusNotModified)
return
}
fi, packed, err := dh.fileInfo(r, file)
if err != nil {
http.Error(rw, "Can't serve file: "+err.Error(), http.StatusInternalServerError)
http.Error(w, "Can't serve file: "+err.Error(), http.StatusInternalServerError)
return
}
defer fi.close()
h := rw.Header()
h := w.Header()
h.Set("Content-Length", fmt.Sprint(fi.size))
h.Set("Expires", time.Now().Add(oneYear).Format(http.TimeFormat))
h.Set("Content-Type", fi.mime)
@ -179,11 +231,11 @@ func (dh *DownloadHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request,
if fileName == "" {
fileName = "file-" + file.String() + ".dat"
}
rw.Header().Set("Content-Disposition", "attachment; filename="+fileName)
w.Header().Set("Content-Disposition", "attachment; filename="+fileName)
}
if req.Method == "HEAD" && req.FormValue("verifycontents") != "" {
vbr, ok := blob.Parse(req.FormValue("verifycontents"))
if r.Method == "HEAD" && r.FormValue("verifycontents") != "" {
vbr, ok := blob.Parse(r.FormValue("verifycontents"))
if !ok {
return
}
@ -193,10 +245,119 @@ func (dh *DownloadHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request,
}
io.Copy(hash, fi.rs) // ignore errors, caught later
if vbr.HashMatches(hash) {
rw.Header().Set("X-Camli-Contents", vbr.String())
w.Header().Set("X-Camli-Contents", vbr.String())
}
return
}
http.ServeContent(rw, req, "", time.Now(), fi.rs)
http.ServeContent(w, r, "", time.Now(), fi.rs)
}
// statFiles stats the given refs and returns an error if any one of them is not
// found.
// It is the responsibility of the caller to check that dh.blobSource() is a
// blobserver.BlobStatter.
func (dh *DownloadHandler) statFiles(refs []blob.Ref) error {
statter, _ := dh.blobSource().(blobserver.BlobStatter)
statted := make(map[blob.Ref]bool)
ch := make(chan (blob.SizedRef))
errc := make(chan (error))
go func() {
err := statter.StatBlobs(ch, refs)
close(ch)
errc <- err
}()
for sbr := range ch {
statted[sbr.Ref] = true
}
if err := <-errc; err != nil {
log.Printf("Error statting blob files for download archive: %v", err)
return fmt.Errorf("error looking for files")
}
for _, v := range refs {
if _, ok := statted[v]; !ok {
return fmt.Errorf("%q was not found", v)
}
}
return nil
}
// serveZip creates a zip archive from the files provided as
// ?files=sha1-foo,sha1-bar,... and serves it as the response.
func (dh *DownloadHandler) serveZip(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Invalid download method", http.StatusBadRequest)
return
}
filesValue := r.FormValue("files")
if filesValue == "" {
http.Error(w, "No files blobRefs specified", http.StatusBadRequest)
return
}
files := strings.Split(filesValue, ",")
var refs []blob.Ref
for _, file := range files {
br, ok := blob.Parse(file)
if !ok {
http.Error(w, fmt.Sprintf("%q is not a valid blobRef", file), http.StatusBadRequest)
return
}
refs = append(refs, br)
}
// We check as many things as we can before writing the zip, because
// once we start sending a response we can't http.Error anymore.
// TODO(mpl): instead of just statting, read the files (from the
// blobSource, which should be Cache then Fetcher), and write them to the
// Cache.
_, ok := dh.blobSource().(blobserver.BlobStatter)
if ok {
if err := dh.statFiles(refs); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
// TODO(mpl): do not zip if only one file is requested?
h := w.Header()
h.Set("Content-Type", "application/zip")
zipName := "camli-download-" + time.Now().Format(downloadTimeLayout) + ".zip"
h.Set("Content-Disposition", "attachment; filename="+zipName)
zw := zip.NewWriter(w)
zipFile := func(br blob.Ref) error {
fi, _, err := dh.fileInfo(r, br)
if err != nil {
return err
}
defer fi.close()
zh := &zip.FileHeader{
Name: fi.name,
Method: zip.Store,
}
zh.SetModTime(fi.modtime.UTC())
zfh, err := zw.CreateHeader(zh)
if err != nil {
return err
}
_, err = io.Copy(zfh, fi.rs)
if err != nil {
return err
}
return nil
}
for _, br := range refs {
if err := zipFile(br); err != nil {
log.Printf("error zipping %v: %v", br, err)
// http.Error is of no use since we've already started sending a response
panic(http.ErrAbortHandler)
}
}
if err := zw.Close(); err != nil {
log.Printf("error closing zip stream: %v", err)
panic(http.ErrAbortHandler)
}
}

2
pkg/server/share.go
View File

@ -240,7 +240,7 @@ func handleGetViaSharing(rw http.ResponseWriter, req *http.Request,
Fetcher: fetcher,
// TODO(aa): It would be nice to specify a local cache here, as the UI handler does.
}
dh.ServeHTTP(rw, req, blobRef)
dh.ServeFile(rw, req, blobRef)
} else {
gethandler.ServeBlobRef(rw, req, blobRef, fetcher)
}

30
pkg/server/ui.go
View File

@ -53,15 +53,8 @@ import (
)
var (
staticFilePattern = regexp.MustCompile(`^([a-zA-Z0-9\-\_\.]+\.(html|js|css|png|jpg|gif|svg))$`)
identOrDotPattern = regexp.MustCompile(`^[a-zA-Z\_]+(\.[a-zA-Z\_]+)*$`)
// Download URL suffix:
// $1: blobref (checked in download handler)
// $2: optional "/filename" to be sent as recommended download name,
// if sane looking
downloadPattern = regexp.MustCompile(`^download/([^/]+)(/.*)?$`)
staticFilePattern = regexp.MustCompile(`^([a-zA-Z0-9\-\_\.]+\.(html|js|css|png|jpg|gif|svg))$`)
identOrDotPattern = regexp.MustCompile(`^[a-zA-Z\_]+(\.[a-zA-Z\_]+)*$`)
thumbnailPattern = regexp.MustCompile(`^thumbnail/([^/]+)(/.*)?$`)
treePattern = regexp.MustCompile(`^tree/([^/]+)(/.*)?$`)
closurePattern = regexp.MustCompile(`^closure/(([^/]+)(/.*)?)$`)
@ -504,22 +497,9 @@ func (ui *UIHandler) discovery() *camtypes.UIDiscovery {
return uiDisco
}
func (ui *UIHandler) serveDownload(rw http.ResponseWriter, req *http.Request) {
func (ui *UIHandler) serveDownload(w http.ResponseWriter, r *http.Request) {
if ui.root.Storage == nil {
http.Error(rw, "No BlobRoot configured", 500)
return
}
suffix := httputil.PathSuffix(req)
m := downloadPattern.FindStringSubmatch(suffix)
if m == nil {
httputil.ErrorRouting(rw, req)
return
}
fbr, ok := blob.Parse(m[1])
if !ok {
http.Error(rw, "Invalid blobref", 400)
http.Error(w, "No BlobRoot configured", 500)
return
}
@ -528,7 +508,7 @@ func (ui *UIHandler) serveDownload(rw http.ResponseWriter, req *http.Request) {
Search: ui.search,
Cache: ui.Cache,
}
dh.ServeHTTP(rw, req, fbr)
dh.ServeHTTP(w, r)
}
func (ui *UIHandler) serveThumbnail(rw http.ResponseWriter, req *http.Request) {

149
server/camlistored/ui/goui/downloadbutton/downloadbutton.go Normal file
View File

@ -0,0 +1,149 @@
/*
Copyright 2017 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package downloadbutton provides a Button element that is used in the sidebar of
// the web UI, to download as a zip file all selected files.
package downloadbutton
import (
"fmt"
"strings"
"camlistore.org/pkg/blob"
"github.com/myitcv/gopherjs/react"
"honnef.co/go/js/dom"
)
// New returns the button element. It should be used as the entry point, to
// create the needed React element.
//
// key is the id for when the button is in a list, see
// https://facebook.github.io/react/docs/lists-and-keys.html
//
// config is the web UI config that was fetched from the server.
//
// getSelection returns the list of files (blobRefs) selected for downloading.
func New(key string, config map[string]string, getSelection func() []string) react.Element {
if config == nil {
fmt.Println("Nil config for DownloadItemsBtn")
return nil
}
downloadHelper, ok := config["downloadHelper"]
if !ok {
fmt.Println("No downloadHelper in config for DownloadItemsBtn")
return nil
}
if getSelection == nil {
fmt.Println("Nil getSelection for DownloadItemsBtn")
return nil
}
if key == "" {
// A key is only needed in the context of a list, which is why
// it is up to the caller to choose it. Just creating it here for
// the sake of consistency.
key = "downloadItemsButton"
}
props := DownloadItemsBtnProps{
Key: key,
GetSelection: getSelection,
Config: config,
downloadHelper: downloadHelper,
}
return DownloadItemsBtn(props).Render()
}
// DownloadItemsBtnDef is the definition for the button, that Renders as a React
// Button.
type DownloadItemsBtnDef struct {
react.ComponentDef
}
type DownloadItemsBtnProps struct {
// Key is the id for when the button is in a list, see
// https://facebook.github.io/react/docs/lists-and-keys.html
Key string
// GetSelection returns the list of files (blobRefs) selected
// for downloading.
GetSelection func() []string
// Config is the web UI config that was fetched from the server.
Config map[string]string
downloadHelper string
}
func (p *DownloadItemsBtnDef) Props() DownloadItemsBtnProps {
uprops := p.ComponentDef.Props()
return uprops.(DownloadItemsBtnProps)
}
func DownloadItemsBtn(p DownloadItemsBtnProps) *DownloadItemsBtnDef {
res := &DownloadItemsBtnDef{}
react.BlessElement(res, p)
return res
}
func (d *DownloadItemsBtnDef) Render() react.Element {
return react.Button(
react.ButtonProps(func(bp *react.ButtonPropsDef) {
bp.OnClick = d.handleDownloadSelection
bp.Key = d.Props().Key
}),
react.S("Download"),
)
}
func (d *DownloadItemsBtnDef) handleDownloadSelection(*react.SyntheticMouseEvent) {
// Note: there's a "memleak", as in: until the selection is cleared and
// another one is started, this button stays allocated. It is of no
// consequence in this case as we don't allocate a lot for this element (in
// previous experiments where the zip archive was in memory, the leak was
// definitely noticeable then), but it is something to keep in mind for
// future elements.
go func() {
if err := d.downloadSelection(); err != nil {
dom.GetWindow().Alert(fmt.Sprintf("%v", err))
}
}()
}
func (d *DownloadItemsBtnDef) downloadSelection() error {
selection := d.Props().GetSelection()
downloadPrefix := d.Props().downloadHelper
fileRefs := []string{}
for _, file := range selection {
ref, ok := blob.Parse(file)
if !ok {
return fmt.Errorf("Cannot download %q, not a valid blobRef\n", file)
}
fileRefs = append(fileRefs, ref.String())
}
el := dom.GetWindow().Document().CreateElement("input")
input := el.(*dom.HTMLInputElement)
input.Type = "text"
input.Name = "files"
input.Value = strings.Join(fileRefs, ",")
el = dom.GetWindow().Document().CreateElement("form")
form := el.(*dom.HTMLFormElement)
form.Action = downloadPrefix
form.Method = "POST"
form.AppendChild(input)
form.Submit()
return nil
}

4
server/camlistored/ui/goui/main.go
View File

@ -20,12 +20,14 @@ package main
import (
"camlistore.org/server/camlistored/ui/goui/aboutdialog"
"camlistore.org/server/camlistored/ui/goui/downloadbutton"
"github.com/gopherjs/gopherjs/js"
)
func main() {
js.Global.Set("goreact", map[string]interface{}{
"AboutMenuItem": aboutdialog.New,
"AboutMenuItem": aboutdialog.New,
"DownloadItemsBtn": downloadbutton.New,
})
}

22
server/camlistored/ui/index.js
View File

@ -1056,6 +1056,27 @@ cam.IndexPage = React.createClass({
);
},
getDownloadSelectionItem_: function() {
return goreact.DownloadItemsBtn('donwloadBtnSidebar',
this.props.config,
// TODO(mpl): I'm doing the selection business in javascript for now,
// since we already have the search session results handy.
// It shouldn't be any problem to move it to Go later.
function() {
var selection = goog.object.getKeys(this.state.selection);
var files = [];
selection.forEach(function(br) {
var meta = this.childSearchSession_.getResolvedMeta(br);
if (!meta || !meta.file || !meta.file.fileName) {
// TODO(mpl): only do direct files for now. maybe recurse later.
return;
}
files.push(meta.blobRef);
}.bind(this))
return files;
}.bind(this));
},
getSidebar_: function(selectedAspect) {
if (selectedAspect) {
if (selectedAspect.fragment == 'search' || selectedAspect.fragment == 'contents') {
@ -1082,6 +1103,7 @@ cam.IndexPage = React.createClass({
this.getRemoveSelectionFromSetItem_(),
this.getDeleteSelectionItem_(),
this.getViewOriginalSelectionItem_(),
this.getDownloadSelectionItem_(),
].filter(goog.functions.identity),
selectedItems: this.state.selection
});