diff --git a/.gitignore b/.gitignore index a57249c37..bc90dd5a6 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ bin/cam* bin/devcam bin/*_* bin/hello +bin/publisher tmp server/camlistored/newui/all.js server/camlistored/newui/all.js.map diff --git a/app/publisher/fileembed.go b/app/publisher/fileembed.go new file mode 100644 index 000000000..6faf20ccf --- /dev/null +++ b/app/publisher/fileembed.go @@ -0,0 +1,28 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +#fileembed pattern .+\.(js|css|html|png|svg)$ +*/ +package main + +import ( + "camlistore.org/pkg/fileembed" +) + +// TODO(mpl): appengine case + +var Files = &fileembed.Files{} diff --git a/server/camlistored/ui/gallery.html b/app/publisher/gallery.html similarity index 83% rename from server/camlistored/ui/gallery.html rename to app/publisher/gallery.html index 85dbab549..90a54baa0 100644 --- a/server/camlistored/ui/gallery.html +++ b/app/publisher/gallery.html @@ -3,12 +3,6 @@ {{if $header := call .Header}} {{$header.Title}} - {{range $js := $header.JSDeps}} - - {{end}} - {{if $header.CamliClosure}} - - {{end}} {{range $css := $header.CSSFiles}} {{end}} @@ -54,14 +48,6 @@ {{end}} {{end}} - {{if $header.CamliClosure}} - {{if $header.ViewerIsOwner}} - - {{end}} - {{end}} {{end}} diff --git a/app/publisher/main.go b/app/publisher/main.go new file mode 100644 index 000000000..119582459 --- /dev/null +++ b/app/publisher/main.go @@ -0,0 +1,1004 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The publisher application serves and renders items published by Camlistore. +// That is, items that are children, through a (direct or not) camliPath relation, +// of a camliRoot node (a permanode with a camliRoot attribute set). +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "html" + "html/template" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + + "camlistore.org/pkg/app" + "camlistore.org/pkg/blob" + "camlistore.org/pkg/blobserver" + "camlistore.org/pkg/blobserver/localdisk" + "camlistore.org/pkg/buildinfo" + "camlistore.org/pkg/constants" + "camlistore.org/pkg/fileembed" + "camlistore.org/pkg/httputil" + "camlistore.org/pkg/publish" + "camlistore.org/pkg/search" + "camlistore.org/pkg/server" + "camlistore.org/pkg/sorted" + "camlistore.org/pkg/syncutil" + "camlistore.org/pkg/types/camtypes" + "camlistore.org/pkg/webserver" + + _ "camlistore.org/pkg/sorted/kvfile" +) + +var ( + flagVersion = flag.Bool("version", false, "show version") +) + +// TODO(mpl): logger with a "PUBLISHER" prefix. + +// config is used to unmarshal the application configuration JSON +// that we get from Camlistore when we request it at $CAMLI_APP_CONFIG_URL. +type config struct { + RootName string `json:"camliRoot"` // Publish root name (i.e. value of the camliRoot attribute on the root permanode). + MaxResizeBytes int64 `json:"maxResizeBytes,omitempty"` // See constants.DefaultMaxResizeMem + SourceRoot string `json:"sourceRoot,omitempty"` // Path to the app's resources dir, such as html and css files. + GoTemplate string `json:"goTemplate"` // Go html template to render the publication. + CacheRoot string `json:"cacheRoot,omitempty"` // Root path for the caching blobserver. No caching if empty. +} + +func appConfig() *config { + configURL := os.Getenv("CAMLI_APP_CONFIG_URL") + if configURL == "" { + log.Fatalf("Publisher application needs a CAMLI_APP_CONFIG_URL env var") + } + cl, err := app.Client() + if err != nil { + log.Fatalf("could not get a client to fetch extra config: %v", err) + } + conf := &config{} + if err := cl.GetJSON(configURL, conf); err != nil { + log.Fatalf("could not get app config at %v: %v", configURL, err) + } + return conf +} + +func main() { + flag.Parse() + + if *flagVersion { + fmt.Fprintf(os.Stderr, "publisher version: %s\nGo version: %s (%s/%s)\n", + buildinfo.Version(), runtime.Version(), runtime.GOOS, runtime.GOARCH) + return + } + + log.Printf("Starting publisher version %s; Go %s (%s/%s)", buildinfo.Version(), runtime.Version(), + runtime.GOOS, runtime.GOARCH) + + listenAddr, err := app.ListenAddress() + if err != nil { + log.Fatalf("Listen address: %v", err) + } + conf := appConfig() + ph := newPublishHandler(conf) + ph.initRootNode() + ws := webserver.New() + ws.Handle("/", ph) + if err := ws.Listen(listenAddr); err != nil { + log.Fatalf("Listen: %v", err) + } + + ws.Serve() +} + +func newPublishHandler(conf *config) *publishHandler { + cl, err := app.Client() + if err != nil { + log.Fatalf("could not get a client for the publish handler %v", err) + } + if conf.RootName == "" { + log.Fatal("camliRoot not found in the app configuration") + } + maxResizeBytes := conf.MaxResizeBytes + if maxResizeBytes == 0 { + maxResizeBytes = constants.DefaultMaxResizeMem + } + var CSSFiles []string + if conf.SourceRoot != "" { + appRoot := filepath.Join(conf.SourceRoot, "app", "publisher") + Files = &fileembed.Files{ + DirFallback: appRoot, + } + // TODO(mpl): Can I readdir by listing with "/" on Files, even with DirFallBack? + // Apparently not, but retry later. + dir, err := os.Open(appRoot) + if err != nil { + log.Fatal(err) + } + defer dir.Close() + names, err := dir.Readdirnames(-1) + if err != nil { + log.Fatal(err) + } + for _, v := range names { + if strings.HasSuffix(v, ".css") { + CSSFiles = append(CSSFiles, v) + } + } + } else { + Files.Listable = true + dir, err := Files.Open("/") + if err != nil { + log.Fatal(err) + } + defer dir.Close() + fis, err := dir.Readdir(-1) + if err != nil { + log.Fatal(err) + } + for _, v := range fis { + name := v.Name() + if strings.HasSuffix(name, ".css") { + CSSFiles = append(CSSFiles, name) + } + } + } + // TODO(mpl): add all htmls found in Files to the template if none specified? + if conf.GoTemplate == "" { + log.Fatal("a go template is required in the app configuration") + } + goTemplate, err := goTemplate(Files, conf.GoTemplate) + if err != nil { + log.Fatal(err) + } + serverURL := os.Getenv("CAMLI_API_HOST") + if serverURL == "" { + log.Fatal("CAMLI_API_HOST var not set") + } + var cache blobserver.Storage + var thumbMeta *server.ThumbMeta + if conf.CacheRoot != "" { + cache, err = localdisk.New(conf.CacheRoot) + if err != nil { + log.Fatalf("Could not create localdisk cache: %v", err) + } + thumbsCacheDir := filepath.Join(os.TempDir(), "camli-publisher-cache") + if err := os.MkdirAll(thumbsCacheDir, 0700); err != nil { + log.Fatalf("Could not create cache dir %s for %v publisher: %v", thumbsCacheDir, conf.RootName, err) + } + kv, err := sorted.NewKeyValue(map[string]interface{}{ + "type": "kv", + "file": filepath.Join(thumbsCacheDir, conf.RootName+"-thumbnails.kv"), + }) + if err != nil { + log.Fatalf("Could not create kv for %v's thumbs cache: %v", conf.RootName, err) + } + thumbMeta = server.NewThumbMeta(kv) + } + + return &publishHandler{ + rootName: conf.RootName, + cl: cl, + resizeSem: syncutil.NewSem(maxResizeBytes), + staticFiles: Files, + goTemplate: goTemplate, + CSSFiles: CSSFiles, + describedCache: make(map[string]*search.DescribedBlob), + cache: cache, + thumbMeta: thumbMeta, + } +} + +func goTemplate(files *fileembed.Files, templateFile string) (*template.Template, error) { + f, err := files.Open(templateFile) + if err != nil { + return nil, fmt.Errorf("Could not open template %v: %v", templateFile, err) + } + defer f.Close() + templateBytes, err := ioutil.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("Could not read template %v: %v", templateFile, err) + } + return template.Must(template.New("subject").Parse(string(templateBytes))), nil +} + +// We're using this interface in a publishHandler, instead of directly +// a *client.Client, so we can use a fake client in tests. +type client interface { + Search(req *search.SearchQuery) (*search.SearchResult, error) + Describe(req *search.DescribeRequest) (*search.DescribeResponse, error) + GetJSON(url string, data interface{}) error + Post(url string, bodyType string, body io.Reader) error + blob.Fetcher +} + +type publishHandler struct { + rootName string // Publish root name (i.e. value of the camliRoot attribute on the root permanode). + + rootNodeMu sync.Mutex + rootNode blob.Ref // Root permanode, origin of all camliPaths for this publish handler. + + cl client // Used for searching, and remote storage. + + staticFiles *fileembed.Files // For static resources. + goTemplate *template.Template // For publishing/rendering. + CSSFiles []string + resizeSem *syncutil.Sem // Limit peak RAM used by concurrent image thumbnail calls. + + describedCacheMu sync.RWMutex + describedCache map[string]*search.DescribedBlob // So that each item in a gallery does not actually require a describe round-trip. + + cache blobserver.Storage // For caching images and files, or nil. + thumbMeta *server.ThumbMeta // For keeping track of cached images, or nil. +} + +func (ph *publishHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ph.rootNodeMu.Lock() + if !ph.rootNode.Valid() { + // we want to retry doing this every time because the rootNode could have been created + // (by e.g. the owner) since last time. + err := ph.initRootNode() + if err != nil { + httputil.ServeError(w, r, fmt.Errorf("No publish root node: %v", err)) + ph.rootNodeMu.Unlock() + return + } + } + ph.rootNodeMu.Unlock() + + preq, err := ph.NewRequest(w, r) + if err != nil { + httputil.ServeError(w, r, fmt.Errorf("Could not create publish request: %v", err)) + return + } + preq.serveHTTP() +} + +func (ph *publishHandler) initRootNode() error { + var getRootNode = func() (blob.Ref, error) { + result, err := ph.camliRootQuery() + if err != nil { + return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.rootName, err) + } + if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() { + return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.rootName, os.ErrNotExist) + } + return result.Blobs[0].Blob, nil + } + node, err := getRootNode() + if err != nil { + return err + } + ph.rootNode = node + return nil +} + +func (ph *publishHandler) camliRootQuery() (*search.SearchResult, error) { + // TODO(mpl): I've voluntarily omitted the owner because it's not clear to + // me that we actually care about that. Same for signer in lookupPathTarget. + return ph.cl.Search(&search.SearchQuery{ + Limit: 1, + Constraint: &search.Constraint{ + Permanode: &search.PermanodeConstraint{ + Attr: "camliRoot", + Value: ph.rootName, + }, + }, + }) +} + +func (ph *publishHandler) lookupPathTarget(root blob.Ref, suffix string) (blob.Ref, error) { + if suffix == "" { + return root, nil + } + // TODO: verify it's optimized: http://camlistore.org/issue/405 + result, err := ph.cl.Search(&search.SearchQuery{ + Limit: 1, + Constraint: &search.Constraint{ + Permanode: &search.PermanodeConstraint{ + SkipHidden: true, + Relation: &search.RelationConstraint{ + Relation: "parent", + EdgeType: "camliPath:" + suffix, + Any: &search.Constraint{ + BlobRefPrefix: root.String(), + }, + }, + }, + }, + }) + if err != nil { + return blob.Ref{}, err + } + if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() { + return blob.Ref{}, os.ErrNotExist + } + return result.Blobs[0].Blob, nil +} + +// Given a blobref and a few hex characters of the digest of the next hop, return the complete +// blobref of the prefix, if that's a valid next hop. +func (ph *publishHandler) resolvePrefixHop(parent blob.Ref, prefix string) (child blob.Ref, err error) { + // TODO: this is a linear scan right now. this should be + // optimized to use a new database table of members so this is + // a quick lookup. in the meantime it should be in memcached + // at least. + if len(prefix) < 8 { + return blob.Ref{}, fmt.Errorf("Member prefix %q too small", prefix) + } + des, err := ph.describe(parent) + if err != nil { + return blob.Ref{}, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent) + } + if des.Permanode != nil { + cr, ok := des.ContentRef() + if ok && strings.HasPrefix(cr.Digest(), prefix) { + return cr, nil + } + for _, member := range des.Members() { + if strings.HasPrefix(member.BlobRef.Digest(), prefix) { + return member.BlobRef, nil + } + } + crdes, err := ph.describe(cr) + if err != nil { + return blob.Ref{}, fmt.Errorf("Failed to describe content %q of parent %q", cr, parent) + } + if crdes.Dir != nil { + return ph.resolvePrefixHop(cr, prefix) + } + } else if des.Dir != nil { + for _, child := range des.DirChildren { + if strings.HasPrefix(child.Digest(), prefix) { + return child, nil + } + } + } + return blob.Ref{}, fmt.Errorf("Member prefix %q not found in %q", prefix, parent) +} + +func (ph *publishHandler) describe(br blob.Ref) (*search.DescribedBlob, error) { + ph.describedCacheMu.RLock() + if des, ok := ph.describedCache[br.String()]; ok { + ph.describedCacheMu.RUnlock() + return des, nil + } + ph.describedCacheMu.RUnlock() + res, err := ph.cl.Describe(&search.DescribeRequest{ + BlobRef: br, + Depth: 1, + }) + if err != nil { + return nil, fmt.Errorf("Could not describe %v: %v", br, err) + } + return res.Meta[br.String()], nil +} + +func (ph *publishHandler) deepDescribe(br blob.Ref) (*search.DescribeResponse, error) { + res, err := ph.cl.Search(&search.SearchQuery{ + Constraint: &search.Constraint{ + BlobRefPrefix: br.String(), + CamliType: "permanode", + }, + Describe: &search.DescribeRequest{ + ThumbnailSize: 1000, + Depth: 1, + Rules: []*search.DescribeRule{ + { + Attrs: []string{"camliContent", "camliContentImage", "camliMember", "camliPath:*"}, + }, + }, + }, + Limit: -1, + }) + if err != nil { + return nil, fmt.Errorf("Could not deep describe %v: %v", br, err) + } + if res == nil || res.Describe == nil { + return nil, fmt.Errorf("no describe result for %v", br) + } + return res.Describe, nil +} + +// publishRequest is the state around a single HTTP request to the +// publish handler +type publishRequest struct { + ph *publishHandler + rw http.ResponseWriter + req *http.Request + base, suffix, subres string + rootpn blob.Ref + subject blob.Ref + inSubjectChain map[string]bool // blobref -> true + subjectBasePath string +} + +func (ph *publishHandler) NewRequest(rw http.ResponseWriter, req *http.Request) (*publishRequest, error) { + // splits a path request into its suffix and subresource parts. + // e.g. /blog/foo/camli/res/file/xxx -> ("foo", "file/xxx") + suffix, res := httputil.PathSuffix(req), "" + if strings.HasPrefix(suffix, "-/") { + suffix, res = "", suffix[2:] + } else if s := strings.SplitN(suffix, "/-/", 2); len(s) == 2 { + suffix, res = s[0], s[1] + } + + return &publishRequest{ + ph: ph, + rw: rw, + req: req, + suffix: suffix, + base: httputil.PathBase(req), + subres: res, + rootpn: ph.rootNode, + inSubjectChain: make(map[string]bool), + subjectBasePath: "", + }, nil +} + +func (pr *publishRequest) serveHTTP() { + if !pr.rootpn.Valid() { + pr.rw.WriteHeader(404) + return + } + + if pr.Debug() { + pr.rw.Header().Set("Content-Type", "text/html") + pr.pf("I am publish handler at base %q, serving root %q (permanode=%s), suffix %q, subreq %q
", + pr.base, pr.ph.rootName, pr.rootpn, html.EscapeString(pr.suffix), html.EscapeString(pr.subres)) + } + + if err := pr.findSubject(); err != nil { + if err == os.ErrNotExist { + pr.rw.WriteHeader(404) + return + } + log.Printf("Error looking up %s/%q: %v", pr.rootpn, pr.suffix, err) + pr.rw.WriteHeader(500) + return + } + + if pr.Debug() { + pr.pf("

Subject: %s

", pr.subject, pr.subject) + return + } + + switch pr.subresourceType() { + case "": + pr.serveSubjectTemplate() + case "b": + // TODO: download a raw blob + case "f": // file download + pr.serveSubresFileDownload() + case "i": // image, scaled + pr.serveSubresImage() + case "s": // static + pr.req.URL.Path = pr.subres[len("/=s"):] + if len(pr.req.URL.Path) <= 1 { + http.Error(pr.rw, "Illegal URL.", http.StatusNotFound) + return + } + file := pr.req.URL.Path[1:] + server.ServeStaticFile(pr.rw, pr.req, pr.ph.staticFiles, file) + case "z": + pr.serveZip() + default: + pr.rw.WriteHeader(400) + pr.pf("

Invalid or unsupported resource request.

") + } +} + +func (pr *publishRequest) Debug() bool { + return pr.req.FormValue("debug") == "1" +} + +var memberRE = regexp.MustCompile(`^/?h([0-9a-f]+)`) + +func (pr *publishRequest) findSubject() error { + if strings.HasPrefix(pr.suffix, "=s/") { + pr.subres = "/" + pr.suffix + return nil + } + + subject, err := pr.ph.lookupPathTarget(pr.rootpn, pr.suffix) + if err != nil { + return err + } + if strings.HasPrefix(pr.subres, "=z/") { + // this happens when we are at the root of the published path, + // e.g /base/suffix/-/=z/foo.zip + // so we need to reset subres as fullpath so that it is detected + // properly when switching on pr.subresourceType() + pr.subres = "/" + pr.subres + // since we return early, we set the subject because that is + // what is going to be used as a root node by the zip handler. + pr.subject = subject + return nil + } + + pr.inSubjectChain[subject.String()] = true + pr.subjectBasePath = pr.base + pr.suffix + + // Chase /h hops in suffix. + for { + m := memberRE.FindStringSubmatch(pr.subres) + if m == nil { + break + } + match, memberPrefix := m[0], m[1] + + if err != nil { + return fmt.Errorf("Error looking up potential member %q in describe of subject %q: %v", + memberPrefix, subject, err) + } + + subject, err = pr.ph.resolvePrefixHop(subject, memberPrefix) + if err != nil { + return err + } + pr.inSubjectChain[subject.String()] = true + pr.subres = pr.subres[len(match):] + pr.subjectBasePath = addPathComponent(pr.subjectBasePath, match) + } + + pr.subject = subject + return nil +} + +func (pr *publishRequest) subresourceType() string { + if len(pr.subres) >= 3 && strings.HasPrefix(pr.subres, "/=") { + return pr.subres[2:3] + } + return "" +} + +func (pr *publishRequest) pf(format string, args ...interface{}) { + fmt.Fprintf(pr.rw, format, args...) +} + +func addPathComponent(base, addition string) string { + if !strings.HasPrefix(addition, "/") { + addition = "/" + addition + } + if strings.Contains(base, "/-/") { + return base + addition + } + return base + "/-" + addition +} + +const ( + resSeparator = "/-" + digestPrefix = "h" + digestLen = 10 +) + +// var hopRE = regexp.MustCompile(fmt.Sprintf("^/%s([0-9a-f]{%d})", digestPrefix, digestLen)) + +func getFileInfo(item blob.Ref, peers map[string]*search.DescribedBlob) (path []blob.Ref, fi *camtypes.FileInfo, ok bool) { + described := peers[item.String()] + if described == nil || + described.Permanode == nil || + described.Permanode.Attr == nil { + return + } + contentRef := described.Permanode.Attr.Get("camliContent") + if contentRef == "" { + return + } + if cdes := peers[contentRef]; cdes != nil && cdes.File != nil { + return []blob.Ref{described.BlobRef, cdes.BlobRef}, cdes.File, true + } + return +} + +// serveSubjectTemplate creates the funcs to generate the PageHeader, PageFile, +// and pageMembers that can be used by the subject template, and serves the template. +func (pr *publishRequest) serveSubjectTemplate() { + res, err := pr.ph.deepDescribe(pr.subject) + if err != nil { + httputil.ServeError(pr.rw, pr.req, err) + return + } + pr.ph.cacheDescribed(res.Meta) + + subdes := res.Meta[pr.subject.String()] + if subdes.CamliType == "file" { + pr.serveFileDownload(subdes) + return + } + + headerFunc := func() *publish.PageHeader { + return pr.subjectHeader(res.Meta) + } + fileFunc := func() *publish.PageFile { + file, err := pr.subjectFile(res.Meta) + if err != nil { + log.Printf("%v", err) + return nil + } + return file + } + membersFunc := func() *publish.PageMembers { + members, err := pr.subjectMembers(res.Meta) + if err != nil { + log.Printf("%v", err) + return nil + } + return members + } + page := &publish.SubjectPage{ + Header: headerFunc, + File: fileFunc, + Members: membersFunc, + } + + err = pr.ph.goTemplate.Execute(pr.rw, page) + if err != nil { + log.Printf("Error serving subject template: %v", err) + http.Error(pr.rw, "Error serving template", http.StatusInternalServerError) + return + } +} + +const cacheSize = 1000 + +func (ph *publishHandler) cacheDescribed(described map[string]*search.DescribedBlob) { + ph.describedCacheMu.Lock() + defer ph.describedCacheMu.Unlock() + if len(ph.describedCache) > cacheSize { + ph.describedCache = described + return + } + for k, v := range described { + ph.describedCache[k] = v + } +} + +func (pr *publishRequest) serveFileDownload(des *search.DescribedBlob) { + fileref, fileinfo, ok := pr.fileSchemaRefFromBlob(des) + if !ok { + log.Printf("Didn't get file schema from described blob %q", des.BlobRef) + return + } + mime := "" + if fileinfo != nil && fileinfo.IsImage() { + mime = fileinfo.MIMEType + } + dh := &server.DownloadHandler{ + Fetcher: pr.ph.cl, + Cache: pr.ph.cache, + ForceMime: mime, + } + dh.ServeHTTP(pr.rw, pr.req, fileref) +} + +// Given a described blob, optionally follows a camliContent and +// returns the file's schema blobref and its fileinfo (if found). +func (pr *publishRequest) fileSchemaRefFromBlob(des *search.DescribedBlob) (fileref blob.Ref, fileinfo *camtypes.FileInfo, ok bool) { + if des == nil { + http.NotFound(pr.rw, pr.req) + return + } + if des.Permanode != nil { + // TODO: get "forceMime" attr out of the permanode? or + // fileName content-disposition? + if cref := des.Permanode.Attr.Get("camliContent"); cref != "" { + cbr, ok2 := blob.Parse(cref) + if !ok2 { + http.Error(pr.rw, "bogus camliContent", 500) + return + } + des = des.PeerBlob(cbr) + if des == nil { + http.Error(pr.rw, "camliContent not a peer in describe", 500) + return + } + } + } + if des.CamliType == "file" { + return des.BlobRef, des.File, true + } + http.Error(pr.rw, "failed to find fileSchemaRefFromBlob", 404) + return +} + +// subjectHeader returns the PageHeader corresponding to the described subject. +func (pr *publishRequest) subjectHeader(described map[string]*search.DescribedBlob) *publish.PageHeader { + subdes := described[pr.subject.String()] + header := &publish.PageHeader{ + Title: html.EscapeString(getTitle(subdes.BlobRef, described)), + CSSFiles: pr.cssFiles(), + Meta: func() string { + jsonRes, _ := json.MarshalIndent(described, "", " ") + return string(jsonRes) + }(), + Subject: pr.subject.String(), + } + return header +} + +func (pr *publishRequest) cssFiles() []string { + files := []string{} + for _, filename := range pr.ph.CSSFiles { + files = append(files, pr.staticPath(filename)) + } + return files +} + +func (pr *publishRequest) staticPath(fileName string) string { + return pr.base + "=s/" + fileName +} + +func getTitle(item blob.Ref, peers map[string]*search.DescribedBlob) string { + described := peers[item.String()] + if described == nil { + return "" + } + if described.Permanode != nil { + if t := described.Permanode.Attr.Get("title"); t != "" { + return t + } + if contentRef := described.Permanode.Attr.Get("camliContent"); contentRef != "" { + if cdes := peers[contentRef]; cdes != nil { + return getTitle(cdes.BlobRef, peers) + } + } + } + if described.File != nil { + return described.File.FileName + } + if described.Dir != nil { + return described.Dir.FileName + } + return "" +} + +// subjectFile returns the relevant PageFile if the described subject is a file permanode. +func (pr *publishRequest) subjectFile(described map[string]*search.DescribedBlob) (*publish.PageFile, error) { + subdes := described[pr.subject.String()] + contentRef, ok := subdes.ContentRef() + if !ok { + return nil, nil + } + fileDes, err := pr.ph.describe(contentRef) + if err != nil { + return nil, err + } + if fileDes.File == nil { + // most likely a dir + return nil, nil + } + + path := []blob.Ref{pr.subject, contentRef} + downloadURL := pr.SubresFileURL(path, fileDes.File.FileName) + thumbnailURL := "" + if fileDes.File.IsImage() { + thumbnailURL = pr.SubresThumbnailURL(path, fileDes.File.FileName, 600) + } + fileName := html.EscapeString(fileDes.File.FileName) + return &publish.PageFile{ + FileName: fileName, + Size: fileDes.File.Size, + MIMEType: fileDes.File.MIMEType, + IsImage: fileDes.File.IsImage(), + DownloadURL: downloadURL, + ThumbnailURL: thumbnailURL, + DomID: contentRef.DomID(), + Nav: func() *publish.Nav { + return nil + }, + }, nil +} + +func (pr *publishRequest) SubresFileURL(path []blob.Ref, fileName string) string { + return pr.SubresThumbnailURL(path, fileName, -1) +} + +func (pr *publishRequest) SubresThumbnailURL(path []blob.Ref, fileName string, maxDimen int) string { + var buf bytes.Buffer + resType := "i" + if maxDimen == -1 { + resType = "f" + } + fmt.Fprintf(&buf, "%s", pr.subjectBasePath) + if !strings.Contains(pr.subjectBasePath, "/-/") { + buf.Write([]byte("/-")) + } + for _, br := range path { + if pr.inSubjectChain[br.String()] { + continue + } + fmt.Fprintf(&buf, "/h%s", br.DigestPrefix(10)) + } + fmt.Fprintf(&buf, "/=%s", resType) + fmt.Fprintf(&buf, "/%s", url.QueryEscape(fileName)) + if maxDimen != -1 { + fmt.Fprintf(&buf, "?mw=%d&mh=%d", maxDimen, maxDimen) + } + return buf.String() +} + +// subjectMembers returns the relevant PageMembers if the described subject is a permanode with members. +func (pr *publishRequest) subjectMembers(resMap map[string]*search.DescribedBlob) (*publish.PageMembers, error) { + subdes := resMap[pr.subject.String()] + res, err := pr.ph.describeMembers(pr.subject) + if err != nil { + return nil, err + } + members := []*search.DescribedBlob{} + for _, v := range res.Blobs { + members = append(members, res.Describe.Meta[v.Blob.String()]) + } + if len(members) == 0 { + return nil, nil + } + + zipName := "" + if title := getTitle(subdes.BlobRef, resMap); title == "" { + zipName = "download.zip" + } else { + zipName = title + ".zip" + } + subjectPath := pr.subjectBasePath + if !strings.Contains(subjectPath, "/-/") { + subjectPath += "/-" + } + + return &publish.PageMembers{ + SubjectPath: subjectPath, + ZipName: zipName, + Members: members, + Description: func(member *search.DescribedBlob) string { + des := member.Description() + if des != "" { + des = " - " + des + } + return des + }, + Title: func(member *search.DescribedBlob) string { + memberTitle := getTitle(member.BlobRef, resMap) + if memberTitle == "" { + memberTitle = member.BlobRef.DigestPrefix(10) + } + return html.EscapeString(memberTitle) + }, + Path: func(member *search.DescribedBlob) string { + return pr.memberPath(member.BlobRef) + }, + DomID: func(member *search.DescribedBlob) string { + return member.DomID() + }, + FileInfo: func(member *search.DescribedBlob) *publish.MemberFileInfo { + if path, fileInfo, ok := getFileInfo(member.BlobRef, resMap); ok { + info := &publish.MemberFileInfo{ + FileName: fileInfo.FileName, + FileDomID: path[len(path)-1].DomID(), + FilePath: html.EscapeString(pr.SubresFileURL(path, fileInfo.FileName)), + } + if fileInfo.IsImage() { + info.FileThumbnailURL = pr.SubresThumbnailURL(path, fileInfo.FileName, 200) + } + return info + } + return nil + }, + }, nil +} + +func (ph *publishHandler) describeMembers(br blob.Ref) (*search.SearchResult, error) { + res, err := ph.cl.Search(&search.SearchQuery{ + Constraint: &search.Constraint{ + Permanode: &search.PermanodeConstraint{ + Relation: &search.RelationConstraint{ + Relation: "parent", + Any: &search.Constraint{ + BlobRefPrefix: br.String(), + }, + }, + }, + CamliType: "permanode", + }, + Describe: &search.DescribeRequest{ + ThumbnailSize: 1000, + Depth: 1, + Rules: []*search.DescribeRule{ + { + Attrs: []string{"camliContent", "camliContentImage"}, + }, + }, + }, + Limit: -1, + }) + if err != nil { + return nil, fmt.Errorf("Could not describe members of %v: %v", br, err) + } + return res, nil +} + +func (pr *publishRequest) memberPath(member blob.Ref) string { + return addPathComponent(pr.subjectBasePath, "/h"+member.DigestPrefix(10)) +} + +func (pr *publishRequest) serveSubresFileDownload() { + des, err := pr.ph.describe(pr.subject) + if err != nil { + log.Printf("error describing subject %q: %v", pr.subject, err) + return + } + pr.serveFileDownload(des) +} + +func (pr *publishRequest) serveSubresImage() { + params := pr.req.URL.Query() + mw, _ := strconv.Atoi(params.Get("mw")) + mh, _ := strconv.Atoi(params.Get("mh")) + des, err := pr.ph.describe(pr.subject) + if err != nil { + log.Printf("error describing subject %q: %v", pr.subject, err) + return + } + pr.serveScaledImage(des, mw, mh, params.Get("square") == "1") +} + +func (pr *publishRequest) serveScaledImage(des *search.DescribedBlob, maxWidth, maxHeight int, square bool) { + fileref, _, ok := pr.fileSchemaRefFromBlob(des) + if !ok { + log.Printf("scaled image fail; failed to get file schema from des %q", des.BlobRef) + return + } + ih := &server.ImageHandler{ + Fetcher: pr.ph.cl, + Cache: pr.ph.cache, + MaxWidth: maxWidth, + MaxHeight: maxHeight, + Square: square, + ThumbMeta: pr.ph.thumbMeta, + ResizeSem: pr.ph.resizeSem, + } + ih.ServeHTTP(pr.rw, pr.req, fileref) +} + +// serveZip streams a zip archive of all the files "under" +// pr.subject. That is, all the files pointed by file permanodes, +// which are directly members of pr.subject or recursively down +// directory permanodes and permanodes members. +func (pr *publishRequest) serveZip() { + filename := "" + if len(pr.subres) > len("/=z/") { + filename = pr.subres[4:] + } + zh := &zipHandler{ + fetcher: pr.ph.cl, + cl: pr.ph.cl, + root: pr.subject, + filename: filename, + } + zh.ServeHTTP(pr.rw, pr.req) +} diff --git a/server/camlistored/ui/pics.css b/app/publisher/pics.css similarity index 100% rename from server/camlistored/ui/pics.css rename to app/publisher/pics.css diff --git a/pkg/server/publish_test.go b/app/publisher/publish_test.go similarity index 80% rename from pkg/server/publish_test.go rename to app/publisher/publish_test.go index 683639dd1..62f0ff91b 100644 --- a/pkg/server/publish_test.go +++ b/app/publisher/publish_test.go @@ -1,5 +1,5 @@ /* -Copyright 2011 Google Inc. +Copyright 2014 The Camlistore Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,15 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package server +package main import ( + "io" "net/http" "net/http/httptest" "strings" "testing" "time" + camliClient "camlistore.org/pkg/client" "camlistore.org/pkg/httputil" "camlistore.org/pkg/index" "camlistore.org/pkg/index/indextest" @@ -113,6 +115,29 @@ func setupContent(rootName string) *indextest.IndexDeps { return idxd } +type fakeClient struct { + *camliClient.Client // for blob.Fetcher + sh *search.Handler +} + +func (fc *fakeClient) Search(req *search.SearchQuery) (*search.SearchResult, error) { + return fc.sh.Query(req) +} + +func (fc *fakeClient) Describe(req *search.DescribeRequest) (*search.DescribeResponse, error) { + return fc.sh.Describe(req) +} + +func (fc *fakeClient) GetJSON(url string, data interface{}) error { + // no need to implement + return nil +} + +func (fc *fakeClient) Post(url string, bodyType string, body io.Reader) error { + // no need to implement + return nil +} + func TestPublishURLs(t *testing.T) { rootName := "foo" idxd := setupContent(rootName) @@ -122,9 +147,14 @@ func TestPublishURLs(t *testing.T) { t.Fatalf("error slurping index to memory: %v", err) } sh.SetCorpus(corpus) - ph := &PublishHandler{ - RootName: rootName, - Search: sh, + cl := camliClient.New("http://whatever.fake") + fcl := &fakeClient{cl, sh} + ph := &publishHandler{ + rootName: rootName, + cl: fcl, + } + if err := ph.initRootNode(); err != nil { + t.Fatalf("initRootNode: %v", err) } for ti, tt := range publishURLTests { @@ -137,9 +167,12 @@ func TestPublishURLs(t *testing.T) { pfxh := &httputil.PrefixHandler{ Prefix: "/pics/", Handler: http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { - pr := ph.NewRequest(rw, req) + pr, err := ph.NewRequest(rw, req) + if err != nil { + t.Fatalf("test #%d, NewRequest: %v", ti, err) + } - err := pr.findSubject() + err = pr.findSubject() if tt.subject != "" { if err != nil { t.Errorf("test #%d, findSubject: %v", ti, err) @@ -168,9 +201,11 @@ func TestPublishMembers(t *testing.T) { t.Fatalf("error slurping index to memory: %v", err) } sh.SetCorpus(corpus) - ph := &PublishHandler{ - RootName: rootName, - Search: sh, + cl := camliClient.New("http://whatever.fake") + fcl := &fakeClient{cl, sh} + ph := &publishHandler{ + rootName: rootName, + cl: fcl, } rw := httptest.NewRecorder() @@ -179,17 +214,17 @@ func TestPublishMembers(t *testing.T) { pfxh := &httputil.PrefixHandler{ Prefix: "/pics/", Handler: http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { - pr := ph.NewRequest(rw, req) - - dr := pr.ph.Search.NewDescribeRequest() - dr.Describe(pr.subject, 3) - res, err := dr.Result() + pr, err := ph.NewRequest(rw, req) if err != nil { - t.Errorf("Result: %v", err) - return + t.Fatalf("NewRequest: %v", err) } - members, err := pr.subjectMembers(res) + res, err := pr.ph.deepDescribe(pr.subject) + if err != nil { + t.Fatalf("deepDescribe: %v", err) + } + + members, err := pr.subjectMembers(res.Meta) if len(members.Members) != 2 { t.Errorf("Expected two members in publish root (one camlipath, one camlimember), got %d", len(members.Members)) } diff --git a/pkg/server/zip.go b/app/publisher/zip.go similarity index 80% rename from pkg/server/zip.go rename to app/publisher/zip.go index 859d70172..1a3ea5f53 100644 --- a/pkg/server/zip.go +++ b/app/publisher/zip.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package server +package main import ( "archive/zip" @@ -31,12 +31,13 @@ import ( "camlistore.org/pkg/httputil" "camlistore.org/pkg/schema" "camlistore.org/pkg/search" + "camlistore.org/pkg/types/camtypes" ) type zipHandler struct { fetcher blob.Fetcher - search *search.Handler - // the "parent" permanode of everything to zip. + cl client // Used for search and describe requests. + // root is the "parent" permanode of everything to zip. // Either a directory permanode, or a permanode with members. root blob.Ref // Optional name to use in the response header @@ -58,17 +59,47 @@ func (s sortedFiles) Less(i, j int) bool { return s[i].path < s[j].path } func (s sortedFiles) Len() int { return len(s) } func (s sortedFiles) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (zh *zipHandler) describeMembers(br blob.Ref) (*search.DescribeResponse, error) { + res, err := zh.cl.Search(&search.SearchQuery{ + Constraint: &search.Constraint{ + BlobRefPrefix: br.String(), + CamliType: "permanode", + }, + Describe: &search.DescribeRequest{ + ThumbnailSize: 1000, + Depth: 1, + Rules: []*search.DescribeRule{ + { + Attrs: []string{"camliContent", "camliContentImage", "camliMember"}, + }, + }, + }, + Limit: -1, + }) + if err != nil { + return nil, fmt.Errorf("Could not describe %v: %v", br, err) + } + if res == nil || res.Describe == nil { + return nil, fmt.Errorf("no describe result for %v", br) + } + return res.Describe, nil +} + // blobList returns the list of file blobs "under" dirBlob. // It traverses permanode directories and permanode with members (collections). func (zh *zipHandler) blobList(dirPath string, dirBlob blob.Ref) ([]*blobFile, error) { - dr := zh.search.NewDescribeRequest() - dr.Describe(dirBlob, 3) - res, err := dr.Result() + // dr := zh.search.NewDescribeRequest() + // dr.Describe(dirBlob, 3) + // res, err := dr.Result() + // if err != nil { + // return nil, fmt.Errorf("Could not describe %v: %v", dirBlob, err) + // } + res, err := zh.describeMembers(dirBlob) if err != nil { - return nil, fmt.Errorf("Could not describe %v: %v", dirBlob, err) + return nil, err } - described := res[dirBlob.String()] + described := res.Meta[dirBlob.String()] members := described.Members() dirBlobPath, _, isDir := described.PermanodeDir() if len(members) == 0 && !isDir { @@ -85,13 +116,13 @@ func (zh *zipHandler) blobList(dirPath string, dirBlob blob.Ref) ([]*blobFile, e return list, nil } for _, member := range members { - if fileBlobPath, fileInfo, ok := member.PermanodeFile(); ok { + if fileBlobPath, fileInfo, ok := getFileInfo(member.BlobRef, res.Meta); ok { // file list = append(list, &blobFile{fileBlobPath[1], path.Join(dirPath, fileInfo.FileName)}) continue } - if dirBlobPath, dirInfo, ok := member.PermanodeDir(); ok { + if dirBlobPath, dirInfo, ok := getDirInfo(member.BlobRef, res.Meta); ok { // directory newZipRoot := dirBlobPath[1] children, err := zh.blobsFromDir( @@ -258,3 +289,21 @@ func (zh *zipHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { return } } + +// TODO(mpl): refactor with getFileInfo +func getDirInfo(item blob.Ref, peers map[string]*search.DescribedBlob) (path []blob.Ref, di *camtypes.FileInfo, ok bool) { + described := peers[item.String()] + if described == nil || + described.Permanode == nil || + described.Permanode.Attr == nil { + return + } + contentRef := described.Permanode.Attr.Get("camliContent") + if contentRef == "" { + return + } + if cdes := peers[contentRef]; cdes != nil && cdes.Dir != nil { + return []blob.Ref{described.BlobRef, cdes.BlobRef}, cdes.Dir, true + } + return +} diff --git a/config/dev-server-config.json b/config/dev-server-config.json index 1070cfb89..db3149761 100644 --- a/config/dev-server-config.json +++ b/config/dev-server-config.json @@ -28,31 +28,17 @@ } }, - "/blog/": { - "enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"], - "handler": "publish", - "handlerArgs": { - "rootName": "dev-blog-root", - "blobRoot": "/bs/", - "searchRoot": "/my-search/", - "cache": "/cache/", - "goTemplate": "blog.html", - "devBootstrapPermanodeUsing": "/sighelper/" - } - }, - "/pics/": { + "handler": "app", "enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"], - "handler": "publish", "handlerArgs": { - "rootName": "dev-pics-root", - "blobRoot": "/bs/", - "searchRoot": "/my-search/", - "cache": "/cache/", - "css": ["pics.css"], - "js": ["pics.js"], - "goTemplate": "gallery.html", - "devBootstrapPermanodeUsing": "/sighelper/" + "program": "publisher", + "appConfig": { + "camliRoot": "dev-pics-root", + "sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""], + "cacheRoot": ["_env", "${CAMLI_ROOT_CACHE}"], + "goTemplate": "gallery.html" + } } }, @@ -72,8 +58,7 @@ "scaledImage": { "type": "kv", "file": ["_env", "${CAMLI_ROOT_CACHE}/thumbnails.kv", ""] - }, - "publishRoots": ["/blog/", "/pics/"] + } } }, diff --git a/dev/devcam/server.go b/dev/devcam/server.go index 0db479c68..ac6df25a2 100644 --- a/dev/devcam/server.go +++ b/dev/devcam/server.go @@ -55,7 +55,7 @@ type serverCmd struct { fullClosure bool mini bool - publish bool // whether to start the publish handlers + publish bool // whether to build and start the publisher app(s) hello bool // whether to build and start the hello demo app openBrowser bool @@ -82,7 +82,7 @@ func init() { flags.BoolVar(&cmd.tls, "tls", false, "Use TLS.") flags.BoolVar(&cmd.wipe, "wipe", false, "Wipe the blobs on disk and the indexer.") flags.BoolVar(&cmd.debug, "debug", false, "Enable http debugging.") - flags.BoolVar(&cmd.publish, "publish", true, "Enable publish handlers") + flags.BoolVar(&cmd.publish, "publish", true, "Enable publisher app(s)") flags.BoolVar(&cmd.hello, "hello", false, "Enable hello (demo) app") flags.BoolVar(&cmd.mini, "mini", false, "Enable minimal mode, where all optional features are disabled. (Currently just publishing)") @@ -384,6 +384,9 @@ func (c *serverCmd) RunCommand(args []string) error { if c.hello { targets = append(targets, filepath.Join("app", "hello")) } + if c.publish { + targets = append(targets, filepath.Join("app", "publisher")) + } for _, name := range targets { err := build(name) if err != nil { diff --git a/doc/publishing/README b/doc/publishing/README index 2d11b0b86..444e3ae2c 100644 --- a/doc/publishing/README +++ b/doc/publishing/README @@ -1,14 +1,13 @@ -Camlistore uses Go html templates (http://golang.org/pkg/text/template/) to publish pages. +Camlistore delegates publishing to the publisher server application, which uses Go html templates (http://golang.org/pkg/text/template/) to publish pages. -Resources for publishing, such as go templates, javascript and css files should be placed in server/camlistored/ui/, so they can be served directly when using the dev server or automatically embedded when using camlistored directly. +Resources for publishing, such as go templates, javascript and css files should be placed in the application source directory - app/publisher/ - so they can be served directly when using the dev server or automatically embedded in production. -You can then specify those resources through the configuration file. For example, there already is a go template (gallery.html), javascript file (pics.js) and css file (pics.css) that work together to provide publishing for image galleries. The dev server config (config/dev-server-config.json) already uses them. Here is how one would use them in the server config ($HOME/.config/camlistore/server-config.json): +You should then specify the Go template to be used through the configuration file. The CSS files are automatically all available to the app. For example, there already is a go template (gallery.html), and css file (pics.css) that work together to provide publishing for image galleries. The dev server config (config/dev-server-config.json) already uses them. Here is how one would configure publishing for an image gallery in the server config ($HOME/.config/camlistore/server-config.json): "publish": { "/pics/": { - "rootPermanode": "sha1-6cbe9e1c35e854eab028cba43d099d35ceae0de8", - "style": "pics.css", - "js": "pics.js", + "camliRoot": "mypics", + "cacheRoot": "/home/joe/var/camlistore/blobs/cache", "goTemplate": "gallery.html" } } diff --git a/make.go b/make.go index d1e82bfdb..e06e867e7 100644 --- a/make.go +++ b/make.go @@ -169,6 +169,7 @@ func main() { "camlistore.org/cmd/camtool", "camlistore.org/server/camlistored", "camlistore.org/app/hello", + "camlistore.org/app/publisher", } switch *targets { case "*": @@ -345,7 +346,7 @@ func buildSrcPath(fromSrc string) string { // kept in between runs. func genEmbeds() error { cmdName := exeName(filepath.Join(buildGoPath, "bin", "genfileembed")) - for _, embeds := range []string{"server/camlistored/ui", "pkg/server", "third_party/react", "third_party/glitch", "third_party/fontawesome"} { + for _, embeds := range []string{"server/camlistored/ui", "pkg/server", "third_party/react", "third_party/glitch", "third_party/fontawesome", "app/publisher"} { embeds := buildSrcPath(embeds) args := []string{embeds} cmd := exec.Command(cmdName, args...) diff --git a/pkg/client/client.go b/pkg/client/client.go index 49b8bb5d0..cbe306b31 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -785,6 +785,22 @@ func (c *Client) GetJSON(url string, data interface{}) error { return httputil.DecodeJSON(resp, data) } +// Post is like http://golang.org/pkg/net/http/#Client.Post +// but with implementation details like gated requests. The +// URL's host must match the client's configured server. +func (c *Client) Post(url string, bodyType string, body io.Reader) error { + if !strings.HasPrefix(url, c.discoRoot()) { + return fmt.Errorf("wrong URL (%q) for this server", url) + } + req := c.newRequest("POST", url, body) + req.Header.Set("Content-Type", bodyType) + res, err := c.expect2XX(req) + if err != nil { + return err + } + return res.Body.Close() +} + func (c *Client) newRequest(method, url string, body ...io.Reader) *http.Request { var bodyR io.Reader if len(body) > 0 { diff --git a/pkg/search/describe.go b/pkg/search/describe.go index af3a74570..ab0173959 100644 --- a/pkg/search/describe.go +++ b/pkg/search/describe.go @@ -456,32 +456,6 @@ func (b *DescribedBlob) peerBlob(br blob.Ref) *DescribedBlob { return &DescribedBlob{Request: b.Request, BlobRef: br, Stub: true} } -// HasSecureLinkTo returns true if there's a valid link from this blob -// to the other blob. This is used in access control (hence the -// somewhat redundant "Secure" in the name) and should be paranoid -// against e.g. random user/attacker-control attributes making links -// to other blobs. -// -// TODO: don't linear scan here. rewrite this in terms of ResolvePrefixHop, -// passing down some policy perhaps? or maybe that's enough. -func (b *DescribedBlob) HasSecureLinkTo(other blob.Ref) bool { - if b == nil || !other.Valid() { - return false - } - ostr := other.String() - if b.Permanode != nil { - if b.Permanode.Attr.Get("camliContent") == ostr { - return true - } - for _, mstr := range b.Permanode.Attr["camliMember"] { - if mstr == ostr { - return true - } - } - } - return false -} - func (b *DescribedBlob) isPermanode() bool { return b.Permanode != nil } diff --git a/pkg/search/handler.go b/pkg/search/handler.go index 1a480b7fe..ff3e26244 100644 --- a/pkg/search/handler.go +++ b/pkg/search/handler.go @@ -639,53 +639,6 @@ func (sh *Handler) serveClaims(rw http.ResponseWriter, req *http.Request) { httputil.ReturnJSON(rw, res) } -// Given a blobref and a few hex characters of the digest of the next hop, return the complete -// blobref of the prefix, if that's a valid next hop. -func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob.Ref, err error) { - // TODO: this is a linear scan right now. this should be - // optimized to use a new database table of members so this is - // a quick lookup. in the meantime it should be in memcached - // at least. - if len(prefix) < 8 { - return blob.Ref{}, fmt.Errorf("Member prefix %q too small", prefix) - } - dr := sh.NewDescribeRequest() - dr.Describe(parent, 1) - res, err := dr.Result() - if err != nil { - return - } - des, ok := res[parent.String()] - if !ok { - return blob.Ref{}, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent) - } - if des.Permanode != nil { - cr, ok := des.ContentRef() - if ok && strings.HasPrefix(cr.Digest(), prefix) { - return cr, nil - } - for _, member := range des.Members() { - if strings.HasPrefix(member.BlobRef.Digest(), prefix) { - return member.BlobRef, nil - } - } - _, err := dr.DescribeSync(cr) - if err != nil { - return blob.Ref{}, fmt.Errorf("Failed to describe content %q of parent %q", cr, parent) - } - if _, _, ok := des.PermanodeDir(); ok { - return sh.ResolvePrefixHop(cr, prefix) - } - } else if des.Dir != nil { - for _, child := range des.DirChildren { - if strings.HasPrefix(child.Digest(), prefix) { - return child, nil - } - } - } - return blob.Ref{}, fmt.Errorf("Member prefix %q not found in %q", prefix, parent) -} - func (sh *Handler) serveFiles(rw http.ResponseWriter, req *http.Request) { ret := jsonMap() defer httputil.ReturnJSON(rw, ret) diff --git a/pkg/server/image.go b/pkg/server/image.go index c61f5b7ba..ef68e46d0 100644 --- a/pkg/server/image.go +++ b/pkg/server/image.go @@ -63,8 +63,8 @@ type ImageHandler struct { Cache blobserver.Storage // optional MaxWidth, MaxHeight int Square bool - thumbMeta *thumbMeta // optional cache for scaled images - resizeSem *syncutil.Sem + ThumbMeta *ThumbMeta // optional cache index for scaled images + ResizeSem *syncutil.Sem // Limit peak RAM used by concurrent image thumbnail calls. } type subImager interface { @@ -117,7 +117,7 @@ func (ih *ImageHandler) cacheScaled(thumbBytes []byte, name string) error { if err != nil { return err } - ih.thumbMeta.Put(name, br) + ih.ThumbMeta.Put(name, br) return nil } @@ -178,7 +178,7 @@ func cacheKey(bref string, width int, height int) string { // Almost all errors are not interesting. Real errors will be logged. func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string) { key := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight) - br, err := ih.thumbMeta.Get(key) + br, err := ih.ThumbMeta.Get(key) if err == errCacheMiss { return } @@ -188,6 +188,9 @@ func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format s } fr, err := ih.cached(br) if err != nil { + if imageDebug { + log.Printf("Could not get cached image %v: %v\n", br, err) + } return } defer fr.Close() @@ -247,10 +250,10 @@ func (ih *ImageHandler) scaleImage(fileRef blob.Ref) (*formatAndImage, error) { // images being resized concurrently. ramSize := int64(conf.Width) * int64(conf.Height) * 3 - if err = ih.resizeSem.Acquire(ramSize); err != nil { + if err = ih.ResizeSem.Acquire(ramSize); err != nil { return nil, err } - defer ih.resizeSem.Release(ramSize) + defer ih.ResizeSem.Release(ramSize) i, imConfig, err := images.Decode(sr, &images.DecodeOpts{ MaxWidth: ih.MaxWidth, @@ -325,7 +328,7 @@ func (ih *ImageHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, fil var imageData []byte format := "" cacheHit := false - if ih.thumbMeta != nil && !disableThumbCache { + if ih.ThumbMeta != nil && !disableThumbCache { var buf bytes.Buffer format = ih.scaledCached(&buf, file) if format != "" { @@ -346,7 +349,7 @@ func (ih *ImageHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, fil im := imi.(*formatAndImage) imageData = im.image format = im.format - if ih.thumbMeta != nil { + if ih.ThumbMeta != nil { err := ih.cacheScaled(imageData, key) if err != nil { log.Printf("image resize: %v", err) diff --git a/pkg/server/publish.go b/pkg/server/publish.go deleted file mode 100644 index 778689f11..000000000 --- a/pkg/server/publish.go +++ /dev/null @@ -1,1073 +0,0 @@ -/* -Copyright 2011 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package server - -import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "html" - "html/template" - "io/ioutil" - "log" - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - - "camlistore.org/pkg/auth" - "camlistore.org/pkg/blob" - "camlistore.org/pkg/blobserver" - "camlistore.org/pkg/client" - "camlistore.org/pkg/constants" // just for NewUploadHandleFromString. move elsewhere? - "camlistore.org/pkg/fileembed" - "camlistore.org/pkg/httputil" - "camlistore.org/pkg/jsonconfig" - "camlistore.org/pkg/jsonsign/signhandler" - "camlistore.org/pkg/publish" - "camlistore.org/pkg/schema" - "camlistore.org/pkg/search" - "camlistore.org/pkg/syncutil" - "camlistore.org/pkg/types/camtypes" - uistatic "camlistore.org/server/camlistored/ui" -) - -// PublishHandler publishes your info to the world, if permanodes have -// appropriate ACLs set. (everything is private by default) -type PublishHandler struct { - RootName string - Search *search.Handler - Storage blobserver.Storage // of blobRoot - Cache blobserver.Storage // or nil - - // Limit peak RAM used by concurrent image thumbnail calls. - resizeSem *syncutil.Sem - thumbMeta *thumbMeta // optional cache of scaled images - - CSSFiles []string - // goTemplate is the go html template used for publishing. - goTemplate *template.Template - closureName string // Name of the closure object used to decorate the published page. - - handlerFinder blobserver.FindHandlerByTyper - - // sourceRoot optionally specifies the path to root of Camlistore's - // source. If empty, the UI files must be compiled in to the - // binary (with go run make.go). This comes from the "sourceRoot" - // publish handler config option. - sourceRoot string - - uiDir string // if sourceRoot != "", this is sourceRoot+"/server/camlistored/ui" - - // closureHandler serves the Closure JS files. - closureHandler http.Handler -} - -func init() { - blobserver.RegisterHandlerConstructor("publish", newPublishFromConfig) -} - -func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { - ph := &PublishHandler{ - handlerFinder: ld, - } - ph.RootName = conf.RequiredString("rootName") - jsFiles := conf.OptionalList("js") - ph.CSSFiles = conf.OptionalList("css") - goTemplateFile := conf.RequiredString("goTemplate") - blobRoot := conf.RequiredString("blobRoot") - searchRoot := conf.RequiredString("searchRoot") - cachePrefix := conf.OptionalString("cache", "") - scaledImageConf := conf.OptionalObject("scaledImage") - bootstrapSignRoot := conf.OptionalString("devBootstrapPermanodeUsing", "") - rootNode := conf.OptionalList("rootPermanode") - ph.sourceRoot = conf.OptionalString("sourceRoot", "") - ph.resizeSem = syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes", constants.DefaultMaxResizeMem))) - if err = conf.Validate(); err != nil { - return - } - - if ph.RootName == "" { - return nil, errors.New("invalid empty rootName") - } - - bs, err := ld.GetStorage(blobRoot) - if err != nil { - return nil, fmt.Errorf("publish handler's blobRoot of %q error: %v", blobRoot, err) - } - ph.Storage = bs - - si, err := ld.GetHandler(searchRoot) - if err != nil { - return nil, fmt.Errorf("publish handler's searchRoot of %q error: %v", searchRoot, err) - } - var ok bool - ph.Search, ok = si.(*search.Handler) - if !ok { - return nil, fmt.Errorf("publish handler's searchRoot of %q is of type %T, expecting a search handler", - searchRoot, si) - } - - if rootNode != nil { - if len(rootNode) != 2 { - return nil, fmt.Errorf("rootPermanode config must contain the jsonSignerHandler and the permanode hash") - } - - if t := ld.GetHandlerType(rootNode[0]); t != "jsonsign" { - return nil, fmt.Errorf("publish handler's rootPermanode first value not a jsonsign") - } - h, _ := ld.GetHandler(rootNode[0]) - jsonSign := h.(*signhandler.Handler) - pn, ok := blob.Parse(rootNode[1]) - if !ok { - return nil, fmt.Errorf("Invalid \"rootPermanode\" value; was expecting a blobRef, got %q.", rootNode[1]) - } - if err := ph.setRootNode(jsonSign, pn); err != nil { - return nil, fmt.Errorf("error setting publish root permanode: %v", err) - } - } else { - if bootstrapSignRoot != "" { - if t := ld.GetHandlerType(bootstrapSignRoot); t != "jsonsign" { - return nil, fmt.Errorf("publish handler's devBootstrapPermanodeUsing must be of type jsonsign") - } - h, _ := ld.GetHandler(bootstrapSignRoot) - jsonSign := h.(*signhandler.Handler) - if err := ph.bootstrapPermanode(jsonSign); err != nil { - return nil, fmt.Errorf("error bootstrapping permanode: %v", err) - } - } - } - - scaledImageKV, err := newKVOrNil(scaledImageConf) - if err != nil { - return nil, fmt.Errorf("in publish handler's scaledImage: %v", err) - } - if scaledImageKV != nil && cachePrefix == "" { - return nil, fmt.Errorf("in publish handler, can't specify scaledImage without cache") - } - if cachePrefix != "" { - bs, err := ld.GetStorage(cachePrefix) - if err != nil { - return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err) - } - ph.Cache = bs - ph.thumbMeta = newThumbMeta(scaledImageKV) - } - - // TODO(mpl): check that it works on appengine too. - if ph.sourceRoot == "" { - ph.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT") - } - if ph.sourceRoot != "" { - ph.uiDir = filepath.Join(ph.sourceRoot, filepath.FromSlash("server/camlistored/ui")) - // Ignore any fileembed files: - Files = &fileembed.Files{ - DirFallback: filepath.Join(ph.sourceRoot, filepath.FromSlash("pkg/server")), - } - uistatic.Files = &fileembed.Files{ - DirFallback: ph.uiDir, - } - } - - ph.closureHandler, err = ph.makeClosureHandler(ph.sourceRoot) - if err != nil { - return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ph.sourceRoot, err) - } - - ph.goTemplate, err = goTemplate(goTemplateFile) - if err != nil { - return nil, err - } - ph.setClosureName(jsFiles) - - return ph, nil -} - -func goTemplate(templateFile string) (*template.Template, error) { - if filepath.Base(templateFile) != templateFile { - hint := fmt.Sprintf("The file should either be embedded or placed in %s.", - filepath.FromSlash("server/camlistored/ui")) - return nil, fmt.Errorf("Unsupported path %v for template. %s", templateFile, hint) - } - f, err := uistatic.Files.Open(templateFile) - if err != nil { - return nil, fmt.Errorf("Could not open template %v: %v", templateFile, err) - } - defer f.Close() - templateBytes, err := ioutil.ReadAll(f) - if err != nil { - return nil, fmt.Errorf("Could not read template %v: %v", templateFile, err) - } - - return template.Must(template.New("subject").Parse(string(templateBytes))), nil -} - -// setClosureName sets ph.closureName with the first found closure -// namespace provided in jsFiles. -func (ph *PublishHandler) setClosureName(jsFiles []string) { - for _, v := range jsFiles { - if ph.closureName == "" { - if cl := camliClosurePage(v); cl != "" { - ph.closureName = cl - break - } - } - } -} - -func (ph *PublishHandler) makeClosureHandler(root string) (http.Handler, error) { - return makeClosureHandler(root, "publish") -} - -func (ph *PublishHandler) camliRootQuery() (*search.SearchResult, error) { - // TODO(mpl): I've voluntarily omitted the owner because it's not clear to - // that we actually care about that. Same for signer in lookupPathTarget. - return ph.Search.Query(&search.SearchQuery{ - Limit: 1, - Constraint: &search.Constraint{ - Permanode: &search.PermanodeConstraint{ - Attr: "camliRoot", - Value: ph.RootName, - }, - }, - }) -} - -func (ph *PublishHandler) rootPermanode() (blob.Ref, error) { - // TODO: caching, but this can change over time (though - // probably rare). might be worth a 5 second cache or - // something in-memory? better invalidation story first would - // be nice. - result, err := ph.camliRootQuery() - if err != nil { - return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.RootName, err) - } - if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() { - return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.RootName, os.ErrNotExist) - } - return result.Blobs[0].Blob, nil -} - -func (ph *PublishHandler) lookupPathTarget(root blob.Ref, suffix string) (blob.Ref, error) { - if suffix == "" { - return root, nil - } - // TODO: verify it's optimized: http://camlistore.org/issue/405 - result, err := ph.Search.Query(&search.SearchQuery{ - Limit: 1, - Constraint: &search.Constraint{ - Permanode: &search.PermanodeConstraint{ - SkipHidden: true, - Relation: &search.RelationConstraint{ - Relation: "parent", - EdgeType: "camliPath:" + suffix, - Any: &search.Constraint{ - BlobRefPrefix: root.String(), - }, - }, - }, - }, - }) - if err != nil { - return blob.Ref{}, err - } - if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() { - return blob.Ref{}, os.ErrNotExist - } - return result.Blobs[0].Blob, nil -} - -func (ph *PublishHandler) serveDiscovery(rw http.ResponseWriter, req *http.Request) { - if !ph.ViewerIsOwner(req) { - discoveryHelper(rw, req, map[string]interface{}{ - "error": "viewer isn't owner", - }) - return - } - _, handler, err := ph.handlerFinder.FindHandlerByType("ui") - if err != nil { - discoveryHelper(rw, req, map[string]interface{}{ - "error": "no admin handler running", - }) - return - } - ui := handler.(*UIHandler) - ui.root.serveDiscovery(rw, req) -} - -func (ph *PublishHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - if req.URL.Query().Get("camli.mode") == "config" { - ph.serveDiscovery(rw, req) - return - } - preq := ph.NewRequest(rw, req) - preq.serveHTTP() -} - -// publishRequest is the state around a single HTTP request to the -// publish handler -type publishRequest struct { - ph *PublishHandler - rw http.ResponseWriter - req *http.Request - base, suffix, subres string - rootpn blob.Ref - subject blob.Ref - inSubjectChain map[string]bool // blobref -> true - subjectBasePath string - - // A describe request that we can reuse, sharing its map of - // blobs already described. - dr *search.DescribeRequest - - // Limit peak RAM used by concurrent image thumbnail calls. - resizeSem *syncutil.Sem -} - -func (ph *PublishHandler) NewRequest(rw http.ResponseWriter, req *http.Request) *publishRequest { - // splits a path request into its suffix and subresource parts. - // e.g. /blog/foo/camli/res/file/xxx -> ("foo", "file/xxx") - suffix, res := httputil.PathSuffix(req), "" - if strings.HasPrefix(suffix, "-/") { - suffix, res = "", suffix[2:] - } else if s := strings.SplitN(suffix, "/-/", 2); len(s) == 2 { - suffix, res = s[0], s[1] - } - rootpn, _ := ph.rootPermanode() - return &publishRequest{ - ph: ph, - rw: rw, - req: req, - suffix: suffix, - base: httputil.PathBase(req), - subres: res, - rootpn: rootpn, - dr: ph.Search.NewDescribeRequest(), - inSubjectChain: make(map[string]bool), - subjectBasePath: "", - resizeSem: ph.resizeSem, - } -} - -func (ph *PublishHandler) ViewerIsOwner(req *http.Request) bool { - // TODO: better check later - return auth.Allowed(req, auth.OpAll) -} - -func (pr *publishRequest) ViewerIsOwner() bool { - return pr.ph.ViewerIsOwner(pr.req) -} - -func (pr *publishRequest) Debug() bool { - return pr.req.FormValue("debug") == "1" -} - -func (pr *publishRequest) SubresourceType() string { - if len(pr.subres) >= 3 && strings.HasPrefix(pr.subres, "/=") { - return pr.subres[2:3] - } - return "" -} - -func (pr *publishRequest) SubresFileURL(path []blob.Ref, fileName string) string { - return pr.SubresThumbnailURL(path, fileName, -1) -} - -func (pr *publishRequest) SubresThumbnailURL(path []blob.Ref, fileName string, maxDimen int) string { - var buf bytes.Buffer - resType := "i" - if maxDimen == -1 { - resType = "f" - } - fmt.Fprintf(&buf, "%s", pr.subjectBasePath) - if !strings.Contains(pr.subjectBasePath, "/-/") { - buf.Write([]byte("/-")) - } - for _, br := range path { - if pr.inSubjectChain[br.String()] { - continue - } - fmt.Fprintf(&buf, "/h%s", br.DigestPrefix(10)) - } - fmt.Fprintf(&buf, "/=%s", resType) - fmt.Fprintf(&buf, "/%s", url.QueryEscape(fileName)) - if maxDimen != -1 { - fmt.Fprintf(&buf, "?mw=%d&mh=%d", maxDimen, maxDimen) - } - return buf.String() -} - -var memberRE = regexp.MustCompile(`^/?h([0-9a-f]+)`) - -func (pr *publishRequest) findSubject() error { - if strings.HasPrefix(pr.suffix, "=s/") { - pr.subres = "/" + pr.suffix - return nil - } - - subject, err := pr.ph.lookupPathTarget(pr.rootpn, pr.suffix) - if err != nil { - return err - } - if strings.HasPrefix(pr.subres, "=z/") { - // this happens when we are at the root of the published path, - // e.g /base/suffix/-/=z/foo.zip - // so we need to reset subres as fullpath so that it is detected - // properly when switching on pr.SubresourceType() - pr.subres = "/" + pr.subres - // since we return early, we set the subject because that is - // what is going to be used as a root node by the zip handler. - pr.subject = subject - return nil - } - - pr.inSubjectChain[subject.String()] = true - pr.subjectBasePath = pr.base + pr.suffix - - // Chase /h hops in suffix. - for { - m := memberRE.FindStringSubmatch(pr.subres) - if m == nil { - break - } - match, memberPrefix := m[0], m[1] - - if err != nil { - return fmt.Errorf("Error looking up potential member %q in describe of subject %q: %v", - memberPrefix, subject, err) - } - - subject, err = pr.ph.Search.ResolvePrefixHop(subject, memberPrefix) - if err != nil { - return err - } - pr.inSubjectChain[subject.String()] = true - pr.subres = pr.subres[len(match):] - pr.subjectBasePath = addPathComponent(pr.subjectBasePath, match) - } - - pr.subject = subject - return nil -} - -func (pr *publishRequest) serveHTTP() { - if !pr.rootpn.Valid() { - pr.rw.WriteHeader(404) - return - } - - if pr.Debug() { - pr.rw.Header().Set("Content-Type", "text/html") - pr.pf("I am publish handler at base %q, serving root %q (permanode=%s), suffix %q, subreq %q
", - pr.base, pr.ph.RootName, pr.rootpn, html.EscapeString(pr.suffix), html.EscapeString(pr.subres)) - } - - if err := pr.findSubject(); err != nil { - if err == os.ErrNotExist { - pr.rw.WriteHeader(404) - return - } - log.Printf("Error looking up %s/%q: %v", pr.rootpn, pr.suffix, err) - pr.rw.WriteHeader(500) - return - } - - if pr.Debug() { - pr.pf("

Subject: %s

", pr.subject, pr.subject) - return - } - - switch pr.SubresourceType() { - case "": - pr.serveSubjectTemplate() - case "b": - // TODO: download a raw blob - case "f": // file download - pr.serveSubresFileDownload() - case "i": // image, scaled - pr.serveSubresImage() - case "s": // static - pr.req.URL.Path = pr.subres[len("/=s"):] - if len(pr.req.URL.Path) <= 1 { - http.Error(pr.rw, "Illegal URL.", http.StatusNotFound) - return - } - file := pr.req.URL.Path[1:] - if m := closurePattern.FindStringSubmatch(file); m != nil { - pr.req.URL.Path = "/" + m[1] - pr.ph.closureHandler.ServeHTTP(pr.rw, pr.req) - return - } - if file == "deps.js" { - serveDepsJS(pr.rw, pr.req, pr.ph.uiDir) - return - } - serveStaticFile(pr.rw, pr.req, uistatic.Files, file) - case "z": - pr.serveZip() - default: - pr.rw.WriteHeader(400) - pr.pf("

Invalid or unsupported resource request.

") - } -} - -func (pr *publishRequest) pf(format string, args ...interface{}) { - fmt.Fprintf(pr.rw, format, args...) -} - -func (pr *publishRequest) staticPath(fileName string) string { - return pr.base + "=s/" + fileName -} - -func addPathComponent(base, addition string) string { - if !strings.HasPrefix(addition, "/") { - addition = "/" + addition - } - if strings.Contains(base, "/-/") { - return base + addition - } - return base + "/-" + addition -} - -func (pr *publishRequest) memberPath(member blob.Ref) string { - return addPathComponent(pr.subjectBasePath, "/h"+member.DigestPrefix(10)) -} - -var provCamliRx = regexp.MustCompile(`^goog\.(provide)\(['"]camlistore\.(.*)['"]\)`) - -// camliClosurePage checks if filename is a .js file using closure -// and if yes, if it provides a page in the camlistore namespace. -// It returns that page name, or the empty string otherwise. -func camliClosurePage(filename string) string { - f, err := uistatic.Files.Open(filename) - if err != nil { - return "" - } - defer f.Close() - br := bufio.NewReader(f) - for { - l, err := br.ReadString('\n') - if err != nil { - return "" - } - if !strings.HasPrefix(l, "goog.") { - continue - } - m := provCamliRx.FindStringSubmatch(l) - if m != nil { - return m[2] - } - } - return "" -} - -// serveZip streams a zip archive of all the files "under" -// pr.subject. That is, all the files pointed by file permanodes, -// which are directly members of pr.subject or recursively down -// directory permanodes and permanodes members. -func (pr *publishRequest) serveZip() { - filename := "" - if len(pr.subres) > len("/=z/") { - filename = pr.subres[4:] - } - zh := &zipHandler{ - fetcher: pr.ph.Storage, - search: pr.ph.Search, - root: pr.subject, - filename: filename, - } - zh.ServeHTTP(pr.rw, pr.req) -} - -const ( - resSeparator = "/-" - digestPrefix = "h" - digestLen = 10 -) - -var hopRE = regexp.MustCompile(fmt.Sprintf("^/%s([0-9a-f]{%d})", digestPrefix, digestLen)) - -// publishedPath is a URL suffix path of the kind -// suffix + resSeparator + subresource(s), such as: -// /foo/bar/-/subres1/subres2 -type publishedPath string - -// splitHops returns a slice containing the subresource(s) -// digests. For example, with /foo/bar/-/he0917e5bcf/h5f46bb454d -// it will yield []string{"e0917e5bcf", "5f46bb454d"} -func (p publishedPath) splitHops() []string { - ps := string(p) - var hops []string - if idx := strings.Index(ps, resSeparator); idx != -1 { - ps = ps[idx+len(resSeparator):] - } - matchLen := 1 + len(digestPrefix) + digestLen - for { - m := memberRE.FindStringSubmatch(ps) - if m == nil { - break - } - hops = append(hops, m[1]) - ps = ps[matchLen:] - } - return hops -} - -// parent returns the base path and the blobRef of pr.subject's parent. -// It returns an error if pr.subject or pr.subjectBasePath were not set -// properly (with findSubject), or if the parent was not found. -func (pr *publishRequest) parent() (parentPath string, parentBlobRef blob.Ref, err error) { - if !pr.subject.Valid() { - return "", blob.Ref{}, errors.New("subject not set") - } - if pr.subjectBasePath == "" { - return "", blob.Ref{}, errors.New("subjectBasePath not set") - } - // TODO(mpl): this fails when the parent is the root. fix it. - hops := publishedPath(pr.subjectBasePath).splitHops() - if len(hops) == 0 { - return "", blob.Ref{}, errors.New("No subresource digest in subjectBasePath") - } - subjectDigest := hops[len(hops)-1] - if subjectDigest != pr.subject.DigestPrefix(digestLen) { - return "", blob.Ref{}, errors.New("subject digest not in subjectBasePath") - } - parentPath = strings.TrimSuffix(pr.subjectBasePath, "/"+digestPrefix+subjectDigest) - - if len(hops) == 1 { - // the parent is the suffix, not one of the subresource hops - for br, _ := range pr.inSubjectChain { - if br != pr.subject.String() { - parentBlobRef = blob.ParseOrZero(br) - break - } - } - } else { - // nested collection(s) - parentDigest := hops[len(hops)-2] - for br, _ := range pr.inSubjectChain { - bref, ok := blob.Parse(br) - if !ok { - return "", blob.Ref{}, fmt.Errorf("Could not parse %q as blobRef", br) - } - if bref.DigestPrefix(10) == parentDigest { - parentBlobRef = bref - break - } - } - } - if !parentBlobRef.Valid() { - return "", blob.Ref{}, fmt.Errorf("No parent found for %v", pr.subjectBasePath) - } - return parentPath, parentBlobRef, nil -} - -func (pr *publishRequest) cssFiles() []string { - files := []string{} - for _, filename := range pr.ph.CSSFiles { - files = append(files, pr.staticPath(filename)) - } - return files -} - -// jsDeps returns the list of paths that should be included -// as javascript files in the published page to enable and use -// additional javascript closure code. -func (pr *publishRequest) jsDeps() []string { - var js []string - closureDeps := []string{ - "closure/goog/base.js", - "deps.js", - // TODO(mpl): fix the deps generator and/or the SHA1.js etc files so they get into deps.js and we - // do not even have to include them here. detection fails for them because the provide statement - // is not at the beginning of the line. - // Not doing it right away because it might have consequences for the rest of the ui I suppose. - "base64.js", - "Crypto.js", - "SHA1.js", - } - for _, v := range closureDeps { - js = append(js, pr.staticPath(v)) - } - js = append(js, pr.base+"?camli.mode=config&var=CAMLISTORE_CONFIG") - return js -} - -// subjectHeader returns the PageHeader corresponding to the described subject. -func (pr *publishRequest) subjectHeader(described map[string]*search.DescribedBlob) *publish.PageHeader { - subdes := described[pr.subject.String()] - header := &publish.PageHeader{ - Title: html.EscapeString(subdes.Title()), - CSSFiles: pr.cssFiles(), - Meta: func() string { - jsonRes, _ := json.MarshalIndent(described, "", " ") - return string(jsonRes) - }(), - Subject: pr.subject.String(), - } - header.JSDeps = pr.jsDeps() - if pr.ph.closureName != "" { - header.CamliClosure = template.JS("camlistore." + pr.ph.closureName) - } - if pr.ViewerIsOwner() { - header.ViewerIsOwner = true - } - return header -} - -// subjectFile returns the relevant PageFile if the described subject is a file permanode. -func (pr *publishRequest) subjectFile(described map[string]*search.DescribedBlob) (*publish.PageFile, error) { - subdes := described[pr.subject.String()] - contentRef, ok := subdes.ContentRef() - if !ok { - return nil, nil - } - fileDes, err := pr.dr.DescribeSync(contentRef) - if err != nil { - return nil, fmt.Errorf("Could not describe %v: %v", contentRef, err) - } - path := []blob.Ref{pr.subject, contentRef} - downloadURL := pr.SubresFileURL(path, fileDes.File.FileName) - thumbnailURL := "" - if fileDes.File.IsImage() { - thumbnailURL = pr.SubresThumbnailURL(path, fileDes.File.FileName, 600) - } - fileName := html.EscapeString(fileDes.File.FileName) - return &publish.PageFile{ - FileName: fileName, - Size: fileDes.File.Size, - MIMEType: fileDes.File.MIMEType, - IsImage: fileDes.File.IsImage(), - DownloadURL: downloadURL, - ThumbnailURL: thumbnailURL, - DomID: contentRef.DomID(), - Nav: func() *publish.Nav { - nv, err := pr.fileNavigation() - if err != nil { - log.Print(err) - return nil - } - return nv - }, - }, nil -} - -func (pr *publishRequest) fileNavigation() (*publish.Nav, error) { - // first get the parent path and blob - parentPath, parentbr, err := pr.parent() - if err != nil { - return nil, fmt.Errorf("Could not get subject %v's parent's info: %v", pr.subject, err) - } - parentNav := strings.TrimSuffix(parentPath, resSeparator) - fileNav := &publish.Nav{ - ParentPath: parentNav, - } - - // describe the parent so we get the siblings (members of the parent) - dr := pr.ph.Search.NewDescribeRequest() - dr.Describe(parentbr, 3) - parentRes, err := dr.Result() - if err != nil { - return nil, fmt.Errorf("Could not \"deeply\" describe subject %v's parent %v: %v", pr.subject, parentbr, err) - } - members := parentRes[parentbr.String()].Members() - if len(members) == 0 { - return fileNav, nil - } - - pos := 0 - var prev, next blob.Ref - for k, member := range members { - if member.BlobRef.String() == pr.subject.String() { - pos = k - break - } - } - if pos > 0 { - prev = members[pos-1].BlobRef - } - if pos < len(members)-1 { - next = members[pos+1].BlobRef - } - if !prev.Valid() && !next.Valid() { - return fileNav, nil - } - if prev.Valid() { - fileNav.PrevPath = fmt.Sprintf("%s/%s%s", parentPath, digestPrefix, prev.DigestPrefix(10)) - } - if next.Valid() { - fileNav.NextPath = fmt.Sprintf("%s/%s%s", parentPath, digestPrefix, next.DigestPrefix(10)) - } - return fileNav, nil -} - -// subjectMembers returns the relevant PageMembers if the described subject is a permanode with members. -func (pr *publishRequest) subjectMembers(resMap map[string]*search.DescribedBlob) (*publish.PageMembers, error) { - subdes := resMap[pr.subject.String()] - members := subdes.Members() - if len(members) == 0 { - return nil, nil - } - zipName := "" - if title := subdes.Title(); title == "" { - zipName = "download.zip" - } else { - zipName = title + ".zip" - } - subjectPath := pr.subjectBasePath - if !strings.Contains(subjectPath, "/-/") { - subjectPath += "/-" - } - - return &publish.PageMembers{ - SubjectPath: subjectPath, - ZipName: zipName, - Members: members, - Description: func(member *search.DescribedBlob) string { - des := member.Description() - if des != "" { - des = " - " + des - } - return des - }, - Title: func(member *search.DescribedBlob) string { - memberTitle := member.Title() - if memberTitle == "" { - memberTitle = member.BlobRef.DigestPrefix(10) - } - return html.EscapeString(memberTitle) - }, - Path: func(member *search.DescribedBlob) string { - return pr.memberPath(member.BlobRef) - }, - DomID: func(member *search.DescribedBlob) string { - return member.DomID() - }, - FileInfo: func(member *search.DescribedBlob) *publish.MemberFileInfo { - if path, fileInfo, ok := member.PermanodeFile(); ok { - info := &publish.MemberFileInfo{ - FileName: fileInfo.FileName, - FileDomID: path[len(path)-1].DomID(), - FilePath: html.EscapeString(pr.SubresFileURL(path, fileInfo.FileName)), - } - if fileInfo.IsImage() { - info.FileThumbnailURL = pr.SubresThumbnailURL(path, fileInfo.FileName, 200) - } - return info - } - return nil - }, - }, nil -} - -// serveSubjectTemplate creates the funcs to generate the PageHeader, PageFile, -// and pageMembers that can be used by the subject template, and serves the template. -func (pr *publishRequest) serveSubjectTemplate() { - dr := pr.ph.Search.NewDescribeRequest() - dr.Describe(pr.subject, 3) - res, err := dr.Result() - if err != nil { - log.Printf("Errors loading %s, permanode %s: %v, %#v", pr.req.URL, pr.subject, err, err) - http.Error(pr.rw, "Error loading describe request", http.StatusInternalServerError) - return - } - subdes := res[pr.subject.String()] - if subdes.CamliType == "file" { - pr.serveFileDownload(subdes) - return - } - - headerFunc := func() *publish.PageHeader { - return pr.subjectHeader(res) - } - fileFunc := func() *publish.PageFile { - file, err := pr.subjectFile(res) - if err != nil { - log.Printf("%v", err) - return nil - } - return file - } - membersFunc := func() *publish.PageMembers { - members, err := pr.subjectMembers(res) - if err != nil { - log.Printf("%v", err) - return nil - } - return members - } - page := &publish.SubjectPage{ - Header: headerFunc, - File: fileFunc, - Members: membersFunc, - } - - err = pr.ph.goTemplate.Execute(pr.rw, page) - if err != nil { - log.Printf("Error serving subject template: %v", err) - http.Error(pr.rw, "Error serving template", http.StatusInternalServerError) - return - } -} - -func (pr *publishRequest) validPathChain(path []blob.Ref) bool { - bi := pr.subject - for len(path) > 0 { - var next blob.Ref - next, path = path[0], path[1:] - - desi, err := pr.dr.DescribeSync(bi) - if err != nil { - return false - } - if !desi.HasSecureLinkTo(next) { - return false - } - bi = next - } - return true -} - -func (pr *publishRequest) serveSubresImage() { - params := pr.req.URL.Query() - mw, _ := strconv.Atoi(params.Get("mw")) - mh, _ := strconv.Atoi(params.Get("mh")) - des, err := pr.dr.DescribeSync(pr.subject) - if err != nil { - log.Printf("error describing subject %q: %v", pr.subject, err) - return - } - pr.serveScaledImage(des, mw, mh, params.Get("square") == "1") -} - -func (pr *publishRequest) serveSubresFileDownload() { - des, err := pr.dr.DescribeSync(pr.subject) - if err != nil { - log.Printf("error describing subject %q: %v", pr.subject, err) - return - } - pr.serveFileDownload(des) -} - -func (pr *publishRequest) serveScaledImage(des *search.DescribedBlob, maxWidth, maxHeight int, square bool) { - fileref, _, ok := pr.fileSchemaRefFromBlob(des) - if !ok { - log.Printf("scaled image fail; failed to get file schema from des %q", des.BlobRef) - return - } - th := &ImageHandler{ - Fetcher: pr.ph.Storage, - Cache: pr.ph.Cache, - MaxWidth: maxWidth, - MaxHeight: maxHeight, - Square: square, - thumbMeta: pr.ph.thumbMeta, - resizeSem: pr.resizeSem, - } - th.ServeHTTP(pr.rw, pr.req, fileref) -} - -func (pr *publishRequest) serveFileDownload(des *search.DescribedBlob) { - fileref, fileinfo, ok := pr.fileSchemaRefFromBlob(des) - if !ok { - log.Printf("Didn't get file schema from described blob %q", des.BlobRef) - return - } - mime := "" - if fileinfo != nil && fileinfo.IsImage() { - mime = fileinfo.MIMEType - } - dh := &DownloadHandler{ - Fetcher: pr.ph.Storage, - Cache: pr.ph.Cache, - ForceMime: mime, - } - dh.ServeHTTP(pr.rw, pr.req, fileref) -} - -// Given a described blob, optionally follows a camliContent and -// returns the file's schema blobref and its fileinfo (if found). -func (pr *publishRequest) fileSchemaRefFromBlob(des *search.DescribedBlob) (fileref blob.Ref, fileinfo *camtypes.FileInfo, ok bool) { - if des == nil { - http.NotFound(pr.rw, pr.req) - return - } - if des.Permanode != nil { - // TODO: get "forceMime" attr out of the permanode? or - // fileName content-disposition? - if cref := des.Permanode.Attr.Get("camliContent"); cref != "" { - cbr, ok2 := blob.Parse(cref) - if !ok2 { - http.Error(pr.rw, "bogus camliContent", 500) - return - } - des = des.PeerBlob(cbr) - if des == nil { - http.Error(pr.rw, "camliContent not a peer in describe", 500) - return - } - } - } - if des.CamliType == "file" { - return des.BlobRef, des.File, true - } - http.Error(pr.rw, "failed to find fileSchemaRefFromBlob", 404) - return -} - -func (ph *PublishHandler) signUpload(jsonSign *signhandler.Handler, name string, bb *schema.Builder) (blob.Ref, error) { - signed, err := jsonSign.Sign(bb) - if err != nil { - return blob.Ref{}, fmt.Errorf("error signing %s: %v", name, err) - } - uh := client.NewUploadHandleFromString(signed) - _, err = blobserver.Receive(ph.Storage, uh.BlobRef, uh.Contents) - if err != nil { - return blob.Ref{}, fmt.Errorf("error uploading %s: %v", name, err) - } - return uh.BlobRef, nil -} - -func (ph *PublishHandler) setRootNode(jsonSign *signhandler.Handler, pn blob.Ref) (err error) { - _, err = ph.signUpload(jsonSign, "set-attr camliRoot", schema.NewSetAttributeClaim(pn, "camliRoot", ph.RootName)) - if err != nil { - return err - } - _, err = ph.signUpload(jsonSign, "set-attr title", schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+ph.RootName)) - return err -} - -func (ph *PublishHandler) bootstrapPermanode(jsonSign *signhandler.Handler) (err error) { - result, err := ph.camliRootQuery() - if err == nil && len(result.Blobs) > 0 && result.Blobs[0].Blob.Valid() { - log.Printf("Publish root %q using existing permanode %s", ph.RootName, result.Blobs[0].Blob) - return nil - } - - log.Printf("Publish root %q needs a permanode + claim", ph.RootName) - pn, err := ph.signUpload(jsonSign, "permanode", schema.NewUnsignedPermanode()) - if err != nil { - return err - } - err = ph.setRootNode(jsonSign, pn) - return err -} diff --git a/pkg/server/root.go b/pkg/server/root.go index 730f031e0..61ab36d7e 100644 --- a/pkg/server/root.go +++ b/pkg/server/root.go @@ -143,7 +143,7 @@ func (rh *RootHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { return } if req.URL.Path == "/favicon.ico" { - serveStaticFile(rw, req, Files, "favicon.ico") + ServeStaticFile(rw, req, Files, "favicon.ico") return } f := func(p string, a ...interface{}) { diff --git a/pkg/server/thumbcache.go b/pkg/server/thumbcache.go index 4e6f401ad..925618f78 100644 --- a/pkg/server/thumbcache.go +++ b/pkg/server/thumbcache.go @@ -27,30 +27,33 @@ import ( const memLRUSize = 1024 // arbitrary -// thumbMeta is a mapping from an image's scaling parameters (encoding +var errCacheMiss = errors.New("not in cache") + +// ThumbMeta is a mapping from an image's scaling parameters (encoding // as an opaque "key" string) and the blobref of the thumbnail -// (currently it's file schema blob) +// (currently its file schema blob). +// ThumbMeta is safe for concurrent use by multiple goroutines. // // The key will be some string containing the original full-sized image's // blobref, its target dimensions, and any possible transformations on // it (e.g. cropping it to square). - -var errCacheMiss = errors.New("not in cache") - -type thumbMeta struct { - mem *lru.Cache // string (see key format) -> blob.Ref +type ThumbMeta struct { + mem *lru.Cache // key -> blob.Ref kv sorted.KeyValue // optional } -// kv is optional -func newThumbMeta(kv sorted.KeyValue) *thumbMeta { - return &thumbMeta{ +// NewThumbMeta returns a new in-memory ThumbMeta, backed with the +// optional kv. +// If kv is nil, key/value pairs are stored in memory only. +func NewThumbMeta(kv sorted.KeyValue) *ThumbMeta { + return &ThumbMeta{ mem: lru.New(memLRUSize), kv: kv, } } -func (m *thumbMeta) Get(key string) (br blob.Ref, err error) { +func (m *ThumbMeta) Get(key string) (blob.Ref, error) { + var br blob.Ref if v, ok := m.mem.Get(key); ok { return v.(blob.Ref), nil } @@ -72,7 +75,7 @@ func (m *thumbMeta) Get(key string) (br blob.Ref, err error) { return br, errCacheMiss } -func (m *thumbMeta) Put(key string, br blob.Ref) error { +func (m *ThumbMeta) Put(key string, br blob.Ref) error { m.mem.Add(key, br) if m.kv != nil { return m.kv.Set(key, br.String()) diff --git a/pkg/server/ui.go b/pkg/server/ui.go index d27f4f8d0..f6937b9c9 100644 --- a/pkg/server/ui.go +++ b/pkg/server/ui.go @@ -38,6 +38,7 @@ import ( "camlistore.org/pkg/jsonsign/signhandler" "camlistore.org/pkg/misc/closure" "camlistore.org/pkg/search" + "camlistore.org/pkg/server/app" "camlistore.org/pkg/sorted" "camlistore.org/pkg/syncutil" uistatic "camlistore.org/server/camlistored/ui" @@ -77,8 +78,7 @@ type UIHandler struct { // if we start having clients (like phones) that we want to upload // but don't trust to have private signing keys? JSONSignRoot string - - publishRoots map[string]*PublishHandler + publishRoots map[string]*publishRoot prefix string // of the UI handler itself root *RootHandler @@ -90,7 +90,7 @@ type UIHandler struct { // Limit peak RAM used by concurrent image thumbnail calls. resizeSem *syncutil.Sem - thumbMeta *thumbMeta // optional thumbnail key->blob.Ref cache + thumbMeta *ThumbMeta // optional thumbnail key->blob.Ref cache // sourceRoot optionally specifies the path to root of Camlistore's // source. If empty, the UI files must be compiled in to the @@ -127,7 +127,6 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes", constants.DefaultMaxResizeMem))), } - pubRoots := conf.OptionalList("publishRoots") cachePrefix := conf.OptionalString("cache", "") scaledImageConf := conf.OptionalObject("scaledImage") if err = conf.Validate(); err != nil { @@ -141,24 +140,6 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er } } - if os.Getenv("CAMLI_PUBLISH_ENABLED") == "false" { - // Hack for dev server, to simplify its config with devcam server --publish=false. - pubRoots = nil - } - - ui.publishRoots = make(map[string]*PublishHandler) - for _, pubRoot := range pubRoots { - h, err := ld.GetHandler(pubRoot) - if err != nil { - return nil, fmt.Errorf("UI handler's publishRoots references invalid %q", pubRoot) - } - pubh, ok := h.(*PublishHandler) - if !ok { - return nil, fmt.Errorf("UI handler's publishRoots references invalid %q; not a PublishHandler", pubRoot) - } - ui.publishRoots[pubRoot] = pubh - } - checkType := func(key string, htype string) { v := conf.OptionalString(key, "") if v == "" { @@ -190,7 +171,7 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err) } ui.Cache = bs - ui.thumbMeta = newThumbMeta(scaledImageKV) + ui.thumbMeta = NewThumbMeta(scaledImageKV) } if ui.sourceRoot == "" { @@ -259,6 +240,79 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er return ui, nil } +type publishRoot struct { + Name string + Permanode blob.Ref + Prefix string +} + +// InitHandler goes through all the other configured handlers to discover +// the publisher ones, and uses them to populate ui.publishRoots. +func (ui *UIHandler) InitHandler(hl blobserver.FindHandlerByTyper) error { + searchPrefix, _, err := hl.FindHandlerByType("search") + if err != nil { + return errors.New("No search handler configured, which is necessary for the ui handler") + } + var sh *search.Handler + htype, hi := hl.AllHandlers() + if h, ok := hi[searchPrefix]; !ok { + return errors.New("failed to find the \"search\" handler") + } else { + sh = h.(*search.Handler) + } + camliRootQuery := func(camliRoot string) (*search.SearchResult, error) { + return sh.Query(&search.SearchQuery{ + Limit: 1, + Constraint: &search.Constraint{ + Permanode: &search.PermanodeConstraint{ + Attr: "camliRoot", + Value: camliRoot, + }, + }, + }) + } + for prefix, typ := range htype { + if typ != "app" { + continue + } + ah, ok := hi[prefix].(*app.Handler) + if !ok { + panic(fmt.Sprintf("UI: handler for %v has type \"app\" but is not app.Handler", prefix)) + } + if ah.ProgramName() != "publisher" { + continue + } + appConfig := ah.AppConfig() + if appConfig == nil { + log.Printf("UI: app handler for %v has no appConfig", prefix) + continue + } + camliRoot, ok := appConfig["camliRoot"].(string) + if !ok { + log.Printf("UI: camliRoot in appConfig is %T, want string, was %T", appConfig["camliRoot"]) + continue + } + result, err := camliRootQuery(camliRoot) + if err != nil { + log.Printf("UI: could not find permanode for camliRoot %v: %v", camliRoot, err) + continue + } + if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() { + log.Printf("UI: no valid permanode for camliRoot %v", camliRoot) + continue + } + if ui.publishRoots == nil { + ui.publishRoots = make(map[string]*publishRoot) + } + ui.publishRoots[prefix] = &publishRoot{ + Name: camliRoot, + Prefix: prefix, + Permanode: result.Blobs[0].Blob, + } + } + return nil +} + func (ui *UIHandler) makeClosureHandler(root string) (http.Handler, error) { return makeClosureHandler(root, "ui") } @@ -415,11 +469,12 @@ func (ui *UIHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { serveDepsJS(rw, req, ui.uiDir) return } - serveStaticFile(rw, req, uistatic.Files, file) + ServeStaticFile(rw, req, uistatic.Files, file) } } -func serveStaticFile(rw http.ResponseWriter, req *http.Request, root http.FileSystem, file string) { +// ServeStaticFile serves file from the root virtual filesystem. +func ServeStaticFile(rw http.ResponseWriter, req *http.Request, root http.FileSystem, file string) { f, err := root.Open("/" + file) if err != nil { http.NotFound(rw, req) @@ -441,19 +496,13 @@ func serveStaticFile(rw http.ResponseWriter, req *http.Request, root http.FileSy func (ui *UIHandler) populateDiscoveryMap(m map[string]interface{}) { pubRoots := map[string]interface{}{} - for key, pubh := range ui.publishRoots { + for _, v := range ui.publishRoots { m := map[string]interface{}{ - "name": pubh.RootName, - "prefix": []string{key}, - // TODO: include gpg key id + "name": v.Name, + "prefix": []string{v.Prefix}, + "currentPermanode": v.Permanode.String(), } - if sh, ok := ui.root.SearchHandler(); ok { - pn, err := sh.Index().PermanodeOfSignerAttrValue(sh.Owner(), "camliRoot", pubh.RootName) - if err == nil { - m["currentPermanode"] = pn.String() - } - } - pubRoots[pubh.RootName] = m + pubRoots[v.Name] = m } uiDisco := map[string]interface{}{ @@ -537,8 +586,8 @@ func (ui *UIHandler) serveThumbnail(rw http.ResponseWriter, req *http.Request) { Cache: ui.Cache, MaxWidth: width, MaxHeight: height, - thumbMeta: ui.thumbMeta, - resizeSem: ui.resizeSem, + ThumbMeta: ui.thumbMeta, + ResizeSem: ui.resizeSem, } th.ServeHTTP(rw, req, blobref) } @@ -597,7 +646,7 @@ func (ui *UIHandler) serveFromDiskOrStatic(rw http.ResponseWriter, req *http.Req req.URL.Path = "/" + file disk.ServeHTTP(rw, req) } else { - serveStaticFile(rw, req, static, file) + ServeStaticFile(rw, req, static, file) } } diff --git a/pkg/serverinit/genconfig.go b/pkg/serverinit/genconfig.go index 7f1a30ca9..a7703e26c 100644 --- a/pkg/serverinit/genconfig.go +++ b/pkg/serverinit/genconfig.go @@ -63,43 +63,34 @@ func addPublishedConfig(prefixes jsonconfig.Obj, sourceRoot string) ([]string, error) { var pubPrefixes []string for k, v := range published { - name := strings.Replace(k, "/", "", -1) - rootName := name + "Root" - if !v.Root.Valid() { - return nil, fmt.Errorf("Invalid or missing \"rootPermanode\" key in configuration for %s.", k) + if v.CamliRoot == "" { + return nil, fmt.Errorf("Missing \"camliRoot\" key in configuration for %s.", k) } if v.GoTemplate == "" { return nil, fmt.Errorf("Missing \"goTemplate\" key in configuration for %s.", k) } ob := map[string]interface{}{} - ob["handler"] = "publish" + ob["handler"] = "app" + + appConfig := map[string]interface{}{ + "camliRoot": v.CamliRoot, + "cacheRoot": v.CacheRoot, + "goTemplate": v.GoTemplate, + } + handlerArgs := map[string]interface{}{ - "rootName": rootName, - "blobRoot": "/bs-and-maybe-also-index/", - "searchRoot": "/my-search/", - "cache": "/cache/", - "rootPermanode": []interface{}{"/sighelper/", v.Root.String()}, + "program": v.Program, + "appConfig": appConfig, } - if sourceRoot != "" { - handlerArgs["sourceRoot"] = sourceRoot + if v.BaseURL != "" { + handlerArgs["baseURL"] = v.BaseURL } - handlerArgs["goTemplate"] = v.GoTemplate - if v.Style != "" { - handlerArgs["css"] = []interface{}{v.Style} - } - if v.Javascript != "" { - handlerArgs["js"] = []interface{}{v.Javascript} - } - // TODO(mpl): we'll probably want to use osutil.CacheDir() if thumbnails.kv - // contains private info? same for some of the other "camli-cache" ones? - thumbsCacheDir := filepath.Join(tempDir(), "camli-cache") - handlerArgs["scaledImage"] = map[string]interface{}{ - "type": "kv", - "file": filepath.Join(thumbsCacheDir, name+"-thumbnails.kv"), - } - if err := os.MkdirAll(thumbsCacheDir, 0700); err != nil { - return nil, fmt.Errorf("Could not create cache dir %s: %v", thumbsCacheDir, err) + program := "publisher" + if v.Program != "" { + program = v.Program } + handlerArgs["program"] = program + ob["handlerArgs"] = handlerArgs prefixes[k] = ob pubPrefixes = append(pubPrefixes, k) @@ -111,20 +102,12 @@ func addPublishedConfig(prefixes jsonconfig.Obj, func addUIConfig(params *configPrefixesParams, prefixes jsonconfig.Obj, uiPrefix string, - published []string, sourceRoot string) { args := map[string]interface{}{ "jsonSignRoot": "/sighelper/", "cache": "/cache/", } - if len(published) > 0 { - var publishedAsList []interface{} - for _, v := range published { - publishedAsList = append(publishedAsList, v) - } - args["publishRoots"] = publishedAsList - } if sourceRoot != "" { args["sourceRoot"] = sourceRoot } @@ -674,19 +657,18 @@ func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err err } } - var published []string if len(conf.Publish) > 0 { if !runIndex { return nil, fmt.Errorf("publishing requires an index") } - published, err = addPublishedConfig(prefixes, conf.Publish, conf.SourceRoot) + _, err = addPublishedConfig(prefixes, conf.Publish, conf.SourceRoot) if err != nil { return nil, fmt.Errorf("Could not generate config for published: %v", err) } } if runIndex { - addUIConfig(prefixesParams, prefixes, "/ui/", published, conf.SourceRoot) + addUIConfig(prefixesParams, prefixes, "/ui/", conf.SourceRoot) } if conf.MySQL != "" { diff --git a/pkg/serverinit/testdata/multipublish-want.json b/pkg/serverinit/testdata/multipublish-want.json index f920763ce..a5fd124a4 100644 --- a/pkg/serverinit/testdata/multipublish-want.json +++ b/pkg/serverinit/testdata/multipublish-want.json @@ -60,27 +60,15 @@ } }, "/music/": { - "handler": "publish", + "handler": "app", "handlerArgs": { - "blobRoot": "/bs-and-maybe-also-index/", - "cache": "/cache/", - "css": [ - "pics.css" - ], - "goTemplate": "gallery.html", - "js": [ - "pics.js" - ], - "rootName": "musicRoot", - "rootPermanode": [ - "/sighelper/", - "sha1-999c6aae4ec8245dfe63edc4a2abb407824a4b5a" - ], - "scaledImage": { - "file": "/tmp/camli-cache/music-thumbnails.kv", - "type": "kv" - }, - "searchRoot": "/my-search/" + "program": "publisher", + "baseURL": "http://localhost:3178/", + "appConfig": { + "camliRoot": "musicRoot", + "goTemplate": "music.html", + "cacheRoot": "/tmp/blobs/cache" + } } }, "/my-search/": { @@ -92,27 +80,14 @@ } }, "/pics/": { - "handler": "publish", + "handler": "app", "handlerArgs": { - "blobRoot": "/bs-and-maybe-also-index/", - "cache": "/cache/", - "css": [ - "pics.css" - ], - "goTemplate": "gallery.html", - "js": [ - "pics.js" - ], - "rootName": "picsRoot", - "rootPermanode": [ - "/sighelper/", - "sha1-046c6aae4ec8245dfe63edc4a2abb407824a4b5a" - ], - "scaledImage": { - "file": "/tmp/camli-cache/pics-thumbnails.kv", - "type": "kv" - }, - "searchRoot": "/my-search/" + "program": "publisher", + "appConfig": { + "camliRoot": "picsRoot", + "goTemplate": "gallery.html", + "cacheRoot": "/tmp/blobs/cache" + } } }, "/setup/": { @@ -151,10 +126,6 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "publishRoots": [ - "/music/", - "/pics/" - ], "scaledImage": { "file": "/tmp/blobs/thumbmeta.kv", "type": "kv" diff --git a/pkg/serverinit/testdata/multipublish.json b/pkg/serverinit/testdata/multipublish.json index 4ce1f8523..db23f87e3 100644 --- a/pkg/serverinit/testdata/multipublish.json +++ b/pkg/serverinit/testdata/multipublish.json @@ -2,23 +2,22 @@ "listen": "localhost:3179", "auth": "userpass:camlistore:pass3179", "blobPath": "/tmp/blobs", - "kvIndexFile": "/path/to/indexkv.db", + "kvIndexFile": "/path/to/indexkv.db", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", "ownerName": "Alice", "shareHandlerPath": "/share/", - "publish": { - "/pics/": { - "rootPermanode": "sha1-046c6aae4ec8245dfe63edc4a2abb407824a4b5a", - "style": "pics.css", - "js": "pics.js", - "goTemplate": "gallery.html" - }, - "/music/": { - "rootPermanode": "sha1-999c6aae4ec8245dfe63edc4a2abb407824a4b5a", - "style": "pics.css", - "js": "pics.js", - "goTemplate": "gallery.html" - } - } + "publish": { + "/pics/": { + "camliRoot": "picsRoot", + "cacheRoot": "/tmp/blobs/cache", + "goTemplate": "gallery.html" + }, + "/music/": { + "camliRoot": "musicRoot", + "baseURL": "http://localhost:3178/", + "cacheRoot": "/tmp/blobs/cache", + "goTemplate": "music.html" + } + } } diff --git a/pkg/serverinit/testdata/with_blog-want.json b/pkg/serverinit/testdata/with_blog-want.json index 05c34fd0c..c20ab925d 100644 --- a/pkg/serverinit/testdata/with_blog-want.json +++ b/pkg/serverinit/testdata/with_blog-want.json @@ -13,24 +13,15 @@ } }, "/blog/": { - "handler": "publish", + "handler": "app", "handlerArgs": { - "blobRoot": "/bs-and-maybe-also-index/", - "cache": "/cache/", - "css": [ - "blog-purple.css" - ], - "goTemplate": "blog.html", - "rootName": "blogRoot", - "rootPermanode": [ - "/sighelper/", - "sha1-2790ec1ec6fd44b9620b21155c8738aa08d4e3a0" - ], - "scaledImage": { - "file": "/tmp/camli-cache/blog-thumbnails.kv", - "type": "kv" - }, - "searchRoot": "/my-search/" + "program": "publisher", + "baseURL": "http://localhost:3178/", + "appConfig": { + "camliRoot": "blogRoot", + "goTemplate": "blog.html", + "cacheRoot": "/tmp/blobs/cache" + } } }, "/bs-and-index/": { @@ -123,9 +114,6 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "publishRoots": [ - "/blog/" - ], "scaledImage": { "file": "/tmp/blobs/thumbmeta.kv", "type": "kv" diff --git a/pkg/serverinit/testdata/with_blog.json b/pkg/serverinit/testdata/with_blog.json index 802189ce1..56096d7e2 100644 --- a/pkg/serverinit/testdata/with_blog.json +++ b/pkg/serverinit/testdata/with_blog.json @@ -9,9 +9,10 @@ "s3": "", "publish": { "/blog/": { - "rootPermanode": "sha1-2790ec1ec6fd44b9620b21155c8738aa08d4e3a0", - "goTemplate": "blog.html", - "style": "blog-purple.css" + "camliRoot": "blogRoot", + "baseURL": "http://localhost:3178/", + "cacheRoot": "/tmp/blobs/cache", + "goTemplate": "blog.html" } }, "replicateTo": [], diff --git a/pkg/serverinit/testdata/with_gallery-want.json b/pkg/serverinit/testdata/with_gallery-want.json index 81bb3645b..267581cc2 100644 --- a/pkg/serverinit/testdata/with_gallery-want.json +++ b/pkg/serverinit/testdata/with_gallery-want.json @@ -67,27 +67,15 @@ } }, "/pics/": { - "handler": "publish", + "handler": "app", "handlerArgs": { - "blobRoot": "/bs-and-maybe-also-index/", - "cache": "/cache/", - "css": [ - "pics.css" - ], - "goTemplate": "gallery.html", - "js": [ - "pics.js" - ], - "rootName": "picsRoot", - "rootPermanode": [ - "/sighelper/", - "sha1-2790ec1ec6fd44b9620b21155c8738aa08d4e3a0" - ], - "scaledImage": { - "file": "/tmp/camli-cache/pics-thumbnails.kv", - "type": "kv" - }, - "searchRoot": "/my-search/" + "program": "publisher", + "baseURL": "http://localhost:3178/", + "appConfig": { + "camliRoot": "picsRoot", + "goTemplate": "gallery.html", + "cacheRoot": "/tmp/blobs/cache" + } } }, "/setup/": { @@ -126,9 +114,6 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "publishRoots": [ - "/pics/" - ], "scaledImage": { "file": "/tmp/blobs/thumbmeta.kv", "type": "kv" diff --git a/pkg/serverinit/testdata/with_gallery.json b/pkg/serverinit/testdata/with_gallery.json index 0fac43266..062208019 100644 --- a/pkg/serverinit/testdata/with_gallery.json +++ b/pkg/serverinit/testdata/with_gallery.json @@ -9,10 +9,10 @@ "s3": "", "publish": { "/pics/": { - "rootPermanode": "sha1-2790ec1ec6fd44b9620b21155c8738aa08d4e3a0", - "goTemplate": "gallery.html", - "js": "pics.js", - "style": "pics.css" + "camliRoot": "picsRoot", + "baseURL": "http://localhost:3178/", + "cacheRoot": "/tmp/blobs/cache", + "goTemplate": "gallery.html" } }, "replicateTo": [], diff --git a/pkg/types/camtypes/search.go b/pkg/types/camtypes/search.go index a6945b6e6..4b09976a8 100644 --- a/pkg/types/camtypes/search.go +++ b/pkg/types/camtypes/search.go @@ -87,6 +87,9 @@ func (cl ClaimsByDate) String() string { type FileInfo struct { FileName string `json:"fileName"` + // TODO(mpl): I've noticed that Size is actually set to the + // number of entries in the dir. fix the doc or the behaviour? + // Size is the size of files. It is not set for directories. Size int64 `json:"size"` diff --git a/pkg/types/serverconfig/config.go b/pkg/types/serverconfig/config.go index f77b20250..fa9f43725 100644 --- a/pkg/types/serverconfig/config.go +++ b/pkg/types/serverconfig/config.go @@ -18,7 +18,6 @@ limitations under the License. package serverconfig import ( - "camlistore.org/pkg/blob" "camlistore.org/pkg/types" ) @@ -70,14 +69,27 @@ type Config struct { Picasa string `json:"picasa,omitempty"` // picasa importer. } -// Publish holds the server configuration values specific to publishing, i.e. to a publish handler. +// Publish holds the server configuration values specific to a publisher, i.e. to a publish prefix. type Publish struct { - // Root is the permanode used as the root for all the paths served by this publish handler. The camliRoot value that is the root path for this handler is a property of this permanode. - Root blob.Ref `json:"rootPermanode"` - // GoTemplate is the name of the Go template file used by this publish handler to represent the data. This file should live in server/camlistored/ui/. + // Program is the server app program to run as the publisher. + // Defaults to "publisher". + Program string `json:"program"` + + // CamliRoot value that defines our root permanode for this + // publisher. The root permanode is used as the root for all the + // paths served by this publisher. + CamliRoot string `json:"camliRoot"` + + // Base URL the app will run at. + BaseURL string `json:"baseURL,omitempty"` + + // GoTemplate is the name of the Go template file used by this + // publisher to represent the data. This file should live in + // app/publisher/. GoTemplate string `json:"goTemplate"` - // Javascript is the name of an optional javascript file used for additional features. This file should live in server/camlistored/ui/. - Javascript string `json:"js,omitempty"` - // Style is the name of an optional css file. This file should live in server/camlistored/ui/. - Style string `json:"style,omitempty"` + + // CacheRoot is the path that will be used as the root for the + // caching blobserver (for images). No caching if empty. + // An example value is Config.BlobPath + "/cache". + CacheRoot string `json:"cacheRoot,omitempty"` } diff --git a/server/camlistored/ui/pics.js b/server/camlistored/ui/pics.js deleted file mode 100644 index c5bdf9099..000000000 --- a/server/camlistored/ui/pics.js +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright 2013 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -goog.provide('cam.GalleryPage'); - -goog.require('goog.dom'); -goog.require('goog.events.EventHandler'); -goog.require('goog.events.EventType'); -goog.require('goog.ui.Component'); - -goog.require('cam.ServerConnection'); - -// @param {cam.ServerType.DiscoveryDocument} config Global config of the current server this page is being rendered for. -// @param {goog.dom.DomHelper=} opt_domHelper DOM helper to use. -cam.GalleryPage = function(config, opt_domHelper) { - goog.base(this, opt_domHelper); - - this.config_ = config; - this.connection_ = new cam.ServerConnection(config); -}; -goog.inherits(cam.GalleryPage, goog.ui.Component); - -cam.GalleryPage.prototype.decorateInternal = function(element) { - cam.GalleryPage.superClass_.decorateInternal.call(this, element); -}; - -cam.GalleryPage.prototype.disposeInternal = function() { - cam.GalleryPage.superClass_.disposeInternal.call(this); -}; - -cam.GalleryPage.prototype.enterDocument = function() { - cam.GalleryPage.superClass_.enterDocument.call(this); - - var members = goog.dom.getElement('members'); - if (!members) { - return; - } - var children = goog.dom.getChildren(members); - if (!children || children.length < 1) { - return; - } - goog.array.forEach(children, function(li) { - li.src = li.src + '&square=1'; - }) - - if (camliViewIsOwner) { - var el = this.getElement(); - goog.dom.classes.add(el, 'camliadmin'); - - goog.array.forEach(children, function(li) { - var lichild = goog.dom.getFirstElementChild(li); - var titleSpan = goog.dom.getLastElementChild(lichild); - var editLink = goog.dom.createElement('a', {'href': '#'}); - goog.dom.classes.add(editLink, 'hidden'); - goog.dom.setTextContent(editLink, 'edit title'); - - var titleInput = goog.dom.createElement('input'); - goog.dom.classes.add(titleInput, 'hidden'); - - goog.events.listen(editLink, - goog.events.EventType.CLICK, - function(e) { - goog.dom.classes.remove(titleSpan, 'visible'); - goog.dom.classes.add(titleSpan, 'hidden'); - goog.dom.classes.remove(titleInput, 'hidden'); - goog.dom.classes.add(titleInput, 'visible'); - titleInput.focus(); - titleInput.select(); - e.stopPropagation(); - e.preventDefault(); - }, - false, this - ); - goog.events.listen(li, - goog.events.EventType.MOUSEOVER, - function(e) { - goog.dom.classes.remove(editLink, 'hidden'); - goog.dom.classes.add(editLink, 'title-edit'); - }, - false, this - ); - goog.events.listen(li, - goog.events.EventType.MOUSEOUT, - function(e) { - goog.dom.classes.remove(editLink, 'title-edit'); - goog.dom.classes.add(editLink, 'hidden'); - goog.dom.classes.remove(titleInput, 'visible'); - goog.dom.classes.add(titleInput, 'hidden'); - goog.dom.classes.remove(titleSpan, 'hidden'); - goog.dom.classes.add(titleSpan, 'visible'); - }, - false, this - ); - goog.events.listen(titleInput, - goog.events.EventType.KEYPRESS, - goog.bind(function(e) { - if (e.keyCode == 13) { - this.saveImgTitle_(titleInput, titleSpan); - } - }, this), - false, this - ); - goog.dom.insertSiblingBefore(editLink, titleSpan); - goog.dom.insertChildAt(li, titleInput, 1); - }, this - ) - } -} - -// @param {string} titleInput text field element for title -// @param {string} titleSpan span element containing the title -cam.GalleryPage.prototype.saveImgTitle_ = function (titleInput, titleSpan) { - var spanText = goog.dom.getTextContent(titleSpan); - var newVal = titleInput.value; - if (newVal != "" && newVal != spanText) { - goog.dom.setTextContent(titleSpan, newVal); - var blobRef = goog.dom.getParentElement(titleInput).id.replace(/^camli-/, ''); - this.connection_.newSetAttributeClaim( - blobRef, - "title", - newVal, - function() { - }, - function(msg) { - alert(msg); - } - ); - } - goog.dom.classes.remove(titleInput, 'visible'); - goog.dom.classes.add(titleInput, 'hidden'); - goog.dom.classes.remove(titleSpan, 'hidden'); - goog.dom.classes.add(titleSpan, 'visible'); -} - -cam.GalleryPage.prototype.exitDocument = function() { - cam.GalleryPage.superClass_.exitDocument.call(this); -}; diff --git a/website/content/docs/server-config b/website/content/docs/server-config index ebeb80874..b519dbf3c 100644 --- a/website/content/docs/server-config +++ b/website/content/docs/server-config @@ -72,14 +72,14 @@ config, as used by devcam server.

 "publish": {
 	"/pics/": {
-		"rootPermanode": "sha1-6cbe9e1c35e854eab028cba43d099d35ceae0de8",
-		"style": "pics.css",
-		"js": "pics.js",
+		"camliRoot": "mypics",
+		"baseURL": "http://localhost:3178/",
+		"cacheRoot": "/home/joe/var/camlistore/blobs/cache",
 		"goTemplate": "gallery.html"
 	}
 }
-

One can create any permanode with camput or the UI and use it as the rootPermanode.

+

One can create any permanode with camput or the UI, and set its camliRoot attribute to the value set in the config, to use it as the root permanode for publishing.

Please see the publishing README if you want to make/contribute more publishing views.