From 2a747ccd5690319546f52a9cf3a6b36a5fda5bac Mon Sep 17 00:00:00 2001 From: Aaron Boodman Date: Mon, 18 Nov 2013 20:53:46 -0800 Subject: [PATCH 1/3] Add ability to specify flickr config for devcam. Change-Id: Ic1139730d969558e2d6375dd86c6741e3dfd2b9b --- config/.gitignore | 1 + config/dev-server-config.json | 8 ++++++++ dev/devcam/server.go | 10 +++++++++- pkg/importer/flickr/auth.go | 4 ++++ pkg/importer/flickr/flickr.go | 19 ++++++++++++++----- pkg/serverconfig/genconfig.go | 9 ++++----- website/content/docs/server-config | 2 +- 7 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 config/.gitignore diff --git a/config/.gitignore b/config/.gitignore new file mode 100644 index 000000000..139232f09 --- /dev/null +++ b/config/.gitignore @@ -0,0 +1 @@ +flickr-credentials.json diff --git a/config/dev-server-config.json b/config/dev-server-config.json index dd80bbaa0..2b5735b5c 100644 --- a/config/dev-server-config.json +++ b/config/dev-server-config.json @@ -269,6 +269,14 @@ } }, + "/importer-flickr/": { + "handler": "importer-flickr", + "enabled": ["_env", "${CAMLI_FLICKR_ENABLED}", false], + "handlerArgs": { + "apiKey": ["_env", "${CAMLI_FLICKR_API_KEY}", ""] + } + }, + "/share/": { "handler": "share", "handlerArgs": { diff --git a/dev/devcam/server.go b/dev/devcam/server.go index 436c9715b..a15c5ae89 100644 --- a/dev/devcam/server.go +++ b/dev/devcam/server.go @@ -61,7 +61,8 @@ type serverCmd struct { fullClosure bool - openBrowser bool + openBrowser bool + flickrAPIKey string // end of flag vars listen string // address + port to listen on @@ -94,6 +95,7 @@ func init() { flags.BoolVar(&cmd.fullClosure, "fullclosure", false, "Use the ondisk closure library.") flags.BoolVar(&cmd.openBrowser, "openbrowser", false, "Open the start page on startup.") + flags.StringVar(&cmd.flickrAPIKey, "flickrapikey", "", "The key and secret to use with the Flickr importer. Formatted as ':'.") return cmd }) } @@ -143,6 +145,7 @@ func (c *serverCmd) setCamliRoot() error { if err := os.RemoveAll(c.camliRoot); err != nil { return fmt.Errorf("Could not wipe %v: %v", c.camliRoot, err) } + os.Remove(filepath.Join("config", "flickr-credentials.json")) } return nil } @@ -249,6 +252,11 @@ func (c *serverCmd) setEnvVars() error { setenv("CAMLI_SECRET_RING", filepath.Join(camliSrcRoot, filepath.FromSlash(defaultSecring))) setenv("CAMLI_KEYID", defaultKeyID) + if c.flickrAPIKey != "" { + setenv("CAMLI_FLICKR_ENABLED", "true") + setenv("CAMLI_FLICKR_API_KEY", c.flickrAPIKey) + } + setenv("CAMLI_CONFIG_DIR", "config") return nil } diff --git a/pkg/importer/flickr/auth.go b/pkg/importer/flickr/auth.go index e783c2fc7..754cca55f 100644 --- a/pkg/importer/flickr/auth.go +++ b/pkg/importer/flickr/auth.go @@ -61,8 +61,12 @@ func writeCredentials(user *userInfo) { } } +// This returns nil,nil if the file doesn't exist. Any other error bad. func readCredentials() (*userInfo, error) { fi, err := os.Open(userFile) + if os.IsNotExist(err) { + return nil, nil + } if err != nil { return nil, err } diff --git a/pkg/importer/flickr/flickr.go b/pkg/importer/flickr/flickr.go index b7755cd62..861f817f2 100644 --- a/pkg/importer/flickr/flickr.go +++ b/pkg/importer/flickr/flickr.go @@ -24,6 +24,7 @@ import ( "log" "net/http" "net/url" + "strings" "camlistore.org/pkg/importer" "camlistore.org/pkg/jsonconfig" @@ -45,14 +46,22 @@ type imp struct { } func newFromConfig(cfg jsonconfig.Obj, host *importer.Host) (importer.Importer, error) { - oauthClient.Credentials = oauth.Credentials{ - Token: cfg.OptionalString("appKey", ""), - Secret: cfg.OptionalString("appSecret", ""), - } + apiKey := cfg.RequiredString("apiKey") if err := cfg.Validate(); err != nil { return nil, err } - user, _ := readCredentials() + parts := strings.Split(apiKey, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("Flickr importer: Invalid apiKey configuration: %q", apiKey) + } + oauthClient.Credentials = oauth.Credentials{ + Token: parts[0], + Secret: parts[1], + } + user, err := readCredentials() + if err != nil { + return nil, err + } return &imp{ host: host, user: user, diff --git a/pkg/serverconfig/genconfig.go b/pkg/serverconfig/genconfig.go index efab31280..ed3b2c9ba 100644 --- a/pkg/serverconfig/genconfig.go +++ b/pkg/serverconfig/genconfig.go @@ -44,7 +44,7 @@ type configPrefixesParams struct { blobPath string searchOwner blob.Ref shareHandlerPath string - flickr map[string]interface{} + flickr string } var ( @@ -445,10 +445,9 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json } } - if len(params.flickr) > 0 { + if params.flickr != "" { m["/importer-flickr/"] = map[string]interface{}{ - "handler": "importer-flickr", - "handlerArgs": params.flickr, + "apiKey": params.flickr, } } @@ -539,7 +538,7 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { kvFile = conf.OptionalString("kvIndexFile", "") // Importer options - flickr = conf.OptionalObject("flickr") + flickr = conf.OptionalString("flickr", "") _ = conf.OptionalList("replicateTo") publish = conf.OptionalObject("publish") diff --git a/website/content/docs/server-config b/website/content/docs/server-config index fa08b0f1a..4cb931259 100644 --- a/website/content/docs/server-config +++ b/website/content/docs/server-config @@ -23,7 +23,7 @@ web browser and restart the server.

  • baseURL: Optional. If non-empty, this is the root of your URL prefix for your Camlistore server. Useful for when running behind a reverse proxy. Should not end in a slash. e.g. https://yourserver.example.com
    • -
  • flickr: Optional, and doesn't do anything yet. Support for continuous import from Flickr. Enter two child keys: appKey and appSecret, which you can get by filling out this form.
    • +
  • flickr: Optional, and doesn't do anything yet. Support for continuous import from Flickr. Should be an API key and secret formatted as key:secret. Get yours by filling out this form.
  • https: if "true", HTTPS is used
      From a6aeff034c9151220bcb5c5bdab71b5d7fa61e36 Mon Sep 17 00:00:00 2001 From: mpl Date: Mon, 28 Oct 2013 16:59:07 +0100 Subject: [PATCH 2/3] server/publish.go: use go html template for publishing Change-Id: Ic1c55d35e52e407864d339927b453de58a64e03c --- config/dev-server-config.json | 2 + doc/publishing/README | 17 ++ pkg/publish/types.go | 89 +++++++ pkg/server/publish.go | 398 +++++++++++++++++------------ pkg/serverconfig/genconfig.go | 29 +-- server/camlistored/ui/blog.html | 6 + server/camlistored/ui/gallery.html | 67 +++++ website/content/docs/server-config | 17 +- 8 files changed, 430 insertions(+), 195 deletions(-) create mode 100644 doc/publishing/README create mode 100644 pkg/publish/types.go create mode 100644 server/camlistored/ui/blog.html create mode 100644 server/camlistored/ui/gallery.html diff --git a/config/dev-server-config.json b/config/dev-server-config.json index de29d7187..ae9f50161 100644 --- a/config/dev-server-config.json +++ b/config/dev-server-config.json @@ -24,6 +24,7 @@ "blobRoot": "/bs-and-maybe-also-index/", "searchRoot": "/my-search/", "cache": "/cache/", + "goTemplate": "blog.html", "devBootstrapPermanodeUsing": "/sighelper/" } }, @@ -38,6 +39,7 @@ "scaledImage": "lrucache", "css": ["pics.css"], "js": ["pics.js"], + "goTemplate": "gallery.html", "devBootstrapPermanodeUsing": "/sighelper/" } }, diff --git a/doc/publishing/README b/doc/publishing/README new file mode 100644 index 000000000..2d11b0b86 --- /dev/null +++ b/doc/publishing/README @@ -0,0 +1,17 @@ +Camlistore uses Go html templates (http://golang.org/pkg/text/template/) to publish pages. + +Resources for publishing, such as go templates, javascript and css files should be placed in server/camlistored/ui/, so they can be served directly when using the dev server or automatically embedded when using camlistored directly. + +You can then specify those resources through the configuration file. For example, there already is a go template (gallery.html), javascript file (pics.js) and css file (pics.css) that work together to provide publishing for image galleries. The dev server config (config/dev-server-config.json) already uses them. Here is how one would use them in the server config ($HOME/.config/camlistore/server-config.json): + +"publish": { + "/pics/": { + "rootPermanode": "sha1-6cbe9e1c35e854eab028cba43d099d35ceae0de8", + "style": "pics.css", + "js": "pics.js", + "goTemplate": "gallery.html" + } +} + +If you want to provide your own (Go) template, see http://camlistore.org/pkg/publish for the data structures and functions available to the template. + diff --git a/pkg/publish/types.go b/pkg/publish/types.go new file mode 100644 index 000000000..8e2f43d29 --- /dev/null +++ b/pkg/publish/types.go @@ -0,0 +1,89 @@ +/* +Copyright 2013 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package publish exposes the types and functions that can be used +// from a Go template, for publishing. +package publish + +import ( + "html/template" + + "camlistore.org/pkg/search" +) + +// SubjectPage is the data structure used when serving a +// publishing template. It contains the functions that can be called +// from the template. +type SubjectPage struct { + Header func() *PageHeader + File func() *PageFile + Members func() *PageMembers +} + +// PageHeader contains the data available to the template, +// and relevant to the page header. +type PageHeader struct { + Title string // Page title. + CSSFiles []string // Available CSS files. + JSDeps []string // Dependencies (for e.g closure) that can/should be included as javascript files. + CamliClosure template.JS // Closure namespace defined in the provided js. e.g camlistore.GalleryPage from pics.js + Subject string // Subject of this page (i.e the object which is described and published). + Meta string // All the metadata describing the subject of this page. + ViewerIsOwner bool // Whether the viewer of the page is also the owner of the displayed subject. (localhost check for now.) +} + +// PageFile contains the file related data available to the subject template, +// if the page describes some file contents. +type PageFile struct { + FileName string + Size int64 + MIMEType string + IsImage bool + DownloadURL string + ThumbnailURL string + DomID string + Nav func() *Nav +} + +// Nav holds links to the previous, next, and parent elements, +// when displaying members. +type Nav struct { + ParentPath string + PrevPath string + NextPath string +} + +// PageMembers contains the data relevant to the members if the published subject +// is a permanode with members. +type PageMembers struct { + SubjectPath string // URL prefix path to the subject (i.e the permanode). + ZipName string // Name of the downloadable zip file which contains all the members. + Members []*search.DescribedBlob // List of the members. + Description func(*search.DescribedBlob) string // Returns the description of the given member. + Title func(*search.DescribedBlob) string // Returns the title for the given member. + Path func(*search.DescribedBlob) string // Returns the url prefix path to the given the member. + DomID func(*search.DescribedBlob) string // Returns the Dom ID of the given member. + FileInfo func(*search.DescribedBlob) *MemberFileInfo // Returns some file info if the given member is a file permanode. +} + +// MemberFileInfo contains the file related data available for each member, +// if the member is the permanode for a file. +type MemberFileInfo struct { + FileName string + FileDomID string + FilePath string + FileThumbnailURL string +} diff --git a/pkg/server/publish.go b/pkg/server/publish.go index f2583a276..908609d71 100644 --- a/pkg/server/publish.go +++ b/pkg/server/publish.go @@ -23,6 +23,8 @@ import ( "errors" "fmt" "html" + "html/template" + "io/ioutil" "log" "net/http" "net/url" @@ -40,7 +42,7 @@ import ( "camlistore.org/pkg/httputil" "camlistore.org/pkg/jsonconfig" "camlistore.org/pkg/jsonsign/signhandler" - "camlistore.org/pkg/osutil" + "camlistore.org/pkg/publish" "camlistore.org/pkg/schema" "camlistore.org/pkg/search" uistatic "camlistore.org/server/camlistored/ui" @@ -55,7 +57,10 @@ type PublishHandler struct { Cache blobserver.Storage // or nil sc ScaledImage // cache of scaled images, optional - JSFiles, CSSFiles []string + CSSFiles []string + // goTemplate is the go html template used for publishing. + goTemplate *template.Template + closureName string // Name of the closure object used to decorate the published page. handlerFinder blobserver.FindHandlerByTyper @@ -80,8 +85,9 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han handlerFinder: ld, } ph.RootName = conf.RequiredString("rootName") - ph.JSFiles = conf.OptionalList("js") + jsFiles := conf.OptionalList("js") ph.CSSFiles = conf.OptionalList("css") + goTemplateFile := conf.RequiredString("goTemplate") blobRoot := conf.RequiredString("blobRoot") searchRoot := conf.RequiredString("searchRoot") cachePrefix := conf.OptionalString("cache", "") @@ -179,9 +185,47 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ph.sourceRoot, err) } + ph.goTemplate, err = goTemplate(goTemplateFile) + if err != nil { + return nil, err + } + ph.setClosureName(jsFiles) + return ph, nil } +func goTemplate(templateFile string) (*template.Template, error) { + if filepath.Base(templateFile) != templateFile { + hint := fmt.Sprintf("The file should either be embedded or placed in %s.", + filepath.FromSlash("server/camlistored/ui")) + return nil, fmt.Errorf("Unsupported path %v for template. %s", templateFile, hint) + } + f, err := uistatic.Files.Open(templateFile) + if err != nil { + return nil, fmt.Errorf("Could not open template %v: %v", templateFile, err) + } + defer f.Close() + templateBytes, err := ioutil.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("Could not read template %v: %v", templateFile, err) + } + + return template.Must(template.New("subject").Parse(string(templateBytes))), nil +} + +// setClosureName sets ph.closureName with the first found closure +// namespace provided in jsFiles. +func (ph *PublishHandler) setClosureName(jsFiles []string) { + for _, v := range jsFiles { + if ph.closureName == "" { + if cl := camliClosurePage(v); cl != "" { + ph.closureName = cl + break + } + } + } +} + func (ph *PublishHandler) makeClosureHandler(root string) (http.Handler, error) { return makeClosureHandler(root, "publish") } @@ -412,7 +456,7 @@ func (pr *publishRequest) serveHTTP() { switch pr.SubresourceType() { case "": - pr.serveSubject() + pr.serveSubjectTemplate() case "b": // TODO: download a raw blob case "f": // file download @@ -472,12 +516,7 @@ var provCamliRx = regexp.MustCompile(`^goog\.(provide)\(['"]camlistore\.(.*)['"] // and if yes, if it provides a page in the camlistore namespace. // It returns that page name, or the empty string otherwise. func camliClosurePage(filename string) string { - camliRootPath, err := osutil.GoPackagePath("camlistore.org") - if err != nil { - return "" - } - fullpath := filepath.Join(camliRootPath, "server", "camlistored", "ui", filename) - f, err := os.Open(fullpath) + f, err := uistatic.Files.Open(filename) if err != nil { return "" } @@ -517,37 +556,6 @@ func (pr *publishRequest) serveZip() { zh.ServeHTTP(pr.rw, pr.req) } -// serveHeader serves the html header with the relevant title, javascript -// and css includes. It is meant to be called from serveSubject. -func (pr *publishRequest) serveHeader(title, camliClosurePage string) { - pr.pf("\n\n\n %s\n", html.EscapeString(title)) - if camliClosurePage != "" && pr.ViewerIsOwner() { - pr.pf(" \n", pr.staticPath("closure/goog/base.js")) - pr.pf(" \n", pr.staticPath("deps.js")) - pr.pf(" \n", pr.base+"?camli.mode=config&var=CAMLISTORE_CONFIG") - pr.pf(" \n", pr.staticPath("base64.js")) - pr.pf(" \n", pr.staticPath("Crypto.js")) - pr.pf(" \n", pr.staticPath("SHA1.js")) - pr.pf("\n", camliClosurePage) - } - for _, filename := range pr.ph.CSSFiles { - pr.pf(" \n", pr.staticPath(filename)) - } -} - -// serveMeta serves all the described meta data about the published items, -// within the html header. It is meant to be called from serveSubject. -func (pr *publishRequest) serveMeta(des map[string]*search.DescribedBlob) { - pr.pf(" \n\n") -} - -// TODO(mpl): use those everywhere else const ( resSeparator = "/-" digestPrefix = "h" @@ -630,28 +638,115 @@ func (pr *publishRequest) parent() (parentPath string, parentBlobRef blob.Ref, e return parentPath, parentBlobRef, nil } -// serveNav serves some navigation links (prev, next, up) if the -// pr.subject is member of a collection (its parent has members). -// It is meant to be called from serveFile. -func (pr *publishRequest) serveNav() error { +func (pr *publishRequest) cssFiles() []string { + files := []string{} + for _, filename := range pr.ph.CSSFiles { + files = append(files, pr.staticPath(filename)) + } + return files +} + +// jsDeps returns the list of paths that should be included +// as javascript files in the published page to enable and use +// additional javascript closure code. +func (pr *publishRequest) jsDeps() []string { + var js []string + closureDeps := []string{ + "closure/goog/base.js", + "deps.js", + // TODO(mpl): fix the deps generator and/or the SHA1.js etc files so they get into deps.js and we + // do not even have to include them here. detection fails for them because the provide statement + // is not at the beginning of the line. + // Not doing it right away because it might have consequences for the rest of the ui I suppose. + "base64.js", + "Crypto.js", + "SHA1.js", + } + for _, v := range closureDeps { + js = append(js, pr.staticPath(v)) + } + js = append(js, pr.base+"?camli.mode=config&var=CAMLISTORE_CONFIG") + return js +} + +// subjectHeader returns the PageHeader corresponding to the described subject. +func (pr *publishRequest) subjectHeader(described map[string]*search.DescribedBlob) *publish.PageHeader { + subdes := described[pr.subject.String()] + header := &publish.PageHeader{ + Title: html.EscapeString(subdes.Title()), + CSSFiles: pr.cssFiles(), + Meta: func() string { + jsonRes, _ := json.MarshalIndent(described, "", " ") + return string(jsonRes) + }(), + Subject: pr.subject.String(), + } + header.JSDeps = pr.jsDeps() + header.CamliClosure = template.JS("camlistore." + pr.ph.closureName) + if pr.ViewerIsOwner() { + header.ViewerIsOwner = true + } + return header +} + +// subjectFile returns the relevant PageFile if the described subject is a file permanode. +func (pr *publishRequest) subjectFile(described map[string]*search.DescribedBlob) (*publish.PageFile, error) { + subdes := described[pr.subject.String()] + contentRef, ok := subdes.ContentRef() + if !ok { + return nil, nil + } + fileDes, err := pr.dr.DescribeSync(contentRef) + if err != nil { + return nil, fmt.Errorf("Could not describe %v: %v", contentRef, err) + } + path := []blob.Ref{pr.subject, contentRef} + downloadURL := pr.SubresFileURL(path, fileDes.File.FileName) + thumbnailURL := "" + if fileDes.File.IsImage() { + thumbnailURL = pr.SubresThumbnailURL(path, fileDes.File.FileName, 600) + } + fileName := html.EscapeString(fileDes.File.FileName) + return &publish.PageFile{ + FileName: fileName, + Size: fileDes.File.Size, + MIMEType: fileDes.File.MIMEType, + IsImage: fileDes.File.IsImage(), + DownloadURL: downloadURL, + ThumbnailURL: thumbnailURL, + DomID: contentRef.DomID(), + Nav: func() *publish.Nav { + nv, err := pr.fileNavigation() + if err != nil { + log.Print(err) + return nil + } + return nv + }, + }, nil +} + +func (pr *publishRequest) fileNavigation() (*publish.Nav, error) { // first get the parent path and blob parentPath, parentbr, err := pr.parent() if err != nil { - return fmt.Errorf("Errors building nav links for %s: %v", pr.subject, err) + return nil, fmt.Errorf("Could not get subject %v's parent's info: %v", pr.subject, err) + } + parentNav := strings.TrimSuffix(parentPath, resSeparator) + fileNav := &publish.Nav{ + ParentPath: parentNav, } - parentNav := fmt.Sprintf("[up]", strings.TrimSuffix(parentPath, resSeparator)) // describe the parent so we get the siblings (members of the parent) dr := pr.ph.Search.NewDescribeRequest() dr.Describe(parentbr, 3) parentRes, err := dr.Result() if err != nil { - return fmt.Errorf("Errors loading %s, permanode %s: %v, %#v", pr.req.URL, pr.subject, err, err) + return nil, fmt.Errorf("Could not \"deeply\" describe subject %v's parent %v: %v", pr.subject, parentbr, err) } members := parentRes[parentbr.String()].Members() if len(members) == 0 { - pr.pf("
      [up]
      ", parentNav) - return nil + return fileNav, nil } pos := 0 @@ -668,68 +763,27 @@ func (pr *publishRequest) serveNav() error { if pos < len(members)-1 { next = members[pos+1].BlobRef } - if prev.Valid() || next.Valid() { - var prevNav, nextNav string - if prev.Valid() { - prevNav = fmt.Sprintf("[prev]", - parentPath, prev.DigestPrefix(10)) - } - if next.Valid() { - nextNav = fmt.Sprintf("[next]", - parentPath, next.DigestPrefix(10)) - } - pr.pf("
      %s %s %s
      ", parentNav, prevNav, nextNav) + if !prev.Valid() && !next.Valid() { + return fileNav, nil } - - return nil + if prev.Valid() { + fileNav.PrevPath = fmt.Sprintf("%s/%s%s", parentPath, digestPrefix, prev.DigestPrefix(10)) + } + if next.Valid() { + fileNav.NextPath = fmt.Sprintf("%s/%s%s", parentPath, digestPrefix, next.DigestPrefix(10)) + } + return fileNav, nil } -// serveFile serves the relevant view when the subject in serveSubject -// is a permanode with some content cref. It is meant to be called -// from serveSubject. -func (pr *publishRequest) serveFile(cref blob.Ref) error { - des, err := pr.dr.DescribeSync(cref) - if err != nil { - pr.pf("

      Error serving file

      ") - return fmt.Errorf("Could not describe %v: %v", cref, err) +// subjectMembers returns the relevant PageMembers if the described subject is a permanode with members. +func (pr *publishRequest) subjectMembers(resMap map[string]*search.DescribedBlob) (*publish.PageMembers, error) { + subdes := resMap[pr.subject.String()] + members := subdes.Members() + if len(members) == 0 { + return nil, nil } - if des.File != nil { - path := []blob.Ref{pr.subject, cref} - downloadURL := pr.SubresFileURL(path, des.File.FileName) - pr.pf("
      File: %s, %d bytes, type %s
      ", - html.EscapeString(des.File.FileName), - des.File.Size, - des.File.MIMEType) - if des.File.IsImage() { - pr.pf("", - downloadURL, - pr.SubresThumbnailURL(path, des.File.FileName, 600)) - } - pr.pf("", - cref.DomID(), - downloadURL) - } - if strings.Contains(pr.subjectBasePath, resSeparator) { - // this permanode has a "parent" collection. - // so we send a deep request on the parent in order to get some info - // about the siblings and build some "prev" and "next" nav links. - // TODO(mpl): nav links everywhere, not just when showing a permanode - // with some content. - err := pr.serveNav() - if err != nil { - pr.pf("

      Error building navs links

      ") - return err - } - } - return nil -} - -// serveMembers serves the relevant view when the subject in serveSubject -// is a collection (permanode with members). It is meant to be called -// from serveSubject. -func (pr *publishRequest) serveMembers(title string, members []*search.DescribedBlob) { zipName := "" - if title == "" { + if title := subdes.Title(); title == "" { zipName = "download.zip" } else { zipName = title + ".zip" @@ -738,91 +792,95 @@ func (pr *publishRequest) serveMembers(title string, members []*search.Described if !strings.Contains(subjectPath, "/-/") { subjectPath += "/-" } - pr.pf("\n", subjectPath, - html.EscapeString(url.QueryEscape(zipName)), html.EscapeString(zipName)) - pr.pf("
        \n") - for _, member := range members { - des := member.Description() - if des != "" { - des = " - " + des - } - var fileLink, thumbnail string - if path, fileInfo, ok := member.PermanodeFile(); ok { - fileLink = fmt.Sprintf("", - path[len(path)-1].DomID(), - html.EscapeString(pr.SubresFileURL(path, fileInfo.FileName)), - ) - if fileInfo.IsImage() { - thumbnail = fmt.Sprintf("", pr.SubresThumbnailURL(path, fileInfo.FileName, 200)) + + return &publish.PageMembers{ + SubjectPath: subjectPath, + ZipName: zipName, + Members: subdes.Members(), + Description: func(member *search.DescribedBlob) string { + des := member.Description() + if des != "" { + des = " - " + des } - } - memberTitle := member.Title() - if memberTitle == "" { - memberTitle = member.BlobRef.DigestPrefix(10) - } - pr.pf("
      • %s%s%s%s
        • \n", - member.DomID(), - pr.memberPath(member.BlobRef), - thumbnail, - html.EscapeString(memberTitle), - des, - fileLink) - } - pr.pf("
      \n") + return des + }, + Title: func(member *search.DescribedBlob) string { + memberTitle := member.Title() + if memberTitle == "" { + memberTitle = member.BlobRef.DigestPrefix(10) + } + return html.EscapeString(memberTitle) + }, + Path: func(member *search.DescribedBlob) string { + return pr.memberPath(member.BlobRef) + }, + DomID: func(member *search.DescribedBlob) string { + return member.DomID() + }, + FileInfo: func(member *search.DescribedBlob) *publish.MemberFileInfo { + if path, fileInfo, ok := member.PermanodeFile(); ok { + info := &publish.MemberFileInfo{ + FileName: fileInfo.FileName, + FileDomID: path[len(path)-1].DomID(), + FilePath: html.EscapeString(pr.SubresFileURL(path, fileInfo.FileName)), + } + if fileInfo.IsImage() { + info.FileThumbnailURL = pr.SubresThumbnailURL(path, fileInfo.FileName, 200) + } + return info + } + return nil + }, + }, nil } -func (pr *publishRequest) serveSubject() { +// serveSubjectTemplate creates the funcs to generate the PageHeader, PageFile, +// and pageMembers that can be used by the subject template, and serves the template. +func (pr *publishRequest) serveSubjectTemplate() { dr := pr.ph.Search.NewDescribeRequest() dr.Describe(pr.subject, 3) res, err := dr.Result() if err != nil { log.Printf("Errors loading %s, permanode %s: %v, %#v", pr.req.URL, pr.subject, err, err) - pr.pf("

      Errors loading.

      ") + http.Error(pr.rw, "Error loading describe request", http.StatusInternalServerError) return } - subdes := res[pr.subject.String()] - if subdes.CamliType == "file" { pr.serveFileDownload(subdes) return } - title := subdes.Title() - - // HTML header + Javascript - var camliPage string - // TODO(mpl): We are only using the first .js file, and expecting it to be - // using closure. We want to be more customizable in the long run and enable - // some sort of templating mechanism. - if len(pr.ph.JSFiles) > 0 { - camliPage = camliClosurePage(pr.ph.JSFiles[0]) + headerFunc := func() *publish.PageHeader { + return pr.subjectHeader(res) } - pr.serveHeader(title, camliPage) - pr.serveMeta(res) - pr.pf("\n") - if title != "" { - pr.pf("

      %s

      \n", html.EscapeString(title)) - } - defer pr.pf("\n\n") - - if cref, ok := subdes.ContentRef(); ok { - err = pr.serveFile(cref) + fileFunc := func() *publish.PageFile { + file, err := pr.subjectFile(res) if err != nil { - log.Print(err) - return + log.Printf("%v", err) + return nil } - } else { - if members := subdes.Members(); len(members) > 0 { - pr.serveMembers(title, members) + return file + } + membersFunc := func() *publish.PageMembers { + members, err := pr.subjectMembers(res) + if err != nil { + log.Printf("%v", err) + return nil } + return members + } + page := &publish.SubjectPage{ + Header: headerFunc, + File: fileFunc, + Members: membersFunc, } - if camliPage != "" && pr.ViewerIsOwner() { - pr.pf("\n") + err = pr.ph.goTemplate.Execute(pr.rw, page) + if err != nil { + log.Printf("Error serving subject template: %v", err) + http.Error(pr.rw, "Error serving template", http.StatusInternalServerError) + return } } diff --git a/pkg/serverconfig/genconfig.go b/pkg/serverconfig/genconfig.go index 1f13de028..c67933921 100644 --- a/pkg/serverconfig/genconfig.go +++ b/pkg/serverconfig/genconfig.go @@ -61,7 +61,7 @@ func addPublishedConfig(prefixes jsonconfig.Obj, return nil, fmt.Errorf("Wrong type for %s; was expecting map[string]interface{}, got %T", k, v) } rootName := strings.Replace(k, "/", "", -1) + "Root" - rootPermanode, template, style := "", "", "" + rootPermanode, goTemplate, style, js := "", "", "", "" for pk, pv := range p { val, ok := pv.(string) if !ok { @@ -70,16 +70,18 @@ func addPublishedConfig(prefixes jsonconfig.Obj, switch pk { case "rootPermanode": rootPermanode = val - case "template": - template = val + case "goTemplate": + goTemplate = val case "style": style = val + case "js": + js = val default: return nil, fmt.Errorf("Unexpected key %q in config for %s", pk, k) } } - if rootPermanode == "" || template == "" { - return nil, fmt.Errorf("Missing key in configuration for %s, need \"rootPermanode\" and \"template\"", k) + if rootPermanode == "" || goTemplate == "" { + return nil, fmt.Errorf("Missing key in configuration for %s, need \"rootPermanode\" and \"goTemplate\"", k) } ob := map[string]interface{}{} ob["handler"] = "publish" @@ -93,19 +95,10 @@ func addPublishedConfig(prefixes jsonconfig.Obj, if sourceRoot != "" { handlerArgs["sourceRoot"] = sourceRoot } - switch template { - case "gallery": - if style == "" { - style = "pics.css" - } - handlerArgs["css"] = []interface{}{style} - handlerArgs["js"] = []interface{}{"pics.js"} - handlerArgs["scaledImage"] = "lrucache" - case "blog": - if style != "" { - handlerArgs["css"] = []interface{}{style} - } - } + handlerArgs["goTemplate"] = goTemplate + handlerArgs["css"] = []interface{}{style} + handlerArgs["js"] = []interface{}{js} + handlerArgs["scaledImage"] = "lrucache" ob["handlerArgs"] = handlerArgs prefixes[k] = ob pubPrefixes = append(pubPrefixes, k) diff --git a/server/camlistored/ui/blog.html b/server/camlistored/ui/blog.html new file mode 100644 index 000000000..f00297814 --- /dev/null +++ b/server/camlistored/ui/blog.html @@ -0,0 +1,6 @@ + + + + TODO + + diff --git a/server/camlistored/ui/gallery.html b/server/camlistored/ui/gallery.html new file mode 100644 index 000000000..85dbab549 --- /dev/null +++ b/server/camlistored/ui/gallery.html @@ -0,0 +1,67 @@ + + +{{if $header := call .Header}} + + {{$header.Title}} + {{range $js := $header.JSDeps}} + + {{end}} + {{if $header.CamliClosure}} + + {{end}} + {{range $css := $header.CSSFiles}} + + {{end}} + + + +

      {{$header.Title}}

      + {{if $file := call .File}} +
      File: {{$file.FileName}}, {{$file.Size}} bytes, type {{$file.MIMEType}}
      + {{if $file.IsImage}} + + {{end}} + + {{if $nav := call $file.Nav}} +
      + {{if $prev := $nav.PrevPath}}[prev] {{end}} + {{if $up := $nav.ParentPath}}[up] {{end}} + {{if $next := $nav.NextPath}}[next] {{end}} +
      + {{end}} + {{else}} + {{if $membersData := call .Members}} + + + + {{end}} + {{end}} + {{if $header.CamliClosure}} + {{if $header.ViewerIsOwner}} + + {{end}} + {{end}} +{{end}} + + diff --git a/website/content/docs/server-config b/website/content/docs/server-config index 18cc8b651..ae9ced053 100644 --- a/website/content/docs/server-config +++ b/website/content/docs/server-config @@ -65,18 +65,21 @@ to pkg/genconfig welcome.

      config, as used by devcam server.

      Publishing options

      -

      Although limited, publishing can be configured through the publish key. There is only support for an image gallery view (even though it will display thumbnails for other kinds of items), which is not really customizable. Here is an example of a value if one wanted to publish some items under /pics/:

      +

      Camlistore uses Go html templates to publish pages, and publishing can be configured through the publish key. There is already support for an image gallery view, which can be enabled similarly to the example below (obviously, the rootPermanode will be different).

      -{
-  "/pics/": {
-    "rootPermanode": "sha1-09888624be84fcb3ae67e8aa2f29682b4ff515d7",
-    "style": "pics.css",
-    "template": "gallery"
-  }
+"publish": {
+	"/pics/": {
+		"rootPermanode": "sha1-6cbe9e1c35e854eab028cba43d099d35ceae0de8",
+		"style": "pics.css",
+		"js": "pics.js",
+		"goTemplate": "gallery.html"
+	}
 }

      One can create any permanode with camput or the UI and use it as the rootPermanode.

      +

      Please see the publishing README if you want to make/contribute more publishing views.

      +

      Windows

      From c81f3147f666646effb29c990a331ac1c6a034f1 Mon Sep 17 00:00:00 2001 From: mpl Date: Mon, 18 Nov 2013 15:51:47 +0100 Subject: [PATCH 3/3] pkg/index: write relevant keys when receiving a delete claim This change: 1) Checks if the incoming claim is a delete claim with the use of GetBlobMeta. 2) write the keyDeleted and keyDeletes keys when it's a delete claim, plus the usual keys when the target is a permanode. Yet to be done in the next CLs: 1) update the index deletes cache upon reception of a delete claim 2) update most of the search functions so they use deletedAt properly 3) add new keys necessary for GetRecentPermanodes to give a fully correct result. I also made indextest.DumpIndex public because it turned to be useful to debug within pkg/search/ as well. http://camlistore.org/issue/191 Change-Id: I8d8b9d12a535b8b1de0018b4a0e359241f14d52a --- pkg/index/indextest/tests.go | 14 ++++---- pkg/index/keys.go | 2 +- pkg/index/receive.go | 63 ++++++++++++++++++++++++++++-------- 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/pkg/index/indextest/tests.go b/pkg/index/indextest/tests.go index 7883414a0..17dca1fee 100644 --- a/pkg/index/indextest/tests.go +++ b/pkg/index/indextest/tests.go @@ -74,7 +74,7 @@ func (id *IndexDeps) Set(key, value string) error { return id.Index.Storage().Set(key, value) } -func (id *IndexDeps) dumpIndex(t *testing.T) { +func (id *IndexDeps) DumpIndex(t *testing.T) { t.Logf("Begin index dump:") it := id.Index.Storage().Find("") for it.Next() { @@ -341,7 +341,7 @@ func Index(t *testing.T, initIdx func() *index.Index) { ) lastPermanodeMutation := id.lastTime() - id.dumpIndex(t) + id.DumpIndex(t) key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007" if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e { @@ -610,7 +610,7 @@ func PathsOfSignerTarget(t *testing.T, initIdx func() *index.Index) { claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124") t.Logf("made path claims %q and %q", claim1, claim2) - id.dumpIndex(t) + id.DumpIndex(t) type test struct { blobref string @@ -657,7 +657,7 @@ func Files(t *testing.T, initIdx func() *index.Index) { fileTime := time.Unix(1361250375, 0) fileRef, wholeRef := id.UploadFile("foo.html", "I am an html file.", fileTime) t.Logf("uploaded fileref %q, wholeRef %q", fileRef, wholeRef) - id.dumpIndex(t) + id.DumpIndex(t) // ExistingFileSchemas { @@ -714,7 +714,7 @@ func EdgesTo(t *testing.T, initIdx func() *index.Index) { t.Logf("edge %s --> %s", pn1, pn2) - id.dumpIndex(t) + id.DumpIndex(t) // Look for pn1 { @@ -740,7 +740,7 @@ func IsDeleted(t *testing.T, initIdx func() *index.Index) { idx := initIdx() id := NewIndexDeps(idx) id.Fataler = t - defer id.dumpIndex(t) + defer id.DumpIndex(t) pn1 := id.NewPermanode() // delete pn1 @@ -790,7 +790,7 @@ func DeletedAt(t *testing.T, initIdx func() *index.Index) { idx := initIdx() id := NewIndexDeps(idx) id.Fataler = t - defer id.dumpIndex(t) + defer id.DumpIndex(t) pn1 := id.NewPermanode() // Test the never, ever, deleted case diff --git a/pkg/index/keys.go b/pkg/index/keys.go index 5b3fafefb..3d51b4b8c 100644 --- a/pkg/index/keys.go +++ b/pkg/index/keys.go @@ -24,7 +24,7 @@ import ( // requiredSchemaVersion is incremented every time // an index key type is added, changed, or removed. -const requiredSchemaVersion = 2 +const requiredSchemaVersion = 3 // type of key returns the identifier in k before the first ":" or "|". // (Originally we packed keys by hand and there are a mix of styles) diff --git a/pkg/index/receive.go b/pkg/index/receive.go index 40101afbd..45fea0712 100644 --- a/pkg/index/receive.go +++ b/pkg/index/receive.go @@ -26,6 +26,7 @@ import ( _ "image/png" "io" "log" + "os" "sort" "strings" "sync" @@ -126,6 +127,8 @@ func (ix *Index) commit(mm mutationMap) error { // the blobref can be trusted at this point (it's been fully consumed // and verified to match), and the sniffer has been populated. func (ix *Index) populateMutationMap(br blob.Ref, sniffer *BlobSniffer) (mutationMap, error) { + // TODO(mpl): shouldn't we remove these two from the map (so they don't get committed) when + // e.g in populateClaim we detect a bogus claim (which does not yield an error)? mm := mutationMap{ "have:" + br.String(): fmt.Sprintf("%d", sniffer.Size()), "meta:" + br.String(): fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()), @@ -137,10 +140,6 @@ func (ix *Index) populateMutationMap(br blob.Ref, sniffer *BlobSniffer) (mutatio if err := ix.populateClaim(blob, mm); err != nil { return nil, err } - case "permanode": - //if err := mi.populatePermanode(blobRef, camli, mm); err != nil { - //return nil, err - //} case "file": if err := ix.populateFile(blob, mm); err != nil { return nil, err @@ -311,6 +310,41 @@ func (ix *Index) populateDir(b *schema.Blob, mm mutationMap) error { return nil } +// populateDeleteClaim adds to mm the entries resulting from the delete claim cl. +// It is assumed cl is a valid claim, and vr has already been verified. +func (ix *Index) populateDeleteClaim(cl schema.Claim, vr *jsonsign.VerifyRequest, mm mutationMap) { + br := cl.Blob().BlobRef() + target := cl.Target() + if !target.Valid() { + log.Print(fmt.Errorf("no valid target for delete claim %v", br)) + return + } + meta, err := ix.GetBlobMeta(target) + if err != nil { + if err == os.ErrNotExist { + // TODO: return a dependency error type, to schedule re-indexing in the future + } + log.Print(fmt.Errorf("Could not get mime type of target blob %v: %v", target, err)) + return + } + // TODO(mpl): create consts somewhere for "claim" and "permanode" as camliTypes, and use them, + // instead of hardcoding. Unless they already exist ? (didn't find them). + if meta.CamliType != "permanode" && meta.CamliType != "claim" { + log.Print(fmt.Errorf("delete claim target in %v is neither a permanode nor a claim: %v", br, meta.CamliType)) + return + } + mm.Set(keyDeleted.Key(target, cl.ClaimDateString(), br), "") + mm.Set(keyDeletes.Key(br, target), "") + if meta.CamliType == "claim" { + return + } + recentKey := keyRecentPermanode.Key(vr.SignerKeyId, cl.ClaimDateString(), br) + mm.Set(recentKey, target.String()) + attr, value := cl.Attribute(), cl.Value() + claimKey := keyPermanodeClaim.Key(target, vr.SignerKeyId, cl.ClaimDateString(), br) + mm.Set(claimKey, keyPermanodeClaim.Val(cl.ClaimType(), attr, value, vr.CamliSigner)) +} + func (ix *Index) populateClaim(b *schema.Blob, mm mutationMap) error { br := b.BlobRef() @@ -320,13 +354,6 @@ func (ix *Index) populateClaim(b *schema.Blob, mm mutationMap) error { return nil } - pnbr := claim.ModifiedPermanode() - if !pnbr.Valid() { - // A different type of claim; not modifying a permanode. - return nil - } - attr, value := claim.Attribute(), claim.Value() - vr := jsonsign.NewVerificationRequest(b.JSON(), ix.KeyFetcher) if !vr.Verify() { // TODO(bradfitz): ask if the vr.Err.(jsonsign.Error).IsPermanent() and retry @@ -337,12 +364,22 @@ func (ix *Index) populateClaim(b *schema.Blob, mm mutationMap) error { return errors.New("index: populateClaim verification failure") } verifiedKeyId := vr.SignerKeyId - mm.Set("signerkeyid:"+vr.CamliSigner.String(), verifiedKeyId) + if claim.ClaimType() == string(schema.DeleteClaim) { + ix.populateDeleteClaim(claim, vr, mm) + return nil + } + + pnbr := claim.ModifiedPermanode() + if !pnbr.Valid() { + // A different type of claim; not modifying a permanode. + return nil + } + + attr, value := claim.Attribute(), claim.Value() recentKey := keyRecentPermanode.Key(verifiedKeyId, claim.ClaimDateString(), br) mm.Set(recentKey, pnbr.String()) - claimKey := keyPermanodeClaim.Key(pnbr, verifiedKeyId, claim.ClaimDateString(), br) mm.Set(claimKey, keyPermanodeClaim.Val(claim.ClaimType(), attr, value, vr.CamliSigner))