diff --git a/TODO b/TODO index 109955a4a..53635de5e 100644 --- a/TODO +++ b/TODO @@ -4,11 +4,14 @@ There are two TODO lists. This file (good for airplanes) and the online bug trac Offline list: --- unexport more stuff from pkg/server. Cache, PublishRoots, etc. +-- dev mode flag -thumbstress to stress thumbnail generation: drop all + thumbMeta keyvalue entries and also append a process random value + to thumbnails URLs to cache-bust browser. will force reload and + re-generation of all thumbnails --- make UI handler's scaledImage cache take a jsonconfig constructor - of a sorted.KeyValue-registered type. Always slap a memory LRU in front - of it. +-- put singleflight in front of thumbnail generation + +-- unexport more stuff from pkg/server. Cache, etc. -- In ImageHandler.cache, write the thumbnail out as one large blob instead of using schema.WriteFileFromReader if the thumbnail diff --git a/config/dev-server-config.json b/config/dev-server-config.json index ade1e1773..823d3bb30 100644 --- a/config/dev-server-config.json +++ b/config/dev-server-config.json @@ -36,7 +36,6 @@ "blobRoot": "/bs/", "searchRoot": "/my-search/", "cache": "/cache/", - "scaledImage": "lrucache", "css": ["pics.css"], "js": ["pics.js"], "goTemplate": "gallery.html", @@ -57,7 +56,10 @@ "sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""], "jsonSignRoot": "/sighelper/", "cache": "/cache/", - "scaledImage": "lrucache", + "scaledImage": { + "type": "kv", + "file": ["_env", "${CAMLI_ROOT_CACHE}/thumbnails.kv", ""] + }, "publishRoots": ["/blog/", "/pics/"] } }, diff --git a/pkg/server/image.go b/pkg/server/image.go index 92f34f955..909ec2579 100644 --- a/pkg/server/image.go +++ b/pkg/server/image.go @@ -48,7 +48,7 @@ type ImageHandler struct { Cache blobserver.Storage // optional MaxWidth, MaxHeight int Square bool - sc scaledImage // optional cache for scaled images + thumbMeta *thumbMeta // optional cache for scaled images } func (ih *ImageHandler) storageSeekFetcher() blob.SeekFetcher { @@ -97,7 +97,7 @@ func (ih *ImageHandler) cacheScaled(tr io.Reader, name string) error { if err != nil { return err } - ih.sc.Put(name, br) + ih.thumbMeta.Put(name, br) return nil } @@ -128,7 +128,7 @@ func cacheKey(bref string, width int, height int) string { // Almost all errors are not interesting. Real errors will be logged. func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string) { key := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight) - br, err := ih.sc.Get(key) + br, err := ih.thumbMeta.Get(key) if err == errCacheMiss { return } @@ -242,7 +242,7 @@ func (ih *ImageHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, fil var err error format := "" cacheHit := false - if ih.sc != nil { + if ih.thumbMeta != nil { format = ih.scaledCached(&buf, file) if format != "" { cacheHit = true @@ -255,7 +255,7 @@ func (ih *ImageHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, fil http.Error(rw, err.Error(), 500) return } - if ih.sc != nil { + if ih.thumbMeta != nil { name := cacheKey(file.String(), mw, mh) bufcopy := buf.Bytes() err = ih.cacheScaled(bytes.NewBuffer(bufcopy), name) diff --git a/pkg/server/publish.go b/pkg/server/publish.go index 2f66eabb0..f093f0806 100644 --- a/pkg/server/publish.go +++ b/pkg/server/publish.go @@ -56,7 +56,8 @@ type PublishHandler struct { Search *search.Handler Storage blobserver.Storage // of blobRoot Cache blobserver.Storage // or nil - sc scaledImage // cache of scaled images, optional + + thumbMeta *thumbMeta // optional cache of scaled images CSSFiles []string // goTemplate is the go html template used for publishing. @@ -92,7 +93,7 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han blobRoot := conf.RequiredString("blobRoot") searchRoot := conf.RequiredString("searchRoot") cachePrefix := conf.OptionalString("cache", "") - scType := conf.OptionalString("scaledImage", "") + scaledImageConf := conf.OptionalObject("scaledImage") bootstrapSignRoot := conf.OptionalString("devBootstrapPermanodeUsing", "") rootNode := conf.OptionalList("rootPermanode") ph.sourceRoot = conf.OptionalString("sourceRoot", "") @@ -151,19 +152,20 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han } } + scaledImageKV, err := newKVOrNil(scaledImageConf) + if err != nil { + return nil, fmt.Errorf("in publish handler's scaledImage: %v", err) + } + if scaledImageKV != nil && cachePrefix == "" { + return nil, fmt.Errorf("in publish handler, can't specify scaledImage without cache") + } if cachePrefix != "" { bs, err := ld.GetStorage(cachePrefix) if err != nil { return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err) } ph.Cache = bs - switch scType { - case "lrucache": - ph.sc = newScaledImageLRU() - case "": - default: - return nil, fmt.Errorf("unsupported publish handler's scType: %q ", scType) - } + ph.thumbMeta = newThumbMeta(scaledImageKV) } // TODO(mpl): check that it works on appengine too. @@ -938,7 +940,7 @@ func (pr *publishRequest) serveScaledImage(des *search.DescribedBlob, maxWidth, MaxWidth: maxWidth, MaxHeight: maxHeight, Square: square, - sc: pr.ph.sc, + thumbMeta: pr.ph.thumbMeta, } th.ServeHTTP(pr.rw, pr.req, fileref) } diff --git a/pkg/server/thumbcache.go b/pkg/server/thumbcache.go index 1c4eddd3b..4e6f401ad 100644 --- a/pkg/server/thumbcache.go +++ b/pkg/server/thumbcache.go @@ -18,46 +18,64 @@ package server import ( "errors" + "fmt" "camlistore.org/pkg/blob" "camlistore.org/pkg/lru" + "camlistore.org/pkg/sorted" ) -const cacheSize = 1024 +const memLRUSize = 1024 // arbitrary -// scaledImage is a mapping between the blobref of an image and -// its scaling parameters, and the blobref of such a rescaled -// version of that image. -// Key will be some string containing the original full-sized image's blobref, -// its target dimensions, and any possible transformations on it (e.g. cropping -// it to square). This string packing should not be parsed by a ScaledImage -// implementation and is not guaranteed to be stable over time. -type scaledImage interface { - Get(key string) (blob.Ref, error) // returns errCacheMiss when item not in cache - Put(key string, br blob.Ref) error -} +// thumbMeta is a mapping from an image's scaling parameters (encoding +// as an opaque "key" string) and the blobref of the thumbnail +// (currently it's file schema blob) +// +// The key will be some string containing the original full-sized image's +// blobref, its target dimensions, and any possible transformations on +// it (e.g. cropping it to square). var errCacheMiss = errors.New("not in cache") -type scaledImageLRU struct { - nameToBlob *lru.Cache // string (see key format) -> blob.Ref +type thumbMeta struct { + mem *lru.Cache // string (see key format) -> blob.Ref + kv sorted.KeyValue // optional } -func newScaledImageLRU() scaledImage { - return &scaledImageLRU{ - nameToBlob: lru.New(cacheSize), +// kv is optional +func newThumbMeta(kv sorted.KeyValue) *thumbMeta { + return &thumbMeta{ + mem: lru.New(memLRUSize), + kv: kv, } } -func (sc *scaledImageLRU) Get(key string) (blob.Ref, error) { - br, ok := sc.nameToBlob.Get(key) - if !ok { - return blob.Ref{}, errCacheMiss +func (m *thumbMeta) Get(key string) (br blob.Ref, err error) { + if v, ok := m.mem.Get(key); ok { + return v.(blob.Ref), nil } - return br.(blob.Ref), nil + if m.kv != nil { + v, err := m.kv.Get(key) + if err == sorted.ErrNotFound { + return br, errCacheMiss + } + if err != nil { + return br, err + } + br, ok := blob.Parse(v) + if !ok { + return br, fmt.Errorf("Invalid blobref %q found for key %q in thumbnail mea", v, key) + } + m.mem.Add(key, br) + return br, nil + } + return br, errCacheMiss } -func (sc *scaledImageLRU) Put(key string, br blob.Ref) error { - sc.nameToBlob.Add(key, br) +func (m *thumbMeta) Put(key string, br blob.Ref) error { + m.mem.Add(key, br) + if m.kv != nil { + return m.kv.Set(key, br.String()) + } return nil } diff --git a/pkg/server/ui.go b/pkg/server/ui.go index 5c0252f17..4a2cea8bd 100644 --- a/pkg/server/ui.go +++ b/pkg/server/ui.go @@ -37,6 +37,7 @@ import ( "camlistore.org/pkg/jsonsign/signhandler" "camlistore.org/pkg/misc/closure" "camlistore.org/pkg/search" + "camlistore.org/pkg/sorted" uistatic "camlistore.org/server/camlistored/ui" closurestatic "camlistore.org/server/camlistored/ui/closure" ) @@ -76,7 +77,7 @@ type UIHandler struct { // caching image thumbnails and other emphemeral data. Cache blobserver.Storage // or nil - sc scaledImage // optional thumbnail key->blob.Ref cache + thumbMeta *thumbMeta // optional thumbnail key->blob.Ref cache // sourceRoot optionally specifies the path to root of Camlistore's // source. If empty, the UI files must be compiled in to the @@ -94,6 +95,15 @@ func init() { blobserver.RegisterHandlerConstructor("ui", uiFromConfig) } +// newKVOrNil wraps sorted.NewKeyValue and adds the ability +// to pass a nil conf to get a (nil, nil) response. +func newKVOrNil(conf jsonconfig.Obj) (sorted.KeyValue, error) { + if len(conf) == 0 { + return nil, nil + } + return sorted.NewKeyValue(conf) +} + func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { ui := &UIHandler{ prefix: ld.MyPrefix(), @@ -102,7 +112,7 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er } pubRoots := conf.OptionalList("publishRoots") cachePrefix := conf.OptionalString("cache", "") - scType := conf.OptionalString("scaledImage", "") + scaledImageConf := conf.OptionalObject("scaledImage") if err = conf.Validate(); err != nil { return } @@ -145,18 +155,20 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er return } + scaledImageKV, err := newKVOrNil(scaledImageConf) + if err != nil { + return nil, fmt.Errorf("in UI handler's scaledImage: %v", err) + } + if scaledImageKV != nil && cachePrefix == "" { + return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache") + } if cachePrefix != "" { bs, err := ld.GetStorage(cachePrefix) if err != nil { return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err) } ui.Cache = bs - switch scType { - case "lrucache": - ui.sc = newScaledImageLRU() - default: - return nil, fmt.Errorf("unsupported ui handler's scType: %q ", scType) - } + ui.thumbMeta = newThumbMeta(scaledImageKV) } if ui.sourceRoot == "" { @@ -461,7 +473,7 @@ func (ui *UIHandler) serveThumbnail(rw http.ResponseWriter, req *http.Request) { Cache: ui.Cache, MaxWidth: width, MaxHeight: height, - sc: ui.sc, + thumbMeta: ui.thumbMeta, } th.ServeHTTP(rw, req, blobref) } diff --git a/pkg/serverconfig/genconfig.go b/pkg/serverconfig/genconfig.go index 1a51107c7..eeb57fe60 100644 --- a/pkg/serverconfig/genconfig.go +++ b/pkg/serverconfig/genconfig.go @@ -108,25 +108,32 @@ func addPublishedConfig(prefixes jsonconfig.Obj, return pubPrefixes, nil } -func addUIConfig(prefixes jsonconfig.Obj, +func addUIConfig(params *configPrefixesParams, + prefixes jsonconfig.Obj, uiPrefix string, published []interface{}, sourceRoot string) { - ob := map[string]interface{}{} - ob["handler"] = "ui" - handlerArgs := map[string]interface{}{ + + args := map[string]interface{}{ "jsonSignRoot": "/sighelper/", "cache": "/cache/", - "scaledImage": "lrucache", } if len(published) > 0 { - handlerArgs["publishRoots"] = published + args["publishRoots"] = published } if sourceRoot != "" { - handlerArgs["sourceRoot"] = sourceRoot + args["sourceRoot"] = sourceRoot + } + if params.blobPath != "" { + args["scaledImage"] = map[string]interface{}{ + "type": "kv", + "file": filepath.Join(params.blobPath, "thumbmeta.kv"), + } + } + prefixes[uiPrefix] = map[string]interface{}{ + "handler": "ui", + "handlerArgs": args, } - ob["handlerArgs"] = handlerArgs - prefixes[uiPrefix] = ob } func addMongoConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) { @@ -686,7 +693,7 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { } if runIndex { - addUIConfig(prefixes, "/ui/", published, sourceRoot) + addUIConfig(prefixesParams, prefixes, "/ui/", published, sourceRoot) } if mysql != "" { diff --git a/pkg/serverconfig/testdata/baseurl-want.json b/pkg/serverconfig/testdata/baseurl-want.json index da012da9b..fe58d7a18 100644 --- a/pkg/serverconfig/testdata/baseurl-want.json +++ b/pkg/serverconfig/testdata/baseurl-want.json @@ -97,7 +97,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/default-want.json b/pkg/serverconfig/testdata/default-want.json index b36211b87..72ffcc91c 100644 --- a/pkg/serverconfig/testdata/default-want.json +++ b/pkg/serverconfig/testdata/default-want.json @@ -96,7 +96,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/diskpacked-want.json b/pkg/serverconfig/testdata/diskpacked-want.json index 845437582..8f2b2b733 100644 --- a/pkg/serverconfig/testdata/diskpacked-want.json +++ b/pkg/serverconfig/testdata/diskpacked-want.json @@ -96,7 +96,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/google_nolocaldisk-want.json b/pkg/serverconfig/testdata/google_nolocaldisk-want.json index 568d2bd62..4075862ee 100644 --- a/pkg/serverconfig/testdata/google_nolocaldisk-want.json +++ b/pkg/serverconfig/testdata/google_nolocaldisk-want.json @@ -96,8 +96,7 @@ "handler": "ui", "handlerArgs": { "cache": "/cache/", - "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "jsonSignRoot": "/sighelper/" } } } diff --git a/pkg/serverconfig/testdata/listenbase-want.json b/pkg/serverconfig/testdata/listenbase-want.json index e3472b703..e16a6093e 100644 --- a/pkg/serverconfig/testdata/listenbase-want.json +++ b/pkg/serverconfig/testdata/listenbase-want.json @@ -96,7 +96,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/mem-want.json b/pkg/serverconfig/testdata/mem-want.json index 2b6c63898..ee07545a9 100644 --- a/pkg/serverconfig/testdata/mem-want.json +++ b/pkg/serverconfig/testdata/mem-want.json @@ -151,7 +151,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/memindex-want.json b/pkg/serverconfig/testdata/memindex-want.json index b36211b87..72ffcc91c 100644 --- a/pkg/serverconfig/testdata/memindex-want.json +++ b/pkg/serverconfig/testdata/memindex-want.json @@ -96,7 +96,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/mongo-want.json b/pkg/serverconfig/testdata/mongo-want.json index 367240664..52fa9d2e2 100644 --- a/pkg/serverconfig/testdata/mongo-want.json +++ b/pkg/serverconfig/testdata/mongo-want.json @@ -100,7 +100,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/s3_alt_host-want.json b/pkg/serverconfig/testdata/s3_alt_host-want.json index a2426fccf..ed8b92a8c 100644 --- a/pkg/serverconfig/testdata/s3_alt_host-want.json +++ b/pkg/serverconfig/testdata/s3_alt_host-want.json @@ -94,8 +94,7 @@ "handler": "ui", "handlerArgs": { "cache": "/cache/", - "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "jsonSignRoot": "/sighelper/" } } } diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk-want.json b/pkg/serverconfig/testdata/s3_nolocaldisk-want.json index 63d5794cd..e8dd17e2d 100644 --- a/pkg/serverconfig/testdata/s3_nolocaldisk-want.json +++ b/pkg/serverconfig/testdata/s3_nolocaldisk-want.json @@ -93,8 +93,7 @@ "handler": "ui", "handlerArgs": { "cache": "/cache/", - "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "jsonSignRoot": "/sighelper/" } } } diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json index c9b784832..8936a9468 100644 --- a/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json +++ b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json @@ -97,8 +97,7 @@ "handler": "ui", "handlerArgs": { "cache": "/cache/", - "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "jsonSignRoot": "/sighelper/" } } } diff --git a/pkg/serverconfig/testdata/sqlite-want.json b/pkg/serverconfig/testdata/sqlite-want.json index 3106bffa3..c294e64c5 100644 --- a/pkg/serverconfig/testdata/sqlite-want.json +++ b/pkg/serverconfig/testdata/sqlite-want.json @@ -95,7 +95,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/tls-want.json b/pkg/serverconfig/testdata/tls-want.json index a43663bee..25ee3bf9c 100644 --- a/pkg/serverconfig/testdata/tls-want.json +++ b/pkg/serverconfig/testdata/tls-want.json @@ -97,7 +97,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/with_blog-want.json b/pkg/serverconfig/testdata/with_blog-want.json index b777e94a2..061c08ae7 100644 --- a/pkg/serverconfig/testdata/with_blog-want.json +++ b/pkg/serverconfig/testdata/with_blog-want.json @@ -116,7 +116,10 @@ "publishRoots": [ "/blog/" ], - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/with_gallery-want.json b/pkg/serverconfig/testdata/with_gallery-want.json index bd14c6862..c4d0cff04 100644 --- a/pkg/serverconfig/testdata/with_gallery-want.json +++ b/pkg/serverconfig/testdata/with_gallery-want.json @@ -119,7 +119,10 @@ "publishRoots": [ "/pics/" ], - "scaledImage": "lrucache" + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } } } } diff --git a/pkg/serverconfig/testdata/with_sourceroot-want.json b/pkg/serverconfig/testdata/with_sourceroot-want.json index bfedaad5f..615bcd72c 100644 --- a/pkg/serverconfig/testdata/with_sourceroot-want.json +++ b/pkg/serverconfig/testdata/with_sourceroot-want.json @@ -114,7 +114,10 @@ "handlerArgs": { "cache": "/cache/", "jsonSignRoot": "/sighelper/", - "scaledImage": "lrucache", + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + }, "sourceRoot": "/path/to/alternative/camli/source" } }