Thumbnail cleanups and let the meta map persist on disk with a sorted.KeyValue

Previously, every time you restarted the server, it forgot about all
thumbnails previously generated.  (in practice it didn't/doesn't
matter with a single user, though, since they're still cached in the
browser and we always reply to If-Modified-Since immediately without
checking the cache)  But it'll matter more with the Publish handler.

Also, rename some stuff, clean up some stuff, drop an unused interface.

And then necessarily change the serverconfig low-level generator to use
a kvfile for the thumbmeta map when using local disk for blobs.

--

Change-Id: I4dcfcb21429a440aa118794c03f7abf7bd69c33b

TODO

@@ -4,11 +4,14 @@ There are two TODO lists. This file (good for airplanes) and the online bug trac
 
 Offline list:
 
--- unexport more stuff from pkg/server. Cache, PublishRoots, etc.
+-- dev mode flag -thumbstress to stress thumbnail generation: drop all
+   thumbMeta keyvalue entries and also append a process random value
+   to thumbnails URLs to cache-bust browser. will force reload and
+   re-generation of all thumbnails
 
--- make UI handler's scaledImage cache take a jsonconfig constructor
-   of a sorted.KeyValue-registered type. Always slap a memory LRU in front
-   of it.
+-- put singleflight in front of thumbnail generation
+
+-- unexport more stuff from pkg/server. Cache, etc.
 
 -- In ImageHandler.cache, write the thumbnail out as one large blob
    instead of using schema.WriteFileFromReader if the thumbnail

config/dev-server-config.json

@@ -36,7 +36,6 @@
               "blobRoot": "/bs/",
               "searchRoot": "/my-search/",
               "cache": "/cache/",
-              "scaledImage": "lrucache",
               "css": ["pics.css"],
               "js": ["pics.js"],
               "goTemplate": "gallery.html",
@@ -57,7 +56,10 @@
              "sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""],
              "jsonSignRoot": "/sighelper/",
              "cache": "/cache/",
-             "scaledImage": "lrucache",
+             "scaledImage": {
+                "type": "kv",
+                "file": ["_env", "${CAMLI_ROOT_CACHE}/thumbnails.kv", ""]
+             },
              "publishRoots": ["/blog/", "/pics/"]
          }
      },

pkg/server/image.go

@@ -48,7 +48,7 @@ type ImageHandler struct {
 	Cache               blobserver.Storage // optional
 	MaxWidth, MaxHeight int
 	Square              bool
-	sc                  scaledImage // optional cache for scaled images
+	thumbMeta           *thumbMeta // optional cache for scaled images
 }
 
 func (ih *ImageHandler) storageSeekFetcher() blob.SeekFetcher {
@@ -97,7 +97,7 @@ func (ih *ImageHandler) cacheScaled(tr io.Reader, name string) error {
 	if err != nil {
 		return err
 	}
-	ih.sc.Put(name, br)
+	ih.thumbMeta.Put(name, br)
 	return nil
 }
 
@@ -128,7 +128,7 @@ func cacheKey(bref string, width int, height int) string {
 // Almost all errors are not interesting. Real errors will be logged.
 func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string) {
 	key := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight)
-	br, err := ih.sc.Get(key)
+	br, err := ih.thumbMeta.Get(key)
 	if err == errCacheMiss {
 		return
 	}
@@ -242,7 +242,7 @@ func (ih *ImageHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, fil
 	var err error
 	format := ""
 	cacheHit := false
-	if ih.sc != nil {
+	if ih.thumbMeta != nil {
 		format = ih.scaledCached(&buf, file)
 		if format != "" {
 			cacheHit = true
@@ -255,7 +255,7 @@ func (ih *ImageHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, fil
 			http.Error(rw, err.Error(), 500)
 			return
 		}
-		if ih.sc != nil {
+		if ih.thumbMeta != nil {
 			name := cacheKey(file.String(), mw, mh)
 			bufcopy := buf.Bytes()
 			err = ih.cacheScaled(bytes.NewBuffer(bufcopy), name)

pkg/server/publish.go

@@ -56,7 +56,8 @@ type PublishHandler struct {
 	Search   *search.Handler
 	Storage  blobserver.Storage // of blobRoot
 	Cache    blobserver.Storage // or nil
-	sc       scaledImage        // cache of scaled images, optional
+
+	thumbMeta *thumbMeta // optional cache of scaled images
 
 	CSSFiles []string
 	// goTemplate is the go html template used for publishing.
@@ -92,7 +93,7 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han
 	blobRoot := conf.RequiredString("blobRoot")
 	searchRoot := conf.RequiredString("searchRoot")
 	cachePrefix := conf.OptionalString("cache", "")
-	scType := conf.OptionalString("scaledImage", "")
+	scaledImageConf := conf.OptionalObject("scaledImage")
 	bootstrapSignRoot := conf.OptionalString("devBootstrapPermanodeUsing", "")
 	rootNode := conf.OptionalList("rootPermanode")
 	ph.sourceRoot = conf.OptionalString("sourceRoot", "")
@@ -151,19 +152,20 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han
 		}
 	}
 
+	scaledImageKV, err := newKVOrNil(scaledImageConf)
+	if err != nil {
+		return nil, fmt.Errorf("in publish handler's scaledImage: %v", err)
+	}
+	if scaledImageKV != nil && cachePrefix == "" {
+		return nil, fmt.Errorf("in publish handler, can't specify scaledImage without cache")
+	}
 	if cachePrefix != "" {
 		bs, err := ld.GetStorage(cachePrefix)
 		if err != nil {
 			return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err)
 		}
 		ph.Cache = bs
-		switch scType {
-		case "lrucache":
-			ph.sc = newScaledImageLRU()
-		case "":
-		default:
-			return nil, fmt.Errorf("unsupported publish handler's scType: %q ", scType)
-		}
+		ph.thumbMeta = newThumbMeta(scaledImageKV)
 	}
 
 	// TODO(mpl): check that it works on appengine too.
@@ -938,7 +940,7 @@ func (pr *publishRequest) serveScaledImage(des *search.DescribedBlob, maxWidth,
 		MaxWidth:  maxWidth,
 		MaxHeight: maxHeight,
 		Square:    square,
-		sc:        pr.ph.sc,
+		thumbMeta: pr.ph.thumbMeta,
 	}
 	th.ServeHTTP(pr.rw, pr.req, fileref)
 }

pkg/server/thumbcache.go

@@ -18,46 +18,64 @@ package server
 
 import (
 	"errors"
+	"fmt"
 
 	"camlistore.org/pkg/blob"
 	"camlistore.org/pkg/lru"
+	"camlistore.org/pkg/sorted"
 )
 
-const cacheSize = 1024
+const memLRUSize = 1024 // arbitrary
 
-// scaledImage is a mapping between the blobref of an image and
-// its scaling parameters, and the blobref of such a rescaled
-// version of that image.
-// Key will be some string containing the original full-sized image's blobref,
-// its target dimensions, and any possible transformations on it (e.g. cropping
-// it to square). This string packing should not be parsed by a ScaledImage
-// implementation and is not guaranteed to be stable over time.
-type scaledImage interface {
-	Get(key string) (blob.Ref, error) // returns errCacheMiss when item not in cache
-	Put(key string, br blob.Ref) error
-}
+// thumbMeta is a mapping from an image's scaling parameters (encoding
+// as an opaque "key" string) and the blobref of the thumbnail
+// (currently it's file schema blob)
+//
+// The key will be some string containing the original full-sized image's
+// blobref, its target dimensions, and any possible transformations on
+// it (e.g. cropping it to square).
 
 var errCacheMiss = errors.New("not in cache")
 
-type scaledImageLRU struct {
-	nameToBlob *lru.Cache // string (see key format) -> blob.Ref
+type thumbMeta struct {
+	mem *lru.Cache      // string (see key format) -> blob.Ref
+	kv  sorted.KeyValue // optional
 }
 
-func newScaledImageLRU() scaledImage {
-	return &scaledImageLRU{
-		nameToBlob: lru.New(cacheSize),
+// kv is optional
+func newThumbMeta(kv sorted.KeyValue) *thumbMeta {
+	return &thumbMeta{
+		mem: lru.New(memLRUSize),
+		kv:  kv,
 	}
 }
 
-func (sc *scaledImageLRU) Get(key string) (blob.Ref, error) {
-	br, ok := sc.nameToBlob.Get(key)
-	if !ok {
-		return blob.Ref{}, errCacheMiss
+func (m *thumbMeta) Get(key string) (br blob.Ref, err error) {
+	if v, ok := m.mem.Get(key); ok {
+		return v.(blob.Ref), nil
 	}
-	return br.(blob.Ref), nil
+	if m.kv != nil {
+		v, err := m.kv.Get(key)
+		if err == sorted.ErrNotFound {
+			return br, errCacheMiss
+		}
+		if err != nil {
+			return br, err
+		}
+		br, ok := blob.Parse(v)
+		if !ok {
+			return br, fmt.Errorf("Invalid blobref %q found for key %q in thumbnail mea", v, key)
+		}
+		m.mem.Add(key, br)
+		return br, nil
+	}
+	return br, errCacheMiss
 }
 
-func (sc *scaledImageLRU) Put(key string, br blob.Ref) error {
-	sc.nameToBlob.Add(key, br)
+func (m *thumbMeta) Put(key string, br blob.Ref) error {
+	m.mem.Add(key, br)
+	if m.kv != nil {
+		return m.kv.Set(key, br.String())
+	}
 	return nil
 }

pkg/server/ui.go

@@ -37,6 +37,7 @@ import (
 	"camlistore.org/pkg/jsonsign/signhandler"
 	"camlistore.org/pkg/misc/closure"
 	"camlistore.org/pkg/search"
+	"camlistore.org/pkg/sorted"
 	uistatic "camlistore.org/server/camlistored/ui"
 	closurestatic "camlistore.org/server/camlistored/ui/closure"
 )
@@ -76,7 +77,7 @@ type UIHandler struct {
 	// caching image thumbnails and other emphemeral data.
 	Cache blobserver.Storage // or nil
 
-	sc scaledImage // optional thumbnail key->blob.Ref cache
+	thumbMeta *thumbMeta // optional thumbnail key->blob.Ref cache
 
 	// sourceRoot optionally specifies the path to root of Camlistore's
 	// source. If empty, the UI files must be compiled in to the
@@ -94,6 +95,15 @@ func init() {
 	blobserver.RegisterHandlerConstructor("ui", uiFromConfig)
 }
 
+// newKVOrNil wraps sorted.NewKeyValue and adds the ability
+// to pass a nil conf to get a (nil, nil) response.
+func newKVOrNil(conf jsonconfig.Obj) (sorted.KeyValue, error) {
+	if len(conf) == 0 {
+		return nil, nil
+	}
+	return sorted.NewKeyValue(conf)
+}
+
 func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
 	ui := &UIHandler{
 		prefix:       ld.MyPrefix(),
@@ -102,7 +112,7 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er
 	}
 	pubRoots := conf.OptionalList("publishRoots")
 	cachePrefix := conf.OptionalString("cache", "")
-	scType := conf.OptionalString("scaledImage", "")
+	scaledImageConf := conf.OptionalObject("scaledImage")
 	if err = conf.Validate(); err != nil {
 		return
 	}
@@ -145,18 +155,20 @@ func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, er
 		return
 	}
 
+	scaledImageKV, err := newKVOrNil(scaledImageConf)
+	if err != nil {
+		return nil, fmt.Errorf("in UI handler's scaledImage: %v", err)
+	}
+	if scaledImageKV != nil && cachePrefix == "" {
+		return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache")
+	}
 	if cachePrefix != "" {
 		bs, err := ld.GetStorage(cachePrefix)
 		if err != nil {
 			return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
 		}
 		ui.Cache = bs
-		switch scType {
-		case "lrucache":
-			ui.sc = newScaledImageLRU()
-		default:
-			return nil, fmt.Errorf("unsupported ui handler's scType: %q ", scType)
-		}
+		ui.thumbMeta = newThumbMeta(scaledImageKV)
 	}
 
 	if ui.sourceRoot == "" {
@@ -461,7 +473,7 @@ func (ui *UIHandler) serveThumbnail(rw http.ResponseWriter, req *http.Request) {
 		Cache:     ui.Cache,
 		MaxWidth:  width,
 		MaxHeight: height,
-		sc:        ui.sc,
+		thumbMeta: ui.thumbMeta,
 	}
 	th.ServeHTTP(rw, req, blobref)
 }

pkg/serverconfig/genconfig.go

@@ -108,25 +108,32 @@ func addPublishedConfig(prefixes jsonconfig.Obj,
 	return pubPrefixes, nil
 }
 
-func addUIConfig(prefixes jsonconfig.Obj,
+func addUIConfig(params *configPrefixesParams,
+	prefixes jsonconfig.Obj,
 	uiPrefix string,
 	published []interface{},
 	sourceRoot string) {
-	ob := map[string]interface{}{}
-	ob["handler"] = "ui"
-	handlerArgs := map[string]interface{}{
+
+	args := map[string]interface{}{
 		"jsonSignRoot": "/sighelper/",
 		"cache":        "/cache/",
-		"scaledImage":  "lrucache",
 	}
 	if len(published) > 0 {
-		handlerArgs["publishRoots"] = published
+		args["publishRoots"] = published
 	}
 	if sourceRoot != "" {
-		handlerArgs["sourceRoot"] = sourceRoot
+		args["sourceRoot"] = sourceRoot
+	}
+	if params.blobPath != "" {
+		args["scaledImage"] = map[string]interface{}{
+			"type": "kv",
+			"file": filepath.Join(params.blobPath, "thumbmeta.kv"),
+		}
+	}
+	prefixes[uiPrefix] = map[string]interface{}{
+		"handler":     "ui",
+		"handlerArgs": args,
 	}
-	ob["handlerArgs"] = handlerArgs
-	prefixes[uiPrefix] = ob
 }
 
 func addMongoConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) {
@@ -686,7 +693,7 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 	}
 
 	if runIndex {
-		addUIConfig(prefixes, "/ui/", published, sourceRoot)
+		addUIConfig(prefixesParams, prefixes, "/ui/", published, sourceRoot)
 	}
 
 	if mysql != "" {

pkg/serverconfig/testdata/baseurl-want.json

@@ -97,7 +97,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/default-want.json

@@ -96,7 +96,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/diskpacked-want.json

@@ -96,7 +96,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/google_nolocaldisk-want.json

@@ -96,8 +96,7 @@
 			"handler": "ui",
 			"handlerArgs": {
 				"cache": "/cache/",
-				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"jsonSignRoot": "/sighelper/"
 			}
 		}
 	}

pkg/serverconfig/testdata/listenbase-want.json

@@ -96,7 +96,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/mem-want.json

@@ -151,7 +151,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/memindex-want.json

@@ -96,7 +96,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/mongo-want.json

@@ -100,7 +100,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/s3_alt_host-want.json

@@ -94,8 +94,7 @@
 			"handler": "ui",
 			"handlerArgs": {
 				"cache": "/cache/",
-				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"jsonSignRoot": "/sighelper/"
 			}
 		}
 	}

pkg/serverconfig/testdata/s3_nolocaldisk-want.json

@@ -93,8 +93,7 @@
 			"handler": "ui",
 			"handlerArgs": {
 				"cache": "/cache/",
-				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"jsonSignRoot": "/sighelper/"
 			}
 		}
 	}

pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json

@@ -97,8 +97,7 @@
 			"handler": "ui",
 			"handlerArgs": {
 				"cache": "/cache/",
-				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"jsonSignRoot": "/sighelper/"
 			}
 		}
 	}

pkg/serverconfig/testdata/sqlite-want.json

@@ -95,7 +95,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/tls-want.json

@@ -97,7 +97,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/with_blog-want.json

@@ -116,7 +116,10 @@
 				"publishRoots": [
 					"/blog/"
 				],
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/with_gallery-want.json

@@ -119,7 +119,10 @@
 				"publishRoots": [
 					"/pics/"
 				],
-				"scaledImage": "lrucache"
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				}
 			}
 		}
 	}

pkg/serverconfig/testdata/with_sourceroot-want.json

@@ -114,7 +114,10 @@
 			"handlerArgs": {
 				"cache": "/cache/",
 				"jsonSignRoot": "/sighelper/",
-				"scaledImage": "lrucache",
+				"scaledImage": {
+					"file": "/tmp/blobs/thumbmeta.kv",
+					"type": "kv"
+				},
 				"sourceRoot": "/path/to/alternative/camli/source"
 			}
 		}