From eb483eb9f61376ff27aa2f3f456824091a9065dd Mon Sep 17 00:00:00 2001
From: Brad Fitzpatrick <brad@danga.com>
Date: Sat, 14 Dec 2013 15:25:39 +0100
Subject: [PATCH] Limit number of image resizes happening at once.

Attempt to ease memory spikes.

Change-Id: I4df7177c063b284ac841335d8f18b1a6da5289d0
---
 TODO                |  2 --
 pkg/server/image.go | 21 +++++++++++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/TODO b/TODO
index fd160f051..109955a4a 100644
--- a/TODO
+++ b/TODO
@@ -14,8 +14,6 @@ Offline list:
    instead of using schema.WriteFileFromReader if the thumbnail
    is smaller than blobserver.MaxBlobSize (16MB).
 
--- gate in front of image resizes. limit a few at a time.
-
 -- blobhub.go NotifyBlobReceived should do the sync hooks before firing off
    goroutines to notify that things were good. they might not be good yet
    if the hook fails. also, no need for the slice. can just fire off goroutines
diff --git a/pkg/server/image.go b/pkg/server/image.go
index d3e7412b8..92f34f955 100644
--- a/pkg/server/image.go
+++ b/pkg/server/image.go
@@ -36,6 +36,7 @@ import (
 	"camlistore.org/pkg/magic"
 	"camlistore.org/pkg/schema"
 	"camlistore.org/pkg/search"
+	"camlistore.org/pkg/syncutil"
 
 	_ "camlistore.org/third_party/github.com/nf/cr2"
 )
@@ -152,6 +153,20 @@ func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format s
 	return format
 }
 
+// These gates control the max concurrency of slurping raw images
+// (e.g. JPEG bytes) to RAM, and then decoding and resizing them,
+// respectively.  We allow more concurrency for the former because
+// it's slower and less memory-intensive.  The actual resizing takes
+// much more CPU and RAM.
+
+// TODO: these numbers were just guesses and not based on any
+// data. measure? make these configurable? Automatically tuned
+// somehow? Based on memory usage/availability?
+var (
+	scaleImageGateSlurp  = syncutil.NewGate(5)
+	scaleImageGateResize = syncutil.NewGate(2)
+)
+
 func (ih *ImageHandler) scaleImage(buf *bytes.Buffer, file blob.Ref) (format string, err error) {
 	fr, err := schema.NewFileReader(ih.storageSeekFetcher(), file)
 	if err != nil {
@@ -159,10 +174,16 @@ func (ih *ImageHandler) scaleImage(buf *bytes.Buffer, file blob.Ref) (format str
 	}
 	defer fr.Close()
 
+	scaleImageGateSlurp.Start()
 	_, err = io.Copy(buf, fr)
+	scaleImageGateSlurp.Done()
 	if err != nil {
 		return format, fmt.Errorf("image resize: error reading image %s: %v", file, err)
 	}
+
+	scaleImageGateResize.Start()
+	defer scaleImageGateResize.Done()
+
 	i, imConfig, err := images.Decode(bytes.NewReader(buf.Bytes()),
 		&images.DecodeOpts{MaxWidth: ih.MaxWidth, MaxHeight: ih.MaxHeight})
 	if err != nil {