Merge "images: performance and memory improvements."

pkg/images/images.go

@@ -221,19 +221,13 @@ func rescale(im image.Image, opts *DecodeOpts) image.Image {
 		mh = int(mhf * float32(b.Dy()))
 	}
 
-	const huge = 2400
 	// If it's gigantic, it's more efficient to downsample first
 	// and then resize; resizing will smooth out the roughness.
 	// (trusting the moustachio guys on that one).
-	if b.Dx() > huge || b.Dy() > huge {
-		w, h := mw*2, mh*2
-		if b.Dx() > b.Dy() {
-			w = b.Dx() * h / b.Dy()
-		} else {
-			h = b.Dy() * w / b.Dx()
-		}
-		im = resize.Resample(im, b, w, h)
-		b = im.Bounds()
+	if b.Dx() > mw*2 || b.Dy() > mh*2 {
+		w, h := ScaledDimensions(b.Dx(), b.Dy(), mw*2, mh*2)
+		im = resize.ResampleInplace(im, b, w, h)
+		return resize.HalveInplace(im)
 	}
 	mw, mh = ScaledDimensions(b.Dx(), b.Dy(), mw, mh)
 	return resize.Resize(im, b, mw, mh)

pkg/misc/resize/bench_test.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2013 The Camlistore Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resize
+
+import (
+	"image"
+	"testing"
+)
+
+func resize(m image.Image) {
+	s := m.Bounds().Size().Div(2)
+	Resize(m, m.Bounds(), s.X, s.Y)
+}
+
+func halve(m image.Image) {
+	HalveInplace(m)
+}
+
+func BenchmarkResizeRGBA(b *testing.B) {
+	m := image.NewRGBA(orig)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		resize(m)
+	}
+}
+
+func BenchmarkHalveRGBA(b *testing.B) {
+	m := image.NewRGBA(orig)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		halve(m)
+	}
+}
+
+func BenchmarkResizeYCrCb(b *testing.B) {
+	m := image.NewYCbCr(orig, image.YCbCrSubsampleRatio422)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		resize(m)
+	}
+}
+
+func BenchmarkHalveYCrCb(b *testing.B) {
+	m := image.NewYCbCr(orig, image.YCbCrSubsampleRatio422)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		halve(m)
+	}
+}

pkg/misc/resize/resize.go

@@ -8,6 +8,7 @@ package resize
 import (
 	"image"
 	"image/color"
+	"image/draw"
 )
 
 // Resize returns a scaled copy of the image slice r of m.
@@ -219,6 +220,125 @@ func resizeRGBA(m *image.RGBA, r image.Rectangle, w, h int) image.Image {
 	return average(sum, w, h, n)
 }
 
+// HalveInplace downsamples the image by 50% using averaging interpolation.
+func HalveInplace(m image.Image) image.Image {
+	b := m.Bounds()
+	switch m := m.(type) {
+	case *image.YCbCr:
+		for y := b.Min.Y; y < b.Max.Y/2; y++ {
+			for x := b.Min.X; x < b.Max.X/2; x++ {
+				y00 := uint32(m.Y[m.YOffset(2*x, 2*y)])
+				y10 := uint32(m.Y[m.YOffset(2*x+1, 2*y)])
+				y01 := uint32(m.Y[m.YOffset(2*x, 2*y+1)])
+				y11 := uint32(m.Y[m.YOffset(2*x+1, 2*y+1)])
+				// Add before divide with uint32 or we get errors in the least
+				// significant bits.
+				m.Y[m.YOffset(x, y)] = uint8((y00 + y10 + y01 + y11) >> 2)
+
+				cb00 := uint32(m.Cb[m.COffset(2*x, 2*y)])
+				cb10 := uint32(m.Cb[m.COffset(2*x+1, 2*y)])
+				cb01 := uint32(m.Cb[m.COffset(2*x, 2*y+1)])
+				cb11 := uint32(m.Cb[m.COffset(2*x+1, 2*y+1)])
+				m.Cb[m.COffset(x, y)] = uint8((cb00 + cb10 + cb01 + cb11) >> 2)
+
+				cr00 := uint32(m.Cr[m.COffset(2*x, 2*y)])
+				cr10 := uint32(m.Cr[m.COffset(2*x+1, 2*y)])
+				cr01 := uint32(m.Cr[m.COffset(2*x, 2*y+1)])
+				cr11 := uint32(m.Cr[m.COffset(2*x+1, 2*y+1)])
+				m.Cr[m.COffset(x, y)] = uint8((cr00 + cr10 + cr01 + cr11) >> 2)
+			}
+		}
+		b.Max = b.Min.Add(b.Size().Div(2))
+		return subImage(m, b)
+	case draw.Image:
+		for y := b.Min.Y; y < b.Max.Y/2; y++ {
+			for x := b.Min.X; x < b.Max.X/2; x++ {
+				r00, g00, b00, a00 := m.At(2*x, 2*y).RGBA()
+				r10, g10, b10, a10 := m.At(2*x+1, 2*y).RGBA()
+				r01, g01, b01, a01 := m.At(2*x, 2*y+1).RGBA()
+				r11, g11, b11, a11 := m.At(2*x+1, 2*y+1).RGBA()
+
+				// Add before divide with uint32 or we get errors in the least
+				// significant bits.
+				r := (r00 + r10 + r01 + r11) >> 2
+				g := (g00 + g10 + g01 + g11) >> 2
+				b := (b00 + b10 + b01 + b11) >> 2
+				a := (a00 + a10 + a01 + a11) >> 2
+
+				m.Set(x, y, color.RGBA{
+					R: uint8(r >> 8),
+					G: uint8(g >> 8),
+					B: uint8(b >> 8),
+					A: uint8(a >> 8),
+				})
+			}
+		}
+		b.Max = b.Min.Add(b.Size().Div(2))
+		return subImage(m, b)
+	default:
+		// TODO(wathiede): fallback to generic Resample somehow?
+		panic("Unhandled image type")
+	}
+}
+
+// ResampleInplace will resample m inplace, overwritting existing pixel data,
+// and return a subimage of m sized to w and h.
+func ResampleInplace(m image.Image, r image.Rectangle, w, h int) image.Image {
+	// We don't support scaling up.
+	if r.Dx() < w || r.Dy() < h {
+		return m
+	}
+
+	switch m := m.(type) {
+	case *image.YCbCr:
+		xStep := float64(r.Dx()) / float64(w)
+		yStep := float64(r.Dy()) / float64(h)
+		for y := r.Min.Y; y < r.Min.Y+h; y++ {
+			for x := r.Min.X; x < r.Min.X+w; x++ {
+				xSrc := int(float64(x) * xStep)
+				ySrc := int(float64(y) * yStep)
+				cSrc := m.COffset(xSrc, ySrc)
+				cDst := m.COffset(x, y)
+				m.Y[m.YOffset(x, y)] = m.Y[m.YOffset(xSrc, ySrc)]
+				m.Cb[cDst] = m.Cb[cSrc]
+				m.Cr[cDst] = m.Cr[cSrc]
+			}
+		}
+	case draw.Image:
+		xStep := float64(r.Dx()) / float64(w)
+		yStep := float64(r.Dy()) / float64(h)
+		for y := r.Min.Y; y < r.Min.Y+h; y++ {
+			for x := r.Min.X; x < r.Min.X+w; x++ {
+				xSrc := int(float64(x) * xStep)
+				ySrc := int(float64(y) * yStep)
+				r, g, b, a := m.At(xSrc, ySrc).RGBA()
+				m.Set(x, y, color.RGBA{
+					R: uint8(r >> 8),
+					G: uint8(g >> 8),
+					B: uint8(b >> 8),
+					A: uint8(a >> 8),
+				})
+			}
+		}
+	default:
+		// TODO fallback to generic Resample somehow?
+		panic("Unhandled image type")
+	}
+	r.Max.X = r.Min.X + w
+	r.Max.Y = r.Min.Y + h
+	return subImage(m, r)
+}
+
+func subImage(m image.Image, r image.Rectangle) image.Image {
+	type subImager interface {
+		SubImage(image.Rectangle) image.Image
+	}
+	if si, ok := m.(subImager); ok {
+		return si.SubImage(r)
+	}
+	panic("Image type doesn't support SubImage")
+}
+
 // Resample returns a resampled copy of the image slice r of m.
 // The returned image has width w and height h.
 func Resample(m image.Image, r image.Rectangle, w, h int) image.Image {
@@ -228,19 +348,22 @@ func Resample(m image.Image, r image.Rectangle, w, h int) image.Image {
 	if w == 0 || h == 0 || r.Dx() <= 0 || r.Dy() <= 0 {
 		return image.NewRGBA64(image.Rect(0, 0, w, h))
 	}
-	curw, curh := r.Dx(), r.Dy()
 	img := image.NewRGBA(image.Rect(0, 0, w, h))
+	xStep := float64(r.Dx()) / float64(w)
+	yStep := float64(r.Dy()) / float64(h)
 	for y := 0; y < h; y++ {
 		for x := 0; x < w; x++ {
-			// Get a source pixel.
-			subx := x * curw / w
-			suby := y * curh / h
-			r32, g32, b32, a32 := m.At(subx, suby).RGBA()
-			r := uint8(r32 >> 8)
-			g := uint8(g32 >> 8)
-			b := uint8(b32 >> 8)
-			a := uint8(a32 >> 8)
-			img.SetRGBA(x, y, color.RGBA{r, g, b, a})
+			xSrc := int(float64(r.Min.X) + float64(x)*xStep)
+			ySrc := int(float64(r.Min.Y) + float64(y)*yStep)
+			//xSrc = r.Min.X + x*r.Dx()/w
+			//ySrc = r.Min.Y + y*r.Dy()/h
+			r, g, b, a := m.At(xSrc, ySrc).RGBA()
+			img.SetRGBA(x, y, color.RGBA{
+				R: uint8(r >> 8),
+				G: uint8(g >> 8),
+				B: uint8(b >> 8),
+				A: uint8(a >> 8),
+			})
 		}
 	}
 	return img

pkg/misc/resize/resize_test.go

@@ -0,0 +1,337 @@
+/*
+Copyright 2013 The Camlistore Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resize
+
+import (
+	"flag"
+	"fmt"
+	"image"
+	"image/color"
+	"image/color/palette"
+	"image/draw"
+	"image/png"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const (
+	// psnrThreshold is the threshold over which images must match to consider
+	// HalveInplace equivalent to Resize. It is in terms of dB and 60-80 is
+	// good for RGB.
+	psnrThreshold = 50.0
+
+	maxPixelDiffPercentage = 10
+)
+
+var (
+	output = flag.String("output", "", "If non-empty, the directory to save comparison images.")
+
+	orig  = image.Rect(0, 0, 1024, 1024)
+	thumb = image.Rect(0, 0, 64, 64)
+)
+
+func makeImages(r image.Rectangle) []image.Image {
+	return []image.Image{
+		image.NewGray(r),
+		image.NewGray16(r),
+		image.NewNRGBA(r),
+		image.NewNRGBA64(r),
+		image.NewPaletted(r, palette.Plan9),
+		image.NewRGBA(r),
+		image.NewRGBA64(r),
+		image.NewYCbCr(r, image.YCbCrSubsampleRatio444),
+		image.NewYCbCr(r, image.YCbCrSubsampleRatio422),
+		image.NewYCbCr(r, image.YCbCrSubsampleRatio420),
+		image.NewYCbCr(r, image.YCbCrSubsampleRatio440),
+	}
+}
+
+func TestResize(t *testing.T) {
+	for i, im := range makeImages(orig) {
+		m := Resize(im, orig, thumb.Dx(), thumb.Dy())
+		got, want := m.Bounds(), thumb
+		if !got.Eq(want) {
+			t.Error(i, "Want bounds", want, "got", got)
+		}
+	}
+}
+
+func TestResampleInplace(t *testing.T) {
+	for i, im := range makeImages(orig) {
+		m := ResampleInplace(im, orig, thumb.Dx(), thumb.Dy())
+		got, want := m.Bounds(), thumb
+		if !got.Eq(want) {
+			t.Error(i, "Want bounds", want, "got", got)
+		}
+	}
+}
+
+func TestResample(t *testing.T) {
+	for i, im := range makeImages(orig) {
+		m := Resample(im, orig, thumb.Dx(), thumb.Dy())
+		got, want := m.Bounds(), thumb
+		if !got.Eq(want) {
+			t.Error(i, "Want bounds", want, "got", got)
+		}
+	}
+
+	for _, d := range []struct {
+		wantFn string
+		r      image.Rectangle
+		w, h   int
+	}{
+		{
+			// Generated with imagemagick:
+			// $ convert -crop 128x128+320+160 -resize 64x64 -filter point \
+			//      testdata/test.png testdata/test-resample-128x128-64x64.png
+			wantFn: "test-resample-128x128-64x64.png",
+			r:      image.Rect(320, 160, 320+128, 160+128),
+			w:      64,
+			h:      64,
+		},
+		{
+			// Generated with imagemagick:
+			// $ convert -resize 128x128 -filter point testdata/test.png \
+			//      testdata/test-resample-768x576-128x96.png
+			wantFn: "test-resample-768x576-128x96.png",
+			r:      image.Rect(0, 0, 768, 576),
+			w:      128,
+			h:      96,
+		},
+	} {
+		m := image.NewRGBA(testIm.Bounds())
+		fillTestImage(m)
+		r, err := os.Open(filepath.Join("testdata", d.wantFn))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer r.Close()
+		want, err := png.Decode(r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got := Resample(m, d.r, d.w, d.h)
+		res := compareImages(got, want)
+		t.Logf("PSNR %.4f", res.psnr)
+		s := got.Bounds().Size()
+		tot := s.X * s.Y
+		per := float32(100*res.diffCnt) / float32(tot)
+		t.Logf("Resample not the same %d pixels different %.2f%%", res.diffCnt, per)
+		if *output != "" {
+			err = savePng(t, want, fmt.Sprintf("Resample.%s->%dx%d.want.png",
+				d.r, d.w, d.h))
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = savePng(t, got, fmt.Sprintf("Resample.%s->%dx%d.got.png",
+				d.r, d.w, d.h))
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = savePng(t, res.diffIm,
+				fmt.Sprintf("Resample.%s->%dx%d.diff.png", d.r, d.w, d.h))
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+}
+
+func TestHalveInplace(t *testing.T) {
+	for i, im := range makeImages(orig) {
+		m := HalveInplace(im)
+		b := im.Bounds()
+		got, want := m.Bounds(), image.Rectangle{
+			Min: b.Min,
+			Max: b.Min.Add(b.Max.Div(2)),
+		}
+		if !got.Eq(want) {
+			t.Error(i, "Want bounds", want, "got", got)
+		}
+	}
+}
+
+type results struct {
+	diffCnt int
+	psnr    float64
+	diffIm  *image.Gray
+}
+
+func compareImages(m1, m2 image.Image) results {
+	b := m1.Bounds()
+	s := b.Size()
+	res := results{}
+	mse := uint32(0)
+	for y := b.Min.Y; y < b.Max.Y; y++ {
+		for x := b.Min.X; x < b.Max.X; x++ {
+			r1, g1, b1, a1 := m1.At(x, y).RGBA()
+			r2, g2, b2, a2 := m2.At(x, y).RGBA()
+
+			mse += ((r1-r2)*(r1-r2) + (g1-g2)*(g1-g2) + (b1-b2)*(b1-b2)) / 3
+			if r1 != r2 || g1 != g2 || b1 != b2 || a1 != a2 {
+				if res.diffIm == nil {
+					res.diffIm = image.NewGray(m1.Bounds())
+				}
+				res.diffCnt++
+				res.diffIm.Set(x, y, color.White)
+			}
+		}
+	}
+	mse = mse / uint32(s.X*s.Y)
+	res.psnr = 20*math.Log10(1<<16) - 10*math.Log10(float64(mse))
+	return res
+}
+
+var testIm image.Image
+
+func init() {
+	r, err := os.Open(filepath.Join("testdata", "test.png"))
+	if err != nil {
+		panic(err)
+	}
+	defer r.Close()
+	testIm, err = png.Decode(r)
+}
+
+func fillTestImage(im image.Image) {
+	b := im.Bounds()
+	if !b.Eq(testIm.Bounds()) {
+		panic("Requested target image dimensions not equal reference image.")
+	}
+	src := testIm
+	if dst, ok := im.(*image.YCbCr); ok {
+		b := testIm.Bounds()
+		for y := b.Min.Y; y < b.Max.Y; y++ {
+			for x := b.Min.X; x < b.Max.X; x++ {
+				r, g, b, _ := src.At(x, y).RGBA()
+				yp, cb, cr := color.RGBToYCbCr(uint8(r), uint8(g), uint8(b))
+
+				dst.Y[dst.YOffset(x, y)] = yp
+				off := dst.COffset(x, y)
+				dst.Cb[off] = cb
+				dst.Cr[off] = cr
+			}
+		}
+		return
+	}
+	draw.Draw(im.(draw.Image), b, testIm, b.Min, draw.Src)
+}
+
+func savePng(t *testing.T, m image.Image, fn string) error {
+	fn = filepath.Join(*output, fn)
+	t.Log("Saving", fn)
+	f, err := os.Create(fn)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return png.Encode(f, m)
+}
+
+func getFilename(im image.Image, method string) string {
+	imgType := fmt.Sprintf("%T", im)
+	imgType = imgType[strings.Index(imgType, ".")+1:]
+	if m, ok := im.(*image.YCbCr); ok {
+		imgType += "." + m.SubsampleRatio.String()
+	}
+	return fmt.Sprintf("%s.%s.png", imgType, method)
+}
+
+func TestCompareResizeToHavleInplace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping TestCompareResizeToHavleInplace in short mode.")
+	}
+	images1, images2 := []image.Image{}, []image.Image{}
+	for _, im := range makeImages(testIm.Bounds()) {
+		fillTestImage(im)
+		images1 = append(images1, HalveInplace(im))
+	}
+	for _, im := range makeImages(testIm.Bounds()) {
+		fillTestImage(im)
+		s := im.Bounds().Size()
+		images2 = append(images2, Resize(im, im.Bounds(), s.X/2, s.Y/2))
+	}
+
+	var (
+		f   io.WriteCloser
+		err error
+	)
+	if *output != "" {
+		os.Mkdir(*output, os.FileMode(0777))
+		f, err = os.Create(filepath.Join(*output, "index.html"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer f.Close()
+		fmt.Fprintf(f, `<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Image comparison for TestCompareResizeToHavleInplace</title>
+  </head>
+  <body style="background-color: grey">
+<table>
+`)
+	}
+	for i, im1 := range images1 {
+		im2 := images2[i]
+		res := compareImages(im1, im2)
+		if *output != "" {
+			fmt.Fprintf(f, "<tr>")
+			fn := getFilename(im1, "halve")
+			err := savePng(t, im1, fn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			fmt.Fprintf(f, `<td><img src="%s"><br>%s`, fn, fn)
+
+			fn = getFilename(im1, "resize")
+			err = savePng(t, im2, fn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			fmt.Fprintf(f, `<td><img src="%s"><br>%s`, fn, fn)
+
+			if res.diffIm != nil {
+				fn = getFilename(im1, "diff")
+				err = savePng(t, res.diffIm, fn)
+				if err != nil {
+					t.Fatal(err)
+				}
+				fmt.Fprintf(f, `<td><img src="%s"><br>%s`, fn, fn)
+			}
+			fmt.Fprintln(f)
+		}
+
+		if res.psnr < psnrThreshold {
+			t.Errorf("%T PSNR too low %.4f", im1, res.psnr)
+		} else {
+			t.Logf("%T PSNR %.4f", im1, res.psnr)
+		}
+		s := im1.Bounds().Size()
+		tot := s.X * s.Y
+		if per := float32(100*res.diffCnt) / float32(tot); per > maxPixelDiffPercentage {
+			t.Errorf("%T not the same %d pixels different %.2f%%", im1, res.diffCnt, per)
+		}
+	}
+
+}

pkg/server/image.go

@@ -19,6 +19,7 @@ package server
 import (
 	"bytes"
 	"errors"
+	"expvar"
 	"fmt"
 	"image"
 	"image/jpeg"
@@ -42,6 +43,11 @@ import (
 
 const imageDebug = false
 
+var (
+	imageBytesServedVar  = expvar.NewInt("image-bytes-served")
+	imageBytesFetchedVar = expvar.NewInt("image-bytes-fetched")
+)
+
 type ImageHandler struct {
 	Fetcher             blob.StreamingFetcher
 	Cache               blobserver.Storage // optional
@@ -158,7 +164,8 @@ func (ih *ImageHandler) scaleImage(buf *bytes.Buffer, file blob.Ref) (format str
 	}
 	defer fr.Close()
 
-	_, err = io.Copy(buf, fr)
+	n, err := io.Copy(buf, fr)
+	imageBytesFetchedVar.Add(int64(n))
 	if err != nil {
 		return format, fmt.Errorf("image resize: error reading image %s: %v", file, err)
 	}
@@ -250,6 +257,7 @@ func (ih *ImageHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request, fil
 	h.Set("Last-Modified", time.Now().Format(http.TimeFormat))
 	h.Set("Content-Type", imageContentTypeOfFormat(format))
 	size := buf.Len()
+	imageBytesServedVar.Add(int64(size))
 	h.Set("Content-Length", fmt.Sprintf("%d", size))
 
 	if req.Method == "GET" {