Merge "support thumbnailing or CR2 files"

pkg/magic/magic.go

@@ -37,6 +37,7 @@ var prefixTable = []prefixEntry{
 	{[]byte("\xff\xd8\xff\xe1"), "image/jpeg"},
 	{[]byte("\xff\xd8\xff\xe0"), "image/jpeg"},
 	{[]byte("\xff\xd8\xff\xdb"), "image/jpeg"},
+	{[]byte("\x49\x49\x2a\x00\x10\x00\x00\x00\x43\x52\x02"), "image/cr2"},
 	{[]byte{137, 'P', 'N', 'G', '\r', '\n', 26, 10}, "image/png"},
 	{[]byte("-----BEGIN PGP PUBLIC KEY BLOCK---"), "text/x-openpgp-public-key"},
 	{[]byte{'I', 'D', '3'}, "audio/mpeg"},

pkg/server/image.go

@@ -35,6 +35,8 @@ import (
 	"camlistore.org/pkg/magic"
 	"camlistore.org/pkg/schema"
 	"camlistore.org/pkg/search"
+
+	_ "camlistore.org/third_party/github.com/nf/cr2"
 )
 
 const imageDebug = false
@@ -166,7 +168,8 @@ func (ih *ImageHandler) scaleImage(buf *bytes.Buffer, file blob.Ref) (format str
 	}
 	b := i.Bounds()
 
-	useBytesUnchanged := !imConfig.Modified
+	useBytesUnchanged := !imConfig.Modified &&
+		format != "cr2" // always recompress CR2 files
 
 	isSquare := b.Dx() == b.Dy()
 	if ih.Square && !isSquare {
@@ -178,6 +181,10 @@ func (ih *ImageHandler) scaleImage(buf *bytes.Buffer, file blob.Ref) (format str
 	if !useBytesUnchanged {
 		// Encode as a new image
 		buf.Reset()
+		// Recompress CR2 files as JPEG
+		if format == "cr2" {
+			format = "jpeg"
+		}
 		switch format {
 		case "jpeg":
 			err = jpeg.Encode(buf, i, nil)

third_party/github.com/nf/cr2/buffer.go

@@ -0,0 +1,69 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cr2
+
+import "io"
+
+// buffer buffers an io.Reader to satisfy io.ReaderAt.
+type buffer struct {
+	r   io.Reader
+	buf []byte
+}
+
+// fill reads data from b.r until the buffer contains at least end bytes.
+func (b *buffer) fill(end int) error {
+	m := len(b.buf)
+	if end > m {
+		if end > cap(b.buf) {
+			newcap := 1024
+			for newcap < end {
+				newcap *= 2
+			}
+			newbuf := make([]byte, end, newcap)
+			copy(newbuf, b.buf)
+			b.buf = newbuf
+		} else {
+			b.buf = b.buf[:end]
+		}
+		if n, err := io.ReadFull(b.r, b.buf[m:end]); err != nil {
+			end = m + n
+			b.buf = b.buf[:end]
+			return err
+		}
+	}
+	return nil
+}
+
+func (b *buffer) ReadAt(p []byte, off int64) (int, error) {
+	o := int(off)
+	end := o + len(p)
+	if int64(end) != off+int64(len(p)) {
+		return 0, io.ErrUnexpectedEOF
+	}
+
+	err := b.fill(end)
+	return copy(p, b.buf[o:end]), err
+}
+
+// Slice returns a slice of the underlying buffer. The slice contains
+// n bytes starting at offset off.
+func (b *buffer) Slice(off, n int) ([]byte, error) {
+	end := off + n
+	if err := b.fill(end); err != nil {
+		return nil, err
+	}
+	return b.buf[off:end], nil
+}
+
+// newReaderAt converts an io.Reader into an io.ReaderAt.
+func newReaderAt(r io.Reader) io.ReaderAt {
+	if ra, ok := r.(io.ReaderAt); ok {
+		return ra
+	}
+	return &buffer{
+		r:   r,
+		buf: make([]byte, 0, 1024),
+	}
+}

third_party/github.com/nf/cr2/buffer_test.go

@@ -0,0 +1,36 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cr2
+
+import (
+	"io"
+	"strings"
+	"testing"
+)
+
+var readAtTests = []struct {
+	n   int
+	off int64
+	s   string
+	err error
+}{
+	{2, 0, "ab", nil},
+	{6, 0, "abcdef", nil},
+	{3, 3, "def", nil},
+	{3, 5, "f", io.EOF},
+	{3, 6, "", io.EOF},
+}
+
+func TestReadAt(t *testing.T) {
+	r := newReaderAt(strings.NewReader("abcdef"))
+	b := make([]byte, 10)
+	for _, test := range readAtTests {
+		n, err := r.ReadAt(b[:test.n], test.off)
+		s := string(b[:n])
+		if s != test.s || err != test.err {
+			t.Errorf("buffer.ReadAt(<%v bytes>, %v): got %v, %q; want %v, %q", test.n, test.off, err, s, test.err, test.s)
+		}
+	}
+}

third_party/github.com/nf/cr2/consts.go

@@ -0,0 +1,113 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cr2
+
+// A tiff image file contains one or more images. The metadata
+// of each image is contained in an Image File Directory (IFD),
+// which contains entries of 12 bytes each and is described
+// on page 14-16 of the specification. An IFD entry consists of
+//
+//  - a tag, which describes the signification of the entry,
+//  - the data type and length of the entry,
+//  - the data itself or a pointer to it if it is more than 4 bytes.
+//
+// The presence of a length means that each IFD is effectively an array.
+
+const (
+	leHeader = "\x49\x49\x2a\x00\x10\x00\x00\x00\x43\x52\x02"
+	ifdLen   = 12 // Length of an IFD entry in bytes.
+)
+
+// Data types (p. 14-16 of the spec).
+const (
+	dtByte     = 1
+	dtASCII    = 2
+	dtShort    = 3
+	dtLong     = 4
+	dtRational = 5
+)
+
+// The length of one instance of each data type in bytes.
+var lengths = [...]uint32{0, 1, 1, 2, 4, 8}
+
+// Tags (see p. 28-41 of the spec).
+const (
+	tImageWidth                = 256
+	tImageLength               = 257
+	tBitsPerSample             = 258
+	tCompression               = 259
+	tPhotometricInterpretation = 262
+
+	tStripOffsets    = 273
+	tSamplesPerPixel = 277
+	tRowsPerStrip    = 278
+	tStripByteCounts = 279
+
+	tTileWidth      = 322
+	tTileLength     = 323
+	tTileOffsets    = 324
+	tTileByteCounts = 325
+
+	tXResolution    = 282
+	tYResolution    = 283
+	tResolutionUnit = 296
+
+	tPredictor    = 317
+	tColorMap     = 320
+	tExtraSamples = 338
+	tSampleFormat = 339
+)
+
+// Compression types (defined in various places in the spec and supplements).
+const (
+	cNone       = 1
+	cCCITT      = 2
+	cG3         = 3 // Group 3 Fax.
+	cG4         = 4 // Group 4 Fax.
+	cLZW        = 5
+	cJPEGOld    = 6 // Superseded by cJPEG.
+	cJPEG       = 7
+	cDeflate    = 8 // zlib compression.
+	cPackBits   = 32773
+	cDeflateOld = 32946 // Superseded by cDeflate.
+)
+
+// Photometric interpretation values (see p. 37 of the spec).
+const (
+	pWhiteIsZero = 0
+	pBlackIsZero = 1
+	pRGB         = 2
+	pPaletted    = 3
+	pTransMask   = 4 // transparency mask
+	pCMYK        = 5
+	pYCbCr       = 6
+	pCIELab      = 8
+)
+
+// Values for the tPredictor tag (page 64-65 of the spec).
+const (
+	prNone       = 1
+	prHorizontal = 2
+)
+
+// Values for the tResolutionUnit tag (page 18).
+const (
+	resNone    = 1
+	resPerInch = 2 // Dots per inch.
+	resPerCM   = 3 // Dots per centimeter.
+)
+
+// imageMode represents the mode of the image.
+type imageMode int
+
+const (
+	mBilevel imageMode = iota
+	mPaletted
+	mGray
+	mGrayInvert
+	mRGB
+	mRGBA
+	mNRGBA
+)

third_party/github.com/nf/cr2/reader.go

@@ -0,0 +1,332 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cr2 implements rudimentary support for reading Canon Camera Raw 2
+// (CR2) files.
+//
+// CR2 is a bastardized TIFF file with a JPEG file inside it (yeah, thanks Canon).
+// This package is a stripped back version of code.google.com/p/go.image/tiff.
+//
+// Known limitations:
+//
+// It has only been tested with files generated by a Canon EOS 550D.
+//
+// Because TIFF files and CR2 files share the same first few bytes, the image
+// package's file type detection will fail to reocgnize a cr2 if the tiff
+// reader is also imported.
+package cr2
+
+import (
+	"encoding/binary"
+	"image"
+	"image/color"
+	"image/jpeg"
+	"io"
+)
+
+// A FormatError reports that the input is not a valid TIFF image.
+type FormatError string
+
+func (e FormatError) Error() string {
+	return "tiff: invalid format: " + string(e)
+}
+
+// An UnsupportedError reports that the input uses a valid but
+// unimplemented feature.
+type UnsupportedError string
+
+func (e UnsupportedError) Error() string {
+	return "tiff: unsupported feature: " + string(e)
+}
+
+// An InternalError reports that an internal error was encountered.
+type InternalError string
+
+func (e InternalError) Error() string {
+	return "tiff: internal error: " + string(e)
+}
+
+type decoder struct {
+	r         io.ReaderAt
+	byteOrder binary.ByteOrder
+	config    image.Config
+	mode      imageMode
+	bpp       uint
+	features  map[int][]uint
+	palette   []color.Color
+
+	buf   []byte
+	off   int    // Current offset in buf.
+	v     uint32 // Buffer value for reading with arbitrary bit depths.
+	nbits uint   // Remaining number of bits in v.
+}
+
+// firstVal returns the first uint of the features entry with the given tag,
+// or 0 if the tag does not exist.
+func (d *decoder) firstVal(tag int) uint {
+	f := d.features[tag]
+	if len(f) == 0 {
+		return 0
+	}
+	return f[0]
+}
+
+// ifdUint decodes the IFD entry in p, which must be of the Byte, Short
+// or Long type, and returns the decoded uint values.
+func (d *decoder) ifdUint(p []byte) (u []uint, err error) {
+	var raw []byte
+	datatype := d.byteOrder.Uint16(p[2:4])
+	count := d.byteOrder.Uint32(p[4:8])
+	if datalen := lengths[datatype] * count; datalen > 4 {
+		// The IFD contains a pointer to the real value.
+		raw = make([]byte, datalen)
+		_, err = d.r.ReadAt(raw, int64(d.byteOrder.Uint32(p[8:12])))
+	} else {
+		raw = p[8 : 8+datalen]
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	u = make([]uint, count)
+	switch datatype {
+	case dtByte:
+		for i := uint32(0); i < count; i++ {
+			u[i] = uint(raw[i])
+		}
+	case dtShort:
+		for i := uint32(0); i < count; i++ {
+			u[i] = uint(d.byteOrder.Uint16(raw[2*i : 2*(i+1)]))
+		}
+	case dtLong:
+		for i := uint32(0); i < count; i++ {
+			u[i] = uint(d.byteOrder.Uint32(raw[4*i : 4*(i+1)]))
+		}
+	default:
+		return nil, UnsupportedError("data type")
+	}
+	return u, nil
+}
+
+// parseIFD decides whether the the IFD entry in p is "interesting" and
+// stows away the data in the decoder.
+func (d *decoder) parseIFD(p []byte) error {
+	tag := d.byteOrder.Uint16(p[0:2])
+	switch tag {
+	case tBitsPerSample,
+		tExtraSamples,
+		tPhotometricInterpretation,
+		tCompression,
+		tPredictor,
+		tStripOffsets,
+		tStripByteCounts,
+		tRowsPerStrip,
+		tTileWidth,
+		tTileLength,
+		tTileOffsets,
+		tTileByteCounts,
+		tImageLength,
+		tImageWidth:
+		val, err := d.ifdUint(p)
+		if err != nil {
+			return err
+		}
+		d.features[int(tag)] = val
+	case tColorMap:
+		val, err := d.ifdUint(p)
+		if err != nil {
+			return err
+		}
+		numcolors := len(val) / 3
+		if len(val)%3 != 0 || numcolors <= 0 || numcolors > 256 {
+			return FormatError("bad ColorMap length")
+		}
+		d.palette = make([]color.Color, numcolors)
+		for i := 0; i < numcolors; i++ {
+			d.palette[i] = color.RGBA64{
+				uint16(val[i]),
+				uint16(val[i+numcolors]),
+				uint16(val[i+2*numcolors]),
+				0xffff,
+			}
+		}
+	case tSampleFormat:
+		// Page 27 of the spec: If the SampleFormat is present and
+		// the value is not 1 [= unsigned integer data], a Baseline
+		// TIFF reader that cannot handle the SampleFormat value
+		// must terminate the import process gracefully.
+		val, err := d.ifdUint(p)
+		if err != nil {
+			return err
+		}
+		for _, v := range val {
+			if v != 1 {
+				return UnsupportedError("sample format")
+			}
+		}
+	}
+	return nil
+}
+
+// readBits reads n bits from the internal buffer starting at the current offset.
+func (d *decoder) readBits(n uint) uint32 {
+	for d.nbits < n {
+		d.v <<= 8
+		d.v |= uint32(d.buf[d.off])
+		d.off++
+		d.nbits += 8
+	}
+	d.nbits -= n
+	rv := d.v >> d.nbits
+	d.v &^= rv << d.nbits
+	return rv
+}
+
+// flushBits discards the unread bits in the buffer used by readBits.
+// It is used at the end of a line.
+func (d *decoder) flushBits() {
+	d.v = 0
+	d.nbits = 0
+}
+
+// minInt returns the smaller of x or y.
+func minInt(a, b int) int {
+	if a <= b {
+		return a
+	}
+	return b
+}
+
+func newDecoder(r io.Reader) (*decoder, error) {
+	d := &decoder{
+		r:        newReaderAt(r),
+		features: make(map[int][]uint),
+	}
+
+	p := make([]byte, len(leHeader))
+	if _, err := d.r.ReadAt(p, 0); err != nil {
+		return nil, err
+	}
+	if string(p[0:len(leHeader)]) != leHeader {
+		return nil, FormatError("malformed header")
+	}
+	d.byteOrder = binary.LittleEndian
+
+	ifdOffset := int64(d.byteOrder.Uint32(p[4:8]))
+
+	// The first two bytes contain the number of entries (12 bytes each).
+	if _, err := d.r.ReadAt(p[0:2], ifdOffset); err != nil {
+		return nil, err
+	}
+	numItems := int(d.byteOrder.Uint16(p[0:2]))
+
+	// All IFD entries are read in one chunk.
+	p = make([]byte, ifdLen*numItems)
+	if _, err := d.r.ReadAt(p, ifdOffset+2); err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < len(p); i += ifdLen {
+		if err := d.parseIFD(p[i : i+ifdLen]); err != nil {
+			return nil, err
+		}
+	}
+
+	d.config.Width = int(d.firstVal(tImageWidth))
+	d.config.Height = int(d.firstVal(tImageLength))
+
+	if _, ok := d.features[tBitsPerSample]; !ok {
+		return nil, FormatError("BitsPerSample tag missing")
+	}
+	d.bpp = d.firstVal(tBitsPerSample)
+
+	// Determine the image mode.
+	switch d.firstVal(tPhotometricInterpretation) {
+	case pRGB:
+		for _, b := range d.features[tBitsPerSample] {
+			if b != 8 {
+				return nil, UnsupportedError("non-8-bit RGB image")
+			}
+		}
+		d.config.ColorModel = color.RGBAModel
+		// RGB images normally have 3 samples per pixel.
+		// If there are more, ExtraSamples (p. 31-32 of the spec)
+		// gives their meaning (usually an alpha channel).
+		//
+		// This implementation does not support extra samples
+		// of an unspecified type.
+		switch len(d.features[tBitsPerSample]) {
+		case 3:
+			d.mode = mRGB
+		case 4:
+			switch d.firstVal(tExtraSamples) {
+			case 1:
+				d.mode = mRGBA
+			case 2:
+				d.mode = mNRGBA
+				d.config.ColorModel = color.NRGBAModel
+			default:
+				return nil, FormatError("wrong number of samples for RGB")
+			}
+		default:
+			return nil, FormatError("wrong number of samples for RGB")
+		}
+	case pPaletted:
+		d.mode = mPaletted
+		d.config.ColorModel = color.Palette(d.palette)
+	case pWhiteIsZero:
+		d.mode = mGrayInvert
+		if d.bpp == 16 {
+			d.config.ColorModel = color.Gray16Model
+		} else {
+			d.config.ColorModel = color.GrayModel
+		}
+	case pBlackIsZero:
+		d.mode = mGray
+		if d.bpp == 16 {
+			d.config.ColorModel = color.Gray16Model
+		} else {
+			d.config.ColorModel = color.GrayModel
+		}
+	default:
+		return nil, UnsupportedError("color model")
+	}
+
+	return d, nil
+}
+
+// DecodeConfig returns the color model and dimensions of a TIFF image without
+// decoding the entire image.
+func DecodeConfig(r io.Reader) (image.Config, error) {
+	d, err := newDecoder(r)
+	if err != nil {
+		return image.Config{}, err
+	}
+	return d.config, nil
+}
+
+// Decode reads a TIFF image from r and returns it as an image.Image.
+// The type of Image returned depends on the contents of the TIFF.
+func Decode(r io.Reader) (img image.Image, err error) {
+	d, err := newDecoder(r)
+	if err != nil {
+		return
+	}
+	offset := int64(d.features[tStripOffsets][0])
+	n := int64(d.features[tStripByteCounts][0])
+	switch d.firstVal(tCompression) {
+	case cJPEG, cJPEGOld:
+	default:
+		return nil, UnsupportedError("compression")
+	}
+	m, err2 := jpeg.Decode(io.NewSectionReader(d.r, offset, n))
+	if err2 != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func init() {
+	image.RegisterFormat("cr2", leHeader, Decode, DecodeConfig)
+}

third_party/github.com/nf/cr2/reader_test.go

@@ -0,0 +1,84 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cr2
+
+import (
+	"errors"
+	"image"
+	"io"
+	"net/http"
+	"os"
+	"testing"
+)
+
+func TestDecode(t *testing.T) {
+	f, err := openSampleFile(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	m, kind, err := image.Decode(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if kind != "cr2" {
+		t.Fatal("unexpected kind:", kind)
+	}
+	r := m.Bounds()
+	if r.Dx() != sampleWidth {
+		t.Error("width = %v, want %v", r.Dx(), sampleWidth)
+	}
+	if r.Dy() != sampleHeight {
+		t.Error("height = %v, want %v", r.Dy(), sampleHeight)
+	}
+}
+
+// Fetch the sample file via HTTP so we don't put a 25mb data file in the repo.
+
+const (
+	sampleFile    = "testdata/sample.cr2"
+	sampleFileURL = "http://nf.wh3rd.net/img/sample.cr2"
+	sampleWidth   = 5184
+	sampleHeight  = 3456
+)
+
+func openSampleFile(t *testing.T) (io.ReadCloser, error) {
+	if f, err := os.Open(sampleFile); err == nil {
+		return f, nil
+	} else if !os.IsNotExist(err) {
+		return nil, err
+	}
+	t.Logf("Fetching sample file...")
+	fi, err := os.Stat("testdata")
+	if err == nil && !fi.IsDir() {
+		return nil, errors.New("testdata is not a directory")
+	}
+	if os.IsNotExist(err) {
+		err = os.Mkdir("testdata", 0777)
+	}
+	if err != nil {
+		return nil, err
+	}
+	r, err := http.Get(sampleFileURL)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Body.Close()
+	f, err := os.Create(sampleFile)
+	if err != nil {
+		return nil, err
+	}
+	if _, err = io.Copy(f, r.Body); err != nil {
+		f.Close()
+		os.Remove(sampleFile)
+		return nil, err
+	}
+	if _, err = f.Seek(0, os.SEEK_SET); err != nil {
+		f.Close()
+		os.Remove(sampleFile)
+		return nil, err
+	}
+	return f, nil
+}