From 7b60af2ac4eef2440b5abda91b34b630e5dd0c0c Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Mon, 9 Sep 2013 23:49:07 +1000 Subject: [PATCH] support thumbnailing or CR2 files Change-Id: I76f102a8cd4283f6fcb54985c52a16ddb8f6a44f --- pkg/magic/magic.go | 1 + pkg/server/image.go | 9 +- third_party/github.com/nf/cr2/buffer.go | 69 ++++ third_party/github.com/nf/cr2/buffer_test.go | 36 ++ third_party/github.com/nf/cr2/consts.go | 113 +++++++ third_party/github.com/nf/cr2/reader.go | 332 +++++++++++++++++++ third_party/github.com/nf/cr2/reader_test.go | 84 +++++ 7 files changed, 643 insertions(+), 1 deletion(-) create mode 100644 third_party/github.com/nf/cr2/buffer.go create mode 100644 third_party/github.com/nf/cr2/buffer_test.go create mode 100644 third_party/github.com/nf/cr2/consts.go create mode 100644 third_party/github.com/nf/cr2/reader.go create mode 100644 third_party/github.com/nf/cr2/reader_test.go diff --git a/pkg/magic/magic.go b/pkg/magic/magic.go index 7563b0a63..0edf01654 100644 --- a/pkg/magic/magic.go +++ b/pkg/magic/magic.go @@ -37,6 +37,7 @@ var prefixTable = []prefixEntry{ {[]byte("\xff\xd8\xff\xe1"), "image/jpeg"}, {[]byte("\xff\xd8\xff\xe0"), "image/jpeg"}, {[]byte("\xff\xd8\xff\xdb"), "image/jpeg"}, + {[]byte("\x49\x49\x2a\x00\x10\x00\x00\x00\x43\x52\x02"), "image/cr2"}, {[]byte{137, 'P', 'N', 'G', '\r', '\n', 26, 10}, "image/png"}, {[]byte("-----BEGIN PGP PUBLIC KEY BLOCK---"), "text/x-openpgp-public-key"}, {[]byte{'I', 'D', '3'}, "audio/mpeg"}, diff --git a/pkg/server/image.go b/pkg/server/image.go index 9bc23b1e9..cc8823b57 100644 --- a/pkg/server/image.go +++ b/pkg/server/image.go @@ -35,6 +35,8 @@ import ( "camlistore.org/pkg/magic" "camlistore.org/pkg/schema" "camlistore.org/pkg/search" + + _ "camlistore.org/third_party/github.com/nf/cr2" ) const imageDebug = false @@ -166,7 +168,8 @@ func (ih *ImageHandler) scaleImage(buf *bytes.Buffer, file blob.Ref) (format str } b := i.Bounds() - useBytesUnchanged := !imConfig.Modified + useBytesUnchanged := !imConfig.Modified && + format != "cr2" // always recompress CR2 files isSquare := b.Dx() == b.Dy() if ih.Square && !isSquare { @@ -178,6 +181,10 @@ func (ih *ImageHandler) scaleImage(buf *bytes.Buffer, file blob.Ref) (format str if !useBytesUnchanged { // Encode as a new image buf.Reset() + // Recompress CR2 files as JPEG + if format == "cr2" { + format = "jpeg" + } switch format { case "jpeg": err = jpeg.Encode(buf, i, nil) diff --git a/third_party/github.com/nf/cr2/buffer.go b/third_party/github.com/nf/cr2/buffer.go new file mode 100644 index 000000000..acc4b9c61 --- /dev/null +++ b/third_party/github.com/nf/cr2/buffer.go @@ -0,0 +1,69 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cr2 + +import "io" + +// buffer buffers an io.Reader to satisfy io.ReaderAt. +type buffer struct { + r io.Reader + buf []byte +} + +// fill reads data from b.r until the buffer contains at least end bytes. +func (b *buffer) fill(end int) error { + m := len(b.buf) + if end > m { + if end > cap(b.buf) { + newcap := 1024 + for newcap < end { + newcap *= 2 + } + newbuf := make([]byte, end, newcap) + copy(newbuf, b.buf) + b.buf = newbuf + } else { + b.buf = b.buf[:end] + } + if n, err := io.ReadFull(b.r, b.buf[m:end]); err != nil { + end = m + n + b.buf = b.buf[:end] + return err + } + } + return nil +} + +func (b *buffer) ReadAt(p []byte, off int64) (int, error) { + o := int(off) + end := o + len(p) + if int64(end) != off+int64(len(p)) { + return 0, io.ErrUnexpectedEOF + } + + err := b.fill(end) + return copy(p, b.buf[o:end]), err +} + +// Slice returns a slice of the underlying buffer. The slice contains +// n bytes starting at offset off. +func (b *buffer) Slice(off, n int) ([]byte, error) { + end := off + n + if err := b.fill(end); err != nil { + return nil, err + } + return b.buf[off:end], nil +} + +// newReaderAt converts an io.Reader into an io.ReaderAt. +func newReaderAt(r io.Reader) io.ReaderAt { + if ra, ok := r.(io.ReaderAt); ok { + return ra + } + return &buffer{ + r: r, + buf: make([]byte, 0, 1024), + } +} diff --git a/third_party/github.com/nf/cr2/buffer_test.go b/third_party/github.com/nf/cr2/buffer_test.go new file mode 100644 index 000000000..ce735a24d --- /dev/null +++ b/third_party/github.com/nf/cr2/buffer_test.go @@ -0,0 +1,36 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cr2 + +import ( + "io" + "strings" + "testing" +) + +var readAtTests = []struct { + n int + off int64 + s string + err error +}{ + {2, 0, "ab", nil}, + {6, 0, "abcdef", nil}, + {3, 3, "def", nil}, + {3, 5, "f", io.EOF}, + {3, 6, "", io.EOF}, +} + +func TestReadAt(t *testing.T) { + r := newReaderAt(strings.NewReader("abcdef")) + b := make([]byte, 10) + for _, test := range readAtTests { + n, err := r.ReadAt(b[:test.n], test.off) + s := string(b[:n]) + if s != test.s || err != test.err { + t.Errorf("buffer.ReadAt(<%v bytes>, %v): got %v, %q; want %v, %q", test.n, test.off, err, s, test.err, test.s) + } + } +} diff --git a/third_party/github.com/nf/cr2/consts.go b/third_party/github.com/nf/cr2/consts.go new file mode 100644 index 000000000..d538b4a40 --- /dev/null +++ b/third_party/github.com/nf/cr2/consts.go @@ -0,0 +1,113 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cr2 + +// A tiff image file contains one or more images. The metadata +// of each image is contained in an Image File Directory (IFD), +// which contains entries of 12 bytes each and is described +// on page 14-16 of the specification. An IFD entry consists of +// +// - a tag, which describes the signification of the entry, +// - the data type and length of the entry, +// - the data itself or a pointer to it if it is more than 4 bytes. +// +// The presence of a length means that each IFD is effectively an array. + +const ( + leHeader = "\x49\x49\x2a\x00\x10\x00\x00\x00\x43\x52\x02" + ifdLen = 12 // Length of an IFD entry in bytes. +) + +// Data types (p. 14-16 of the spec). +const ( + dtByte = 1 + dtASCII = 2 + dtShort = 3 + dtLong = 4 + dtRational = 5 +) + +// The length of one instance of each data type in bytes. +var lengths = [...]uint32{0, 1, 1, 2, 4, 8} + +// Tags (see p. 28-41 of the spec). +const ( + tImageWidth = 256 + tImageLength = 257 + tBitsPerSample = 258 + tCompression = 259 + tPhotometricInterpretation = 262 + + tStripOffsets = 273 + tSamplesPerPixel = 277 + tRowsPerStrip = 278 + tStripByteCounts = 279 + + tTileWidth = 322 + tTileLength = 323 + tTileOffsets = 324 + tTileByteCounts = 325 + + tXResolution = 282 + tYResolution = 283 + tResolutionUnit = 296 + + tPredictor = 317 + tColorMap = 320 + tExtraSamples = 338 + tSampleFormat = 339 +) + +// Compression types (defined in various places in the spec and supplements). +const ( + cNone = 1 + cCCITT = 2 + cG3 = 3 // Group 3 Fax. + cG4 = 4 // Group 4 Fax. + cLZW = 5 + cJPEGOld = 6 // Superseded by cJPEG. + cJPEG = 7 + cDeflate = 8 // zlib compression. + cPackBits = 32773 + cDeflateOld = 32946 // Superseded by cDeflate. +) + +// Photometric interpretation values (see p. 37 of the spec). +const ( + pWhiteIsZero = 0 + pBlackIsZero = 1 + pRGB = 2 + pPaletted = 3 + pTransMask = 4 // transparency mask + pCMYK = 5 + pYCbCr = 6 + pCIELab = 8 +) + +// Values for the tPredictor tag (page 64-65 of the spec). +const ( + prNone = 1 + prHorizontal = 2 +) + +// Values for the tResolutionUnit tag (page 18). +const ( + resNone = 1 + resPerInch = 2 // Dots per inch. + resPerCM = 3 // Dots per centimeter. +) + +// imageMode represents the mode of the image. +type imageMode int + +const ( + mBilevel imageMode = iota + mPaletted + mGray + mGrayInvert + mRGB + mRGBA + mNRGBA +) diff --git a/third_party/github.com/nf/cr2/reader.go b/third_party/github.com/nf/cr2/reader.go new file mode 100644 index 000000000..19fed7401 --- /dev/null +++ b/third_party/github.com/nf/cr2/reader.go @@ -0,0 +1,332 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cr2 implements rudimentary support for reading Canon Camera Raw 2 +// (CR2) files. +// +// CR2 is a bastardized TIFF file with a JPEG file inside it (yeah, thanks Canon). +// This package is a stripped back version of code.google.com/p/go.image/tiff. +// +// Known limitations: +// +// It has only been tested with files generated by a Canon EOS 550D. +// +// Because TIFF files and CR2 files share the same first few bytes, the image +// package's file type detection will fail to reocgnize a cr2 if the tiff +// reader is also imported. +package cr2 + +import ( + "encoding/binary" + "image" + "image/color" + "image/jpeg" + "io" +) + +// A FormatError reports that the input is not a valid TIFF image. +type FormatError string + +func (e FormatError) Error() string { + return "tiff: invalid format: " + string(e) +} + +// An UnsupportedError reports that the input uses a valid but +// unimplemented feature. +type UnsupportedError string + +func (e UnsupportedError) Error() string { + return "tiff: unsupported feature: " + string(e) +} + +// An InternalError reports that an internal error was encountered. +type InternalError string + +func (e InternalError) Error() string { + return "tiff: internal error: " + string(e) +} + +type decoder struct { + r io.ReaderAt + byteOrder binary.ByteOrder + config image.Config + mode imageMode + bpp uint + features map[int][]uint + palette []color.Color + + buf []byte + off int // Current offset in buf. + v uint32 // Buffer value for reading with arbitrary bit depths. + nbits uint // Remaining number of bits in v. +} + +// firstVal returns the first uint of the features entry with the given tag, +// or 0 if the tag does not exist. +func (d *decoder) firstVal(tag int) uint { + f := d.features[tag] + if len(f) == 0 { + return 0 + } + return f[0] +} + +// ifdUint decodes the IFD entry in p, which must be of the Byte, Short +// or Long type, and returns the decoded uint values. +func (d *decoder) ifdUint(p []byte) (u []uint, err error) { + var raw []byte + datatype := d.byteOrder.Uint16(p[2:4]) + count := d.byteOrder.Uint32(p[4:8]) + if datalen := lengths[datatype] * count; datalen > 4 { + // The IFD contains a pointer to the real value. + raw = make([]byte, datalen) + _, err = d.r.ReadAt(raw, int64(d.byteOrder.Uint32(p[8:12]))) + } else { + raw = p[8 : 8+datalen] + } + if err != nil { + return nil, err + } + + u = make([]uint, count) + switch datatype { + case dtByte: + for i := uint32(0); i < count; i++ { + u[i] = uint(raw[i]) + } + case dtShort: + for i := uint32(0); i < count; i++ { + u[i] = uint(d.byteOrder.Uint16(raw[2*i : 2*(i+1)])) + } + case dtLong: + for i := uint32(0); i < count; i++ { + u[i] = uint(d.byteOrder.Uint32(raw[4*i : 4*(i+1)])) + } + default: + return nil, UnsupportedError("data type") + } + return u, nil +} + +// parseIFD decides whether the the IFD entry in p is "interesting" and +// stows away the data in the decoder. +func (d *decoder) parseIFD(p []byte) error { + tag := d.byteOrder.Uint16(p[0:2]) + switch tag { + case tBitsPerSample, + tExtraSamples, + tPhotometricInterpretation, + tCompression, + tPredictor, + tStripOffsets, + tStripByteCounts, + tRowsPerStrip, + tTileWidth, + tTileLength, + tTileOffsets, + tTileByteCounts, + tImageLength, + tImageWidth: + val, err := d.ifdUint(p) + if err != nil { + return err + } + d.features[int(tag)] = val + case tColorMap: + val, err := d.ifdUint(p) + if err != nil { + return err + } + numcolors := len(val) / 3 + if len(val)%3 != 0 || numcolors <= 0 || numcolors > 256 { + return FormatError("bad ColorMap length") + } + d.palette = make([]color.Color, numcolors) + for i := 0; i < numcolors; i++ { + d.palette[i] = color.RGBA64{ + uint16(val[i]), + uint16(val[i+numcolors]), + uint16(val[i+2*numcolors]), + 0xffff, + } + } + case tSampleFormat: + // Page 27 of the spec: If the SampleFormat is present and + // the value is not 1 [= unsigned integer data], a Baseline + // TIFF reader that cannot handle the SampleFormat value + // must terminate the import process gracefully. + val, err := d.ifdUint(p) + if err != nil { + return err + } + for _, v := range val { + if v != 1 { + return UnsupportedError("sample format") + } + } + } + return nil +} + +// readBits reads n bits from the internal buffer starting at the current offset. +func (d *decoder) readBits(n uint) uint32 { + for d.nbits < n { + d.v <<= 8 + d.v |= uint32(d.buf[d.off]) + d.off++ + d.nbits += 8 + } + d.nbits -= n + rv := d.v >> d.nbits + d.v &^= rv << d.nbits + return rv +} + +// flushBits discards the unread bits in the buffer used by readBits. +// It is used at the end of a line. +func (d *decoder) flushBits() { + d.v = 0 + d.nbits = 0 +} + +// minInt returns the smaller of x or y. +func minInt(a, b int) int { + if a <= b { + return a + } + return b +} + +func newDecoder(r io.Reader) (*decoder, error) { + d := &decoder{ + r: newReaderAt(r), + features: make(map[int][]uint), + } + + p := make([]byte, len(leHeader)) + if _, err := d.r.ReadAt(p, 0); err != nil { + return nil, err + } + if string(p[0:len(leHeader)]) != leHeader { + return nil, FormatError("malformed header") + } + d.byteOrder = binary.LittleEndian + + ifdOffset := int64(d.byteOrder.Uint32(p[4:8])) + + // The first two bytes contain the number of entries (12 bytes each). + if _, err := d.r.ReadAt(p[0:2], ifdOffset); err != nil { + return nil, err + } + numItems := int(d.byteOrder.Uint16(p[0:2])) + + // All IFD entries are read in one chunk. + p = make([]byte, ifdLen*numItems) + if _, err := d.r.ReadAt(p, ifdOffset+2); err != nil { + return nil, err + } + + for i := 0; i < len(p); i += ifdLen { + if err := d.parseIFD(p[i : i+ifdLen]); err != nil { + return nil, err + } + } + + d.config.Width = int(d.firstVal(tImageWidth)) + d.config.Height = int(d.firstVal(tImageLength)) + + if _, ok := d.features[tBitsPerSample]; !ok { + return nil, FormatError("BitsPerSample tag missing") + } + d.bpp = d.firstVal(tBitsPerSample) + + // Determine the image mode. + switch d.firstVal(tPhotometricInterpretation) { + case pRGB: + for _, b := range d.features[tBitsPerSample] { + if b != 8 { + return nil, UnsupportedError("non-8-bit RGB image") + } + } + d.config.ColorModel = color.RGBAModel + // RGB images normally have 3 samples per pixel. + // If there are more, ExtraSamples (p. 31-32 of the spec) + // gives their meaning (usually an alpha channel). + // + // This implementation does not support extra samples + // of an unspecified type. + switch len(d.features[tBitsPerSample]) { + case 3: + d.mode = mRGB + case 4: + switch d.firstVal(tExtraSamples) { + case 1: + d.mode = mRGBA + case 2: + d.mode = mNRGBA + d.config.ColorModel = color.NRGBAModel + default: + return nil, FormatError("wrong number of samples for RGB") + } + default: + return nil, FormatError("wrong number of samples for RGB") + } + case pPaletted: + d.mode = mPaletted + d.config.ColorModel = color.Palette(d.palette) + case pWhiteIsZero: + d.mode = mGrayInvert + if d.bpp == 16 { + d.config.ColorModel = color.Gray16Model + } else { + d.config.ColorModel = color.GrayModel + } + case pBlackIsZero: + d.mode = mGray + if d.bpp == 16 { + d.config.ColorModel = color.Gray16Model + } else { + d.config.ColorModel = color.GrayModel + } + default: + return nil, UnsupportedError("color model") + } + + return d, nil +} + +// DecodeConfig returns the color model and dimensions of a TIFF image without +// decoding the entire image. +func DecodeConfig(r io.Reader) (image.Config, error) { + d, err := newDecoder(r) + if err != nil { + return image.Config{}, err + } + return d.config, nil +} + +// Decode reads a TIFF image from r and returns it as an image.Image. +// The type of Image returned depends on the contents of the TIFF. +func Decode(r io.Reader) (img image.Image, err error) { + d, err := newDecoder(r) + if err != nil { + return + } + offset := int64(d.features[tStripOffsets][0]) + n := int64(d.features[tStripByteCounts][0]) + switch d.firstVal(tCompression) { + case cJPEG, cJPEGOld: + default: + return nil, UnsupportedError("compression") + } + m, err2 := jpeg.Decode(io.NewSectionReader(d.r, offset, n)) + if err2 != nil { + return nil, err + } + return m, nil +} + +func init() { + image.RegisterFormat("cr2", leHeader, Decode, DecodeConfig) +} diff --git a/third_party/github.com/nf/cr2/reader_test.go b/third_party/github.com/nf/cr2/reader_test.go new file mode 100644 index 000000000..f6faca8e1 --- /dev/null +++ b/third_party/github.com/nf/cr2/reader_test.go @@ -0,0 +1,84 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cr2 + +import ( + "errors" + "image" + "io" + "net/http" + "os" + "testing" +) + +func TestDecode(t *testing.T) { + f, err := openSampleFile(t) + if err != nil { + t.Fatal(err) + } + defer f.Close() + m, kind, err := image.Decode(f) + if err != nil { + t.Fatal(err) + } + if kind != "cr2" { + t.Fatal("unexpected kind:", kind) + } + r := m.Bounds() + if r.Dx() != sampleWidth { + t.Error("width = %v, want %v", r.Dx(), sampleWidth) + } + if r.Dy() != sampleHeight { + t.Error("height = %v, want %v", r.Dy(), sampleHeight) + } +} + +// Fetch the sample file via HTTP so we don't put a 25mb data file in the repo. + +const ( + sampleFile = "testdata/sample.cr2" + sampleFileURL = "http://nf.wh3rd.net/img/sample.cr2" + sampleWidth = 5184 + sampleHeight = 3456 +) + +func openSampleFile(t *testing.T) (io.ReadCloser, error) { + if f, err := os.Open(sampleFile); err == nil { + return f, nil + } else if !os.IsNotExist(err) { + return nil, err + } + t.Logf("Fetching sample file...") + fi, err := os.Stat("testdata") + if err == nil && !fi.IsDir() { + return nil, errors.New("testdata is not a directory") + } + if os.IsNotExist(err) { + err = os.Mkdir("testdata", 0777) + } + if err != nil { + return nil, err + } + r, err := http.Get(sampleFileURL) + if err != nil { + return nil, err + } + defer r.Body.Close() + f, err := os.Create(sampleFile) + if err != nil { + return nil, err + } + if _, err = io.Copy(f, r.Body); err != nil { + f.Close() + os.Remove(sampleFile) + return nil, err + } + if _, err = f.Seek(0, os.SEEK_SET); err != nil { + f.Close() + os.Remove(sampleFile) + return nil, err + } + return f, nil +}