encrypt: start of tests, and notes/TODOs for metadata compaction

Change-Id: I78f060fca5e486585c16a4e33b04ba29a38dc71a

pkg/blobserver/encrypt/encrypt.go

@@ -19,6 +19,7 @@ package encrypt
 import (
 	"bufio"
 	"bytes"
+	"container/heap"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
@@ -65,11 +66,14 @@ $ ./dev-camtool sync --src=http://localhost:3179/enc/ --dest=stdout
 type storage struct {
 	*blobserver.SimpleBlobHubPartitionMap
 
-	block cipher.Block
-	index index.Storage // meta index
+	// index is the meta index.
+	// it's keyed by plaintext blobref.
+	// the value is the meta key (encodeMetaValue)
+	index index.Storage
 
 	// Encryption key.
-	key []byte
+	key   []byte
+	block cipher.Block // aes.NewCipher(key)
 
 	// blobs holds encrypted versions of all plaintext blobs.
 	blobs blobserver.Storage
@@ -83,11 +87,70 @@ type storage struct {
 	// into bigger blobs with multiple blob descriptions.
 	meta blobserver.Storage
 
-	mu sync.Mutex
-	// TODO: all meta blobs sorted by their size
+	// TODO(bradfitz): finish metdata compaction
+	/*
+		// mu guards the following
+		mu sync.Mutex
+		// toDelete are the meta blobrefs that are no longer
+		// necessary, as they're subsets of others.
+		toDelete []*blobref.BlobRef
+		// plainIn maps from a plaintext blobref to its currently-largest-describing metablob.
+		plainIn map[string]*metaBlobInfo
+		// smallMeta tracks a heap of meta blobs, sorted by their encrypted size
+		smallMeta metaBlobHeap
+	*/
+
+	// Hooks for testing
+	testRandIV func() []byte
 }
 
+func (s *storage) setKey(key []byte) error {
+	var err error
+	s.block, err = aes.NewCipher(key)
+	if err != nil {
+		return fmt.Errorf("The key must be exactly 16 bytes (currently only AES-128 is supported): %v", err)
+	}
+	s.key = key
+	return nil
+}
+
+type metaBlobInfo struct {
+	br     *blobref.BlobRef // of meta blob
+	n      int              // size of meta blob
+	plains []*blobref.BlobRef
+}
+
+type metaBlobHeap []*metaBlobInfo
+
+var _ heap.Interface = (*metaBlobHeap)(nil)
+
+func (s *metaBlobHeap) Push(x interface{}) {
+	*s = append(*s, x.(*metaBlobInfo))
+}
+
+func (s *metaBlobHeap) Pop() interface{} {
+	l := s.Len()
+	v := (*s)[l]
+	*s = (*s)[:l-1]
+	return v
+}
+
+func (s *metaBlobHeap) Len() int { return len(*s) }
+func (s *metaBlobHeap) Less(i, j int) bool {
+	sl := *s
+	v := sl[i].n < sl[j].n
+	if !v && sl[i].n == sl[j].n {
+		v = sl[i].br.String() < sl[j].br.String()
+	}
+	return v
+}
+
+func (s *metaBlobHeap) Swap(i, j int) { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] }
+
 func (s *storage) randIV() []byte {
+	if f := s.testRandIV; f != nil {
+		return f()
+	}
 	iv := make([]byte, s.block.BlockSize())
 	n, err := rand.Read(iv)
 	if err != nil {
@@ -285,6 +348,10 @@ func (s *storage) EnumerateBlobs(dest chan<- blobref.SizedBlobRef, after string,
 //
 // processEncryptedMetaBlob is not thread-safe.
 func (s *storage) processEncryptedMetaBlob(br *blobref.BlobRef, dat []byte) error {
+	mi := &metaBlobInfo{
+		br: br,
+		n:  len(dat),
+	}
 	log.Printf("processing meta blob %v: %d bytes", br, len(dat))
 	ivSize := s.block.BlockSize()
 	if len(dat) < ivSize+sha1.Size {
@@ -324,6 +391,7 @@ func (s *storage) processEncryptedMetaBlob(br *blobref.BlobRef, dat []byte) erro
 		}
 		plainBR, meta := line[:slash], line[slash+1:]
 		log.Printf("Adding meta: %q = %q", plainBR, meta)
+		mi.plains = append(mi.plains, blobref.Parse(plainBR))
 		if err := s.index.Set(plainBR, meta); err != nil {
 			return err
 		}
@@ -472,15 +540,16 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.S
 
 	key := config.OptionalString("key", "")
 	keyFile := config.OptionalString("keyFile", "")
+	var keyb []byte
 	switch {
 	case key != "":
-		sto.key, err = hex.DecodeString(key)
-		if err != nil || len(sto.key) != 16 {
+		keyb, err = hex.DecodeString(key)
+		if err != nil || len(keyb) != 16 {
 			return nil, fmt.Errorf("The 'key' parameter must be 16 bytes of 32 hex digits. (currently fixed at AES-128)")
 		}
 	case keyFile != "":
 		// TODO: check that keyFile's unix permissions aren't too permissive.
-		sto.key, err = ioutil.ReadFile(keyFile)
+		keyb, err = ioutil.ReadFile(keyFile)
 		if err != nil {
 			return nil, fmt.Errorf("Reading key file %v: %v", keyFile, err)
 		}
@@ -499,13 +568,14 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.S
 	if err != nil {
 		return
 	}
+
 	if sto.key == nil {
 		// TODO: add a way to prompt from stdin on start? or keychain support?
 		return nil, errors.New("no encryption key set with 'key' or 'keyFile'")
 	}
-	sto.block, err = aes.NewCipher(sto.key)
-	if err != nil {
-		return nil, fmt.Errorf("The key must be exactly 16 bytes (currently only AES-128 is supported): %v", err)
+
+	if err := sto.setKey(keyb); err != nil {
+		return nil, err
 	}
 
 	log.Printf("Reading encryption metadata...")

pkg/blobserver/encrypt/encrypt_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2013 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package encrypt
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io/ioutil"
+	"sync"
+	"testing"
+
+	"camlistore.org/pkg/blobref"
+	"camlistore.org/pkg/index"
+	"camlistore.org/pkg/test"
+)
+
+var testKey = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+
+type testStorage struct {
+	sto   *storage
+	blobs *test.Fetcher
+	meta  *test.Fetcher
+
+	mu sync.Mutex // guards iv
+	iv uint64
+}
+
+// fetchOrErrorString fetches br from sto and returns its body as a string.
+// If an error occurs the stringified error is returned, prefixed by "Error: ".
+func (ts *testStorage) fetchOrErrorString(br *blobref.BlobRef) string {
+	rc, _, err := ts.sto.FetchStreaming(br)
+	var slurp []byte
+	if err == nil {
+		defer rc.Close()
+		slurp, err = ioutil.ReadAll(rc)
+	}
+	if err != nil {
+		return fmt.Sprintf("Error: %v", err)
+	}
+	return string(slurp)
+}
+
+func newTestStorage() *testStorage {
+	sto := &storage{
+		index: index.NewMemoryStorage(),
+	}
+	if err := sto.setKey(testKey); err != nil {
+		panic(err)
+	}
+	ts := &testStorage{
+		sto:   sto,
+		blobs: new(test.Fetcher),
+		meta:  new(test.Fetcher),
+	}
+	sto.blobs = ts.blobs
+	sto.meta = ts.meta
+	sto.testRandIV = func() []byte {
+		ts.mu.Lock()
+		defer ts.mu.Unlock()
+		var ret [16]byte
+		ts.iv++
+		binary.BigEndian.PutUint64(ret[8:], ts.iv)
+		return ret[:]
+	}
+	return ts
+}
+
+func TestEncryptBasic(t *testing.T) {
+	ts := newTestStorage()
+	const blobData = "foo"
+	tb := &test.Blob{blobData}
+	tb.MustUpload(t, ts.sto)
+
+	if got := ts.fetchOrErrorString(tb.BlobRef()); got != blobData {
+		t.Errorf("Fetching plaintext blobref %v = %v; want %q", tb.BlobRef(), got, blobData)
+	}
+
+	if g, w := fmt.Sprintf("%q", ts.meta.BlobrefStrings()), `["sha1-370c753f7158504d11d8941efff4129112f2f975"]`; g != w {
+		t.Errorf("meta blobs = %v; want %v", g, w)
+	}
+	if g, w := fmt.Sprintf("%q", ts.blobs.BlobrefStrings()), `["sha1-64f05b6b313162b01db154fcc7b83238eb36c343"]`; g != w {
+		t.Errorf("enc blobs = %v; want %v", g, w)
+	}
+
+	// Make sure plainBR doesn't show up anywhere.
+	plainBR := tb.BlobRef().String()
+	for _, br := range append(ts.meta.BlobrefStrings(), ts.blobs.BlobrefStrings()...) {
+		if br == plainBR {
+			t.Fatal("plaintext blobref found in storage")
+		}
+	}
+}