From c3e37c2b569610c47e95dc10d9c62b51afce9981 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 7 Jul 2013 13:07:06 -0700 Subject: [PATCH] encrypt: start of tests, and notes/TODOs for metadata compaction Change-Id: I78f060fca5e486585c16a4e33b04ba29a38dc71a --- pkg/blobserver/encrypt/encrypt.go | 92 ++++++++++++++++++--- pkg/blobserver/encrypt/encrypt_test.go | 106 +++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 11 deletions(-) create mode 100644 pkg/blobserver/encrypt/encrypt_test.go diff --git a/pkg/blobserver/encrypt/encrypt.go b/pkg/blobserver/encrypt/encrypt.go index 2e7399744..2154d6816 100644 --- a/pkg/blobserver/encrypt/encrypt.go +++ b/pkg/blobserver/encrypt/encrypt.go @@ -19,6 +19,7 @@ package encrypt import ( "bufio" "bytes" + "container/heap" "crypto/aes" "crypto/cipher" "crypto/rand" @@ -65,11 +66,14 @@ $ ./dev-camtool sync --src=http://localhost:3179/enc/ --dest=stdout type storage struct { *blobserver.SimpleBlobHubPartitionMap - block cipher.Block - index index.Storage // meta index + // index is the meta index. + // it's keyed by plaintext blobref. + // the value is the meta key (encodeMetaValue) + index index.Storage // Encryption key. - key []byte + key []byte + block cipher.Block // aes.NewCipher(key) // blobs holds encrypted versions of all plaintext blobs. blobs blobserver.Storage @@ -83,11 +87,70 @@ type storage struct { // into bigger blobs with multiple blob descriptions. meta blobserver.Storage - mu sync.Mutex - // TODO: all meta blobs sorted by their size + // TODO(bradfitz): finish metdata compaction + /* + // mu guards the following + mu sync.Mutex + // toDelete are the meta blobrefs that are no longer + // necessary, as they're subsets of others. + toDelete []*blobref.BlobRef + // plainIn maps from a plaintext blobref to its currently-largest-describing metablob. + plainIn map[string]*metaBlobInfo + // smallMeta tracks a heap of meta blobs, sorted by their encrypted size + smallMeta metaBlobHeap + */ + + // Hooks for testing + testRandIV func() []byte } +func (s *storage) setKey(key []byte) error { + var err error + s.block, err = aes.NewCipher(key) + if err != nil { + return fmt.Errorf("The key must be exactly 16 bytes (currently only AES-128 is supported): %v", err) + } + s.key = key + return nil +} + +type metaBlobInfo struct { + br *blobref.BlobRef // of meta blob + n int // size of meta blob + plains []*blobref.BlobRef +} + +type metaBlobHeap []*metaBlobInfo + +var _ heap.Interface = (*metaBlobHeap)(nil) + +func (s *metaBlobHeap) Push(x interface{}) { + *s = append(*s, x.(*metaBlobInfo)) +} + +func (s *metaBlobHeap) Pop() interface{} { + l := s.Len() + v := (*s)[l] + *s = (*s)[:l-1] + return v +} + +func (s *metaBlobHeap) Len() int { return len(*s) } +func (s *metaBlobHeap) Less(i, j int) bool { + sl := *s + v := sl[i].n < sl[j].n + if !v && sl[i].n == sl[j].n { + v = sl[i].br.String() < sl[j].br.String() + } + return v +} + +func (s *metaBlobHeap) Swap(i, j int) { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] } + func (s *storage) randIV() []byte { + if f := s.testRandIV; f != nil { + return f() + } iv := make([]byte, s.block.BlockSize()) n, err := rand.Read(iv) if err != nil { @@ -285,6 +348,10 @@ func (s *storage) EnumerateBlobs(dest chan<- blobref.SizedBlobRef, after string, // // processEncryptedMetaBlob is not thread-safe. func (s *storage) processEncryptedMetaBlob(br *blobref.BlobRef, dat []byte) error { + mi := &metaBlobInfo{ + br: br, + n: len(dat), + } log.Printf("processing meta blob %v: %d bytes", br, len(dat)) ivSize := s.block.BlockSize() if len(dat) < ivSize+sha1.Size { @@ -324,6 +391,7 @@ func (s *storage) processEncryptedMetaBlob(br *blobref.BlobRef, dat []byte) erro } plainBR, meta := line[:slash], line[slash+1:] log.Printf("Adding meta: %q = %q", plainBR, meta) + mi.plains = append(mi.plains, blobref.Parse(plainBR)) if err := s.index.Set(plainBR, meta); err != nil { return err } @@ -472,15 +540,16 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.S key := config.OptionalString("key", "") keyFile := config.OptionalString("keyFile", "") + var keyb []byte switch { case key != "": - sto.key, err = hex.DecodeString(key) - if err != nil || len(sto.key) != 16 { + keyb, err = hex.DecodeString(key) + if err != nil || len(keyb) != 16 { return nil, fmt.Errorf("The 'key' parameter must be 16 bytes of 32 hex digits. (currently fixed at AES-128)") } case keyFile != "": // TODO: check that keyFile's unix permissions aren't too permissive. - sto.key, err = ioutil.ReadFile(keyFile) + keyb, err = ioutil.ReadFile(keyFile) if err != nil { return nil, fmt.Errorf("Reading key file %v: %v", keyFile, err) } @@ -499,13 +568,14 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.S if err != nil { return } + if sto.key == nil { // TODO: add a way to prompt from stdin on start? or keychain support? return nil, errors.New("no encryption key set with 'key' or 'keyFile'") } - sto.block, err = aes.NewCipher(sto.key) - if err != nil { - return nil, fmt.Errorf("The key must be exactly 16 bytes (currently only AES-128 is supported): %v", err) + + if err := sto.setKey(keyb); err != nil { + return nil, err } log.Printf("Reading encryption metadata...") diff --git a/pkg/blobserver/encrypt/encrypt_test.go b/pkg/blobserver/encrypt/encrypt_test.go new file mode 100644 index 000000000..56562a598 --- /dev/null +++ b/pkg/blobserver/encrypt/encrypt_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package encrypt + +import ( + "encoding/binary" + "fmt" + "io/ioutil" + "sync" + "testing" + + "camlistore.org/pkg/blobref" + "camlistore.org/pkg/index" + "camlistore.org/pkg/test" +) + +var testKey = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + +type testStorage struct { + sto *storage + blobs *test.Fetcher + meta *test.Fetcher + + mu sync.Mutex // guards iv + iv uint64 +} + +// fetchOrErrorString fetches br from sto and returns its body as a string. +// If an error occurs the stringified error is returned, prefixed by "Error: ". +func (ts *testStorage) fetchOrErrorString(br *blobref.BlobRef) string { + rc, _, err := ts.sto.FetchStreaming(br) + var slurp []byte + if err == nil { + defer rc.Close() + slurp, err = ioutil.ReadAll(rc) + } + if err != nil { + return fmt.Sprintf("Error: %v", err) + } + return string(slurp) +} + +func newTestStorage() *testStorage { + sto := &storage{ + index: index.NewMemoryStorage(), + } + if err := sto.setKey(testKey); err != nil { + panic(err) + } + ts := &testStorage{ + sto: sto, + blobs: new(test.Fetcher), + meta: new(test.Fetcher), + } + sto.blobs = ts.blobs + sto.meta = ts.meta + sto.testRandIV = func() []byte { + ts.mu.Lock() + defer ts.mu.Unlock() + var ret [16]byte + ts.iv++ + binary.BigEndian.PutUint64(ret[8:], ts.iv) + return ret[:] + } + return ts +} + +func TestEncryptBasic(t *testing.T) { + ts := newTestStorage() + const blobData = "foo" + tb := &test.Blob{blobData} + tb.MustUpload(t, ts.sto) + + if got := ts.fetchOrErrorString(tb.BlobRef()); got != blobData { + t.Errorf("Fetching plaintext blobref %v = %v; want %q", tb.BlobRef(), got, blobData) + } + + if g, w := fmt.Sprintf("%q", ts.meta.BlobrefStrings()), `["sha1-370c753f7158504d11d8941efff4129112f2f975"]`; g != w { + t.Errorf("meta blobs = %v; want %v", g, w) + } + if g, w := fmt.Sprintf("%q", ts.blobs.BlobrefStrings()), `["sha1-64f05b6b313162b01db154fcc7b83238eb36c343"]`; g != w { + t.Errorf("enc blobs = %v; want %v", g, w) + } + + // Make sure plainBR doesn't show up anywhere. + plainBR := tb.BlobRef().String() + for _, br := range append(ts.meta.BlobrefStrings(), ts.blobs.BlobrefStrings()...) { + if br == plainBR { + t.Fatal("plaintext blobref found in storage") + } + } +}