From fa3715134a7e000ea9508030b6672490ae07d68e Mon Sep 17 00:00:00 2001
From: Brad Fitzpatrick <brad@danga.com>
Date: Wed, 8 Jun 2011 17:49:31 -0700
Subject: [PATCH] Start of indexer indexing file schema's content digests.

---
 .last_go_version                          |  2 +-
 config/dev-server-config.json             |  9 ++++---
 lib/go/camli/mysqlindexer/mysqlindexer.go | 13 +++++++++-
 lib/go/camli/mysqlindexer/receive.go      | 31 +++++++++++++++++++----
 4 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/.last_go_version b/.last_go_version
index 5a93864a0..d32337ca4 100644
--- a/.last_go_version
+++ b/.last_go_version
@@ -1 +1 @@
-6g version weekly.2011-06-02 8660+
+6g version weekly.2011-06-02 8682
diff --git a/config/dev-server-config.json b/config/dev-server-config.json
index 987d4aac1..21990f137 100644
--- a/config/dev-server-config.json
+++ b/config/dev-server-config.json
@@ -121,10 +121,11 @@
      "/indexer/": {
          "handler": "storage-mysqlindexer",
          "handlerArgs": {
-              "database": "devcamlistore",
-              "user": "root",
-              "password": "root",
-              "host": "127.0.0.1"
+             "database": "devcamlistore",
+             "user": "root",
+             "password": "root",
+             "host": "127.0.0.1",
+             "blobSource": "/bs/"
          }
      },
 
diff --git a/lib/go/camli/mysqlindexer/mysqlindexer.go b/lib/go/camli/mysqlindexer/mysqlindexer.go
index 8c4b0ca4f..1c0b0d3b8 100644
--- a/lib/go/camli/mysqlindexer/mysqlindexer.go
+++ b/lib/go/camli/mysqlindexer/mysqlindexer.go
@@ -39,11 +39,16 @@ type Indexer struct {
 	KeyFetcher   blobref.StreamingFetcher // for verifying claims
 	OwnerBlobRef *blobref.BlobRef
 
+	// Used for fetching blobs to find the complete sha1 of schema
+	// blobs.
+	BlobSource    blobserver.Storage
+
 	clientLock    sync.Mutex
 	cachedClients []*mysql.Client
 }
 
-func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, os.Error) {
+func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, os.Error) {
+	blobPrefix := config.RequiredString("blobSource")
 	indexer := &Indexer{
 		SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
 		Host:                      config.OptionalString("host", "localhost"),
@@ -55,6 +60,12 @@ func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Stora
 		return nil, err
 	}
 
+	sto, err := ld.GetStorage(blobPrefix)
+	if err != nil {
+		return nil, err
+	}
+	indexer.BlobSource = sto
+
 		//ownerBlobRef = client.SignerPublicKeyBlobref()
 		//if ownerBlobRef == nil {
 		//	log.Fatalf("Public key not configured.")
diff --git a/lib/go/camli/mysqlindexer/receive.go b/lib/go/camli/mysqlindexer/receive.go
index 02b0bcee4..cc298a264 100644
--- a/lib/go/camli/mysqlindexer/receive.go
+++ b/lib/go/camli/mysqlindexer/receive.go
@@ -17,17 +17,18 @@ limitations under the License.
 package mysqlindexer
 
 import (
-	"camli/blobref"
-	"camli/blobserver"
-	"camli/magic"
-	"camli/schema"
-
+	"crypto/sha1"
 	"io"
 	"json"
 	"log"
 	"os"
 
 	mysql "camli/third_party/github.com/Philio/GoMySQL"
+
+	"camli/blobref"
+	"camli/blobserver"
+	"camli/magic"
+	"camli/schema"
 )
 
 const maxSniffSize = 1024 * 16
@@ -140,6 +141,10 @@ func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (rets
 			if err = populatePermanode(client, blobRef, camli); err != nil {
 				return
 			}
+		case "file":
+			if err = mi.populateFile(client, blobRef, camli); err != nil {
+				return
+			}
 		}
 	}
 
@@ -202,3 +207,19 @@ func populatePermanode(client *mysql.Client, blobRef *blobref.BlobRef, camli *sc
 		blobRef.String(), camli.Signer)
 	return
 }
+
+func (mi *Indexer) populateFile(client *mysql.Client, blobRef *blobref.BlobRef, ss *schema.Superset) (err os.Error) {
+	seekFetcher, err := blobref.SeekerFromStreamingFetcher(mi.BlobSource)
+	if err != nil {
+		return err
+	}
+
+	sha1 := sha1.New()
+	fr := ss.NewFileReader(seekFetcher)
+	n, err := io.Copy(sha1, fr)
+	if err != nil {
+		return err
+	}
+	log.Printf("file %s blobref is %s, size %d", blobRef, blobref.FromHash("sha1", sha1), n)
+	return nil
+}