mysqlindex: verify claims; add to signerattrvalue table

Change-Id: I44d6d6a6defe972c1a7dab9f8d39217704a809cc

lib/go/camli/mysqlindexer/dbschema.go

@@ -18,7 +18,7 @@ package mysqlindexer
 
 import ()
 
-const requiredSchemaVersion = 10
+const requiredSchemaVersion = 11
 
 func SchemaVersion() int {
 	return requiredSchemaVersion
@@ -35,12 +35,14 @@ type VARCHAR(100))`,
 		`CREATE TABLE claims (
 blobref VARCHAR(128) NOT NULL PRIMARY KEY,
 signer VARCHAR(128) NOT NULL,
+verifiedkeyid VARCHAR(128) NULL,
 date VARCHAR(40) NOT NULL, 
-INDEX (signer, date),
+  INDEX (signer, date),
+  INDEX (verifiedkeyid, date),
 unverified CHAR(1) NULL,
 claim VARCHAR(50) NOT NULL,
 permanode VARCHAR(128) NOT NULL,
-INDEX (permanode, signer, date),
+  INDEX (permanode, signer, date),
 attr VARCHAR(128) NULL,
 value VARCHAR(128) NULL)`,
 
@@ -65,18 +67,18 @@ INDEX (bytesref))`,
 		// Rows are one per camliType "claim", for claimType "set-attribute" or "add-attribute",
 		// for attribute values that are known (needed to be indexed, e.g. "camliNamedRoot")
 		//
-		// signer is verified GPG KeyId (e.g. "2931A67C26F5ABDA")
+		// keyid is verified GPG KeyId (e.g. "2931A67C26F5ABDA")
 		// attr is e.g. "camliNamedRoot"
 		// value is the claim's "value" field
 		// claimdate is the "claimDate" field.
 		// blobref is the blobref of the claim.
 		// permanode is the claim's "permaNode" field.
 		`CREATE TABLE signerattrvalue (
-signer VARCHAR(128) NOT NULL,
+keyid VARCHAR(128) NOT NULL,
 attr VARCHAR(128) NOT NULL,
 value VARCHAR(255) NOT NULL,
 claimdate VARCHAR(40) NOT NULL,
-INDEX (signer, attr, value, claimdate),
+INDEX (keyid, attr, value, claimdate),
 blobref VARCHAR(128) NOT NULL,
 PRIMARY KEY (blobref),
 permanode VARCHAR(128) NOT NULL,

lib/go/camli/mysqlindexer/mysqlindexer.go

@@ -36,9 +36,7 @@ type Indexer struct {
 	Host, User, Password, Database string
 	Port                           int
 
-	// TODO: does this belong at this layer?
 	KeyFetcher   blobref.StreamingFetcher // for verifying claims
-	OwnerBlobRef *blobref.BlobRef
 
 	// Used for fetching blobs to find the complete sha1 of schema
 	// blobs.
@@ -67,16 +65,14 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Stor
 	}
 	indexer.BlobSource = sto
 
+	// Good enough, for now:
+	indexer.KeyFetcher = indexer.BlobSource
+
 		//ownerBlobRef = client.SignerPublicKeyBlobref()
 		//if ownerBlobRef == nil {
 		//	log.Fatalf("Public key not configured.")
 		//}
 
-	//KeyFetcher: blobref.NewSerialStreamingFetcher(
-	//			blobref.NewConfigDirFetcher(),
-	//			storage),
-	//}
-
 	ok, err := indexer.IsAlive()
 	if !ok {
 		return nil, fmt.Errorf("Failed to connect to MySQL: %v", err)

lib/go/camli/mysqlindexer/receive.go

@@ -28,11 +28,17 @@ import (
 
 	"camli/blobref"
 	"camli/blobserver"
+	"camli/jsonsign"
 	"camli/magic"
 	"camli/schema"
 )
 
-const maxSniffSize = 1024 * 16
+// maxSniffSize is how much of a blob to buffer in memory for both
+// MIME sniffing (in which case 1MB is way overkill) and also for
+// holding a schema blob in memory for analysis in later steps (where
+// 1MB is about the max size a claim can be, with about 1023K (of
+// slack space)
+const maxSniffSize = 1024 * 1024
 
 type blobSniffer struct {
 	header   []byte
@@ -57,6 +63,13 @@ func (sn *blobSniffer) IsTruncated() bool {
 	return sn.written > maxSniffSize
 }
 
+func (sn *blobSniffer) Body() (string, os.Error) {
+	if sn.IsTruncated() {
+		return "", os.NewError("was truncated")
+	}
+	return string(sn.header), nil
+}
+
 // returns content type (string) or nil if unknown
 func (sn *blobSniffer) MimeType() interface{} {
 	if sn.mimeType != nil {
@@ -135,7 +148,7 @@ func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (rets
 	if camli := sniffer.camli; camli != nil {
 		switch camli.Type {
 		case "claim":
-			if err = mi.populateClaim(client, blobRef, camli); err != nil {
+			if err = mi.populateClaim(client, blobRef, camli, sniffer); err != nil {
 				return
 			}
 		case "permanode":
@@ -170,22 +183,44 @@ func execSQL(client *mysql.Client, sql string, args ...interface{}) (err os.Erro
 	return
 }
 
-func (mi *Indexer) populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) {
+func (mi *Indexer) populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset, sniffer *blobSniffer) (err os.Error) {
 	pnBlobref := blobref.Parse(camli.Permanode)
 	if pnBlobref == nil {
 		// Skip bogus claim with malformed permanode.
 		return
 	}
 
+	verifiedKeyId := ""
+	if rawJson, err := sniffer.Body(); err == nil {
+		vr := jsonsign.NewVerificationRequest(rawJson, mi.KeyFetcher)
+		if vr.Verify() {
+			verifiedKeyId = vr.SignerKeyId
+			log.Printf("mysqlindex: verified claim %s from %s", blobRef, verifiedKeyId)
+		} else {
+			log.Printf("mysqlindex: verification failure on claim %s: %v", blobRef, vr.Err)
+		}
+	}
+
 	if err = execSQL(client,
-		"INSERT IGNORE INTO claims (blobref, signer, date, unverified, claim, permanode, attr, value) "+
-			"VALUES (?, ?, ?, 'Y', ?, ?, ?, ?)",
-		blobRef.String(), camli.Signer, camli.ClaimDate,
+		"INSERT IGNORE INTO claims (blobref, signer, verifiedkeyid, date, unverified, claim, permanode, attr, value) "+
+			"VALUES (?, ?, ?, ?, 'Y', ?, ?, ?, ?)",
+		blobRef.String(), camli.Signer, verifiedKeyId, camli.ClaimDate,
 		camli.ClaimType, camli.Permanode,
 		camli.Attribute, camli.Value); err != nil {
 		return
 	}
 
+	if verifiedKeyId != "" {
+		// TODO: limit this to only certain attributes (for now, just "camliRoot") once search handler
+		// is working and the UI permits setting camliRoot.
+		if err = execSQL(client, "INSERT IGNORE INTO signerattrvalue (keyid, attr, value, claimdate, blobref, permanode) "+
+			"VALUES (?, ?, ?, ?, ?, ?)",
+			verifiedKeyId, camli.Attribute, camli.Value,
+			camli.ClaimDate, blobRef.String(), camli.Permanode); err != nil {
+			return
+		}
+	}
+
 	// And update the lastmod on the permanode row.
 	if err = execSQL(client,
 		"INSERT IGNORE INTO permanodes (blobref) VALUES (?)",