diff --git a/lib/go/camli/mysqlindexer/dbschema.go b/lib/go/camli/mysqlindexer/dbschema.go index 78196b81c..e54df81e9 100644 --- a/lib/go/camli/mysqlindexer/dbschema.go +++ b/lib/go/camli/mysqlindexer/dbschema.go @@ -18,7 +18,7 @@ package mysqlindexer import () -const requiredSchemaVersion = 10 +const requiredSchemaVersion = 11 func SchemaVersion() int { return requiredSchemaVersion @@ -35,12 +35,14 @@ type VARCHAR(100))`, `CREATE TABLE claims ( blobref VARCHAR(128) NOT NULL PRIMARY KEY, signer VARCHAR(128) NOT NULL, +verifiedkeyid VARCHAR(128) NULL, date VARCHAR(40) NOT NULL, -INDEX (signer, date), + INDEX (signer, date), + INDEX (verifiedkeyid, date), unverified CHAR(1) NULL, claim VARCHAR(50) NOT NULL, permanode VARCHAR(128) NOT NULL, -INDEX (permanode, signer, date), + INDEX (permanode, signer, date), attr VARCHAR(128) NULL, value VARCHAR(128) NULL)`, @@ -65,18 +67,18 @@ INDEX (bytesref))`, // Rows are one per camliType "claim", for claimType "set-attribute" or "add-attribute", // for attribute values that are known (needed to be indexed, e.g. "camliNamedRoot") // - // signer is verified GPG KeyId (e.g. "2931A67C26F5ABDA") + // keyid is verified GPG KeyId (e.g. "2931A67C26F5ABDA") // attr is e.g. "camliNamedRoot" // value is the claim's "value" field // claimdate is the "claimDate" field. // blobref is the blobref of the claim. // permanode is the claim's "permaNode" field. `CREATE TABLE signerattrvalue ( -signer VARCHAR(128) NOT NULL, +keyid VARCHAR(128) NOT NULL, attr VARCHAR(128) NOT NULL, value VARCHAR(255) NOT NULL, claimdate VARCHAR(40) NOT NULL, -INDEX (signer, attr, value, claimdate), +INDEX (keyid, attr, value, claimdate), blobref VARCHAR(128) NOT NULL, PRIMARY KEY (blobref), permanode VARCHAR(128) NOT NULL, diff --git a/lib/go/camli/mysqlindexer/mysqlindexer.go b/lib/go/camli/mysqlindexer/mysqlindexer.go index 551ebc7b8..9a84356f4 100644 --- a/lib/go/camli/mysqlindexer/mysqlindexer.go +++ b/lib/go/camli/mysqlindexer/mysqlindexer.go @@ -36,9 +36,7 @@ type Indexer struct { Host, User, Password, Database string Port int - // TODO: does this belong at this layer? KeyFetcher blobref.StreamingFetcher // for verifying claims - OwnerBlobRef *blobref.BlobRef // Used for fetching blobs to find the complete sha1 of schema // blobs. @@ -67,16 +65,14 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Stor } indexer.BlobSource = sto + // Good enough, for now: + indexer.KeyFetcher = indexer.BlobSource + //ownerBlobRef = client.SignerPublicKeyBlobref() //if ownerBlobRef == nil { // log.Fatalf("Public key not configured.") //} - //KeyFetcher: blobref.NewSerialStreamingFetcher( - // blobref.NewConfigDirFetcher(), - // storage), - //} - ok, err := indexer.IsAlive() if !ok { return nil, fmt.Errorf("Failed to connect to MySQL: %v", err) diff --git a/lib/go/camli/mysqlindexer/receive.go b/lib/go/camli/mysqlindexer/receive.go index 4faaa930b..ddc412b1e 100644 --- a/lib/go/camli/mysqlindexer/receive.go +++ b/lib/go/camli/mysqlindexer/receive.go @@ -28,11 +28,17 @@ import ( "camli/blobref" "camli/blobserver" + "camli/jsonsign" "camli/magic" "camli/schema" ) -const maxSniffSize = 1024 * 16 +// maxSniffSize is how much of a blob to buffer in memory for both +// MIME sniffing (in which case 1MB is way overkill) and also for +// holding a schema blob in memory for analysis in later steps (where +// 1MB is about the max size a claim can be, with about 1023K (of +// slack space) +const maxSniffSize = 1024 * 1024 type blobSniffer struct { header []byte @@ -57,6 +63,13 @@ func (sn *blobSniffer) IsTruncated() bool { return sn.written > maxSniffSize } +func (sn *blobSniffer) Body() (string, os.Error) { + if sn.IsTruncated() { + return "", os.NewError("was truncated") + } + return string(sn.header), nil +} + // returns content type (string) or nil if unknown func (sn *blobSniffer) MimeType() interface{} { if sn.mimeType != nil { @@ -135,7 +148,7 @@ func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (rets if camli := sniffer.camli; camli != nil { switch camli.Type { case "claim": - if err = mi.populateClaim(client, blobRef, camli); err != nil { + if err = mi.populateClaim(client, blobRef, camli, sniffer); err != nil { return } case "permanode": @@ -170,22 +183,44 @@ func execSQL(client *mysql.Client, sql string, args ...interface{}) (err os.Erro return } -func (mi *Indexer) populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) { +func (mi *Indexer) populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset, sniffer *blobSniffer) (err os.Error) { pnBlobref := blobref.Parse(camli.Permanode) if pnBlobref == nil { // Skip bogus claim with malformed permanode. return } + verifiedKeyId := "" + if rawJson, err := sniffer.Body(); err == nil { + vr := jsonsign.NewVerificationRequest(rawJson, mi.KeyFetcher) + if vr.Verify() { + verifiedKeyId = vr.SignerKeyId + log.Printf("mysqlindex: verified claim %s from %s", blobRef, verifiedKeyId) + } else { + log.Printf("mysqlindex: verification failure on claim %s: %v", blobRef, vr.Err) + } + } + if err = execSQL(client, - "INSERT IGNORE INTO claims (blobref, signer, date, unverified, claim, permanode, attr, value) "+ - "VALUES (?, ?, ?, 'Y', ?, ?, ?, ?)", - blobRef.String(), camli.Signer, camli.ClaimDate, + "INSERT IGNORE INTO claims (blobref, signer, verifiedkeyid, date, unverified, claim, permanode, attr, value) "+ + "VALUES (?, ?, ?, ?, 'Y', ?, ?, ?, ?)", + blobRef.String(), camli.Signer, verifiedKeyId, camli.ClaimDate, camli.ClaimType, camli.Permanode, camli.Attribute, camli.Value); err != nil { return } + if verifiedKeyId != "" { + // TODO: limit this to only certain attributes (for now, just "camliRoot") once search handler + // is working and the UI permits setting camliRoot. + if err = execSQL(client, "INSERT IGNORE INTO signerattrvalue (keyid, attr, value, claimdate, blobref, permanode) "+ + "VALUES (?, ?, ?, ?, ?, ?)", + verifiedKeyId, camli.Attribute, camli.Value, + camli.ClaimDate, blobRef.String(), camli.Permanode); err != nil { + return + } + } + // And update the lastmod on the permanode row. if err = execSQL(client, "INSERT IGNORE INTO permanodes (blobref) VALUES (?)",