Put claims in memory too for in-memory search. Required index schema version bump.

Change-Id: I194d65476bddea111277cd0b1472c56b5527226b

pkg/index/corpus.go

@@ -44,8 +44,8 @@ type FileMeta struct {
 }
 
 type PermanodeMeta struct {
-	OwnerKeyId string
-	// Claims     ClaimList
+	// TODO: OwnerKeyId string
+	Claims []camtypes.Claim
 }
 
 func newCorpus() *Corpus {
@@ -95,7 +95,7 @@ func (s crashStorage) Find(key string) Iterator {
 // *********** Updating the corpus
 
 func (c *Corpus) scanFromStorage(s Storage) error {
-	for _, prefix := range []string{"meta:", "signerkeyid:"} {
+	for _, prefix := range []string{"meta:", "signerkeyid:", "claim|"} {
 		if err := c.scanPrefix(s, prefix); err != nil {
 			return err
 		}
@@ -122,6 +122,7 @@ var corpusMergeFunc = map[string]func(c *Corpus, k, v string) error{
 	"have":        nil, // redundant with "meta"
 	"meta":        (*Corpus).mergeMetaRow,
 	"signerkeyid": (*Corpus).mergeSignerKeyIdRow,
+	"claim":       (*Corpus).mergeClaimRow,
 }
 
 func (c *Corpus) addBlob(br blob.Ref, mm mutationMap) error {
@@ -148,7 +149,7 @@ func (c *Corpus) mergeMetaRow(k, v string) error {
 	if !ok {
 		return fmt.Errorf("bogus meta row: %q -> %q", k, v)
 	}
-	bm.CamliType = c.strLocked(bm.CamliType)
+	bm.CamliType = c.str(bm.CamliType)
 	c.blobs[bm.Ref] = &bm
 	if bm.CamliType != "" {
 		m, ok := c.camBlobs[bm.CamliType]
@@ -170,15 +171,27 @@ func (c *Corpus) mergeSignerKeyIdRow(k, v string) error {
 	return nil
 }
 
-// str returns s, interned.
-func (c *Corpus) str(s string) string {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.strLocked(s)
+func (c *Corpus) mergeClaimRow(k, v string) error {
+	cl, ok := kvClaim(k, v)
+	if !ok || !cl.Permanode.Valid() {
+		return fmt.Errorf("bogus claim row: %q -> %q", k, v)
+	}
+	cl.Type = c.str(cl.Type)
+	cl.Attr = c.str(cl.Attr)
+	cl.Value = c.str(cl.Value) // less likely to intern, but some (tags) do
+
+	pn := cl.Permanode
+	pm, ok := c.permanodes[pn]
+	if !ok {
+		pm = new(PermanodeMeta)
+		c.permanodes[pn] = pm
+	}
+	pm.Claims = append(pm.Claims, cl)
+	return nil
 }
 
-// strLocked returns s, interned.
-func (c *Corpus) strLocked(s string) string {
+// str returns s, interned.
+func (c *Corpus) str(s string) string {
 	if s == "" {
 		return ""
 	}
@@ -241,3 +254,44 @@ func (c *Corpus) KeyId(signer blob.Ref) (string, error) {
 	}
 	return "", ErrNotFound
 }
+
+func (c *Corpus) isDeletedLocked(br blob.Ref) bool {
+	// TODO: implement
+	return false
+}
+
+func (c *Corpus) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref,
+	signerFilter blob.Ref,
+	attrFilter string) ([]camtypes.Claim, error) {
+	needSort := false
+	defer func() {
+		if needSort {
+			// TODO: schedule sort of these.  It's not
+			// required by the interface, but we know our
+			// caller will want to do it, so make their
+			// job easier and give it to them
+			// pre-sorted. We do it here rather than
+			// during on-start scanning to save CPU, to do
+			// it fewer times per permanode.
+		}
+	}()
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	pm, ok := c.permanodes[permaNode]
+	if !ok {
+		return nil, nil
+	}
+	for _, cl := range pm.Claims {
+		if c.isDeletedLocked(cl.BlobRef) {
+			continue
+		}
+		if signerFilter.Valid() && cl.Signer != signerFilter {
+			continue
+		}
+		if attrFilter != "" && cl.Attr != attrFilter {
+			continue
+		}
+		dst = append(dst, cl)
+	}
+	return dst, nil
+}

pkg/index/index.go

@@ -494,6 +494,9 @@ func (x *Index) GetRecentPermanodes(dest chan<- camtypes.RecentPermanode, owner
 func (x *Index) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref,
 	signerFilter blob.Ref,
 	attrFilter string) ([]camtypes.Claim, error) {
+	if x.corpus != nil {
+		return x.corpus.AppendClaims(dst, permaNode, signerFilter, attrFilter)
+	}
 	var (
 		keyId string
 		err   error
@@ -509,9 +512,6 @@ func (x *Index) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref,
 		}
 		it = x.queryPrefix(keyPermanodeClaim, permaNode, keyId)
 	} else {
-		// TODO: for the Signer field below, we'll need a keyId->blob.Ref lookup
-		// too. For now just bail.
-		panic("TODO: not implemented. See comment.")
 		it = x.queryPrefix(keyPermanodeClaim, permaNode)
 	}
 	defer closeIterator(it, &err)
@@ -529,33 +529,55 @@ func (x *Index) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref,
 		if mustHave != "" && !strings.Contains(val, mustHave) {
 			continue
 		}
-		keyPart := strings.Split(it.Key(), "|")
-		valPart := strings.Split(val, "|")
-		if len(keyPart) < 5 || len(valPart) < 3 {
-			continue
-		}
-		attr := urld(valPart[1])
-		if attrFilter != "" && attr != attrFilter {
-			continue
-		}
-		claimRef, ok := blob.Parse(keyPart[4])
+		cl, ok := kvClaim(it.Key(), val)
 		if !ok {
 			continue
 		}
-		date, _ := time.Parse(time.RFC3339, keyPart[3])
-		dst = append(dst, camtypes.Claim{
-			BlobRef:   claimRef,
-			Signer:    signerFilter,
-			Permanode: permaNode,
-			Date:      date,
-			Type:      urld(valPart[0]),
-			Attr:      attr,
-			Value:     urld(valPart[2]),
-		})
+		if attrFilter != "" && cl.Attr != attrFilter {
+			continue
+		}
+		if signerFilter.Valid() && cl.Signer != signerFilter {
+			continue
+		}
+		dst = append(dst, cl)
 	}
 	return dst, nil
 }
 
+func kvClaim(k, v string) (c camtypes.Claim, ok bool) {
+	// TODO(bradfitz): remove the strings.Split calls to reduce allocations.
+	keyPart := strings.Split(k, "|")
+	valPart := strings.Split(v, "|")
+	if len(keyPart) < 5 || len(valPart) < 4 {
+		return
+	}
+	signerRef, ok := blob.Parse(valPart[3])
+	if !ok {
+		return
+	}
+	permaNode, ok := blob.Parse(keyPart[1])
+	if !ok {
+		return
+	}
+	claimRef, ok := blob.Parse(keyPart[4])
+	if !ok {
+		return
+	}
+	date, err := time.Parse(time.RFC3339, keyPart[3])
+	if err != nil {
+		return
+	}
+	return camtypes.Claim{
+		BlobRef:   claimRef,
+		Signer:    signerRef,
+		Permanode: permaNode,
+		Date:      date,
+		Type:      urld(valPart[0]),
+		Attr:      urld(valPart[1]),
+		Value:     urld(valPart[2]),
+	}, true
+}
+
 func (x *Index) GetBlobMeta(br blob.Ref) (camtypes.BlobMeta, error) {
 	if x.corpus != nil {
 		return x.corpus.GetBlobMeta(br)

pkg/index/interface.go

@@ -27,6 +27,7 @@ type Interface interface {
 	// they filter the return items to only claims made by the given signer
 	// or claims about the given attribute, respectively.
 	// Deleted claims are never returned.
+	// The items may be appended in any order.
 	AppendClaims(dst []camtypes.Claim, permaNode blob.Ref,
 		signerFilter blob.Ref,
 		attrFilter string) ([]camtypes.Claim, error)

pkg/index/keys.go

@@ -24,7 +24,7 @@ import (
 
 // requiredSchemaVersion is incremented every time
 // an index key type is added, changed, or removed.
-const requiredSchemaVersion = 1
+const requiredSchemaVersion = 2
 
 // type of key returns the identifier in k before the first ":" or "|".
 // (Originally we packed keys by hand and there are a mix of styles)
@@ -159,6 +159,11 @@ var (
 			{"claimType", typeStr},
 			{"attr", typeStr},
 			{"value", typeStr},
+			// And the signerRef, which seems redundant
+			// with the signer keyId in the jey, but the
+			// Claim struct needs this, and there's 1:m
+			// for keyId:blobRef, so:
+			{"signerRef", typeBlobRef},
 		},
 	}
 

pkg/index/receive.go

@@ -344,7 +344,7 @@ func (ix *Index) populateClaim(b *schema.Blob, mm mutationMap) error {
 	mm.Set(recentKey, pnbr.String())
 
 	claimKey := keyPermanodeClaim.Key(pnbr, verifiedKeyId, claim.ClaimDateString(), br)
-	mm.Set(claimKey, keyPermanodeClaim.Val(claim.ClaimType(), attr, value))
+	mm.Set(claimKey, keyPermanodeClaim.Val(claim.ClaimType(), attr, value, vr.CamliSigner))
 
 	if strings.HasPrefix(attr, "camliPath:") {
 		targetRef, ok := blob.Parse(value)

pkg/search/query_test.go

@@ -425,7 +425,7 @@ func TestQueryPermanodeAttrValueMatches(t *testing.T) {
 // find permanodes matching a certain file query
 func TestQueryFileConstraint(t *testing.T) { testQuery(t, testQueryFileConstraint, indexClassic) }
 func TestQueryFileConstraint_Scan(t *testing.T) {
-	t.Skip("TODO: claims in memory")
+	t.Skip("TODO: fileinfo in memory")
 	testQuery(t, testQueryFileConstraint, indexCorpusScan)
 }
 func TestQueryFileConstraint_Build(t *testing.T) {

pkg/types/camtypes/search.go

@@ -38,9 +38,10 @@ func (a RecentPermanode) Equal(b RecentPermanode) bool {
 		a.LastModTime.Equal(b.LastModTime)
 }
 
-// TODO: document/decide how to represent "multi" claims here. One Claim each? Add Multi in here?
-// Move/merge this in with the schema package?
 type Claim struct {
+	// TODO: document/decide how to represent "multi" claims here. One Claim each? Add Multi in here?
+	// Move/merge this in with the schema package?
+
 	BlobRef, Signer, Permanode blob.Ref
 
 	Date time.Time