diff --git a/pkg/index/corpus.go b/pkg/index/corpus.go index 98094eb74..846901fc2 100644 --- a/pkg/index/corpus.go +++ b/pkg/index/corpus.go @@ -44,8 +44,8 @@ type FileMeta struct { } type PermanodeMeta struct { - OwnerKeyId string - // Claims ClaimList + // TODO: OwnerKeyId string + Claims []camtypes.Claim } func newCorpus() *Corpus { @@ -95,7 +95,7 @@ func (s crashStorage) Find(key string) Iterator { // *********** Updating the corpus func (c *Corpus) scanFromStorage(s Storage) error { - for _, prefix := range []string{"meta:", "signerkeyid:"} { + for _, prefix := range []string{"meta:", "signerkeyid:", "claim|"} { if err := c.scanPrefix(s, prefix); err != nil { return err } @@ -122,6 +122,7 @@ var corpusMergeFunc = map[string]func(c *Corpus, k, v string) error{ "have": nil, // redundant with "meta" "meta": (*Corpus).mergeMetaRow, "signerkeyid": (*Corpus).mergeSignerKeyIdRow, + "claim": (*Corpus).mergeClaimRow, } func (c *Corpus) addBlob(br blob.Ref, mm mutationMap) error { @@ -148,7 +149,7 @@ func (c *Corpus) mergeMetaRow(k, v string) error { if !ok { return fmt.Errorf("bogus meta row: %q -> %q", k, v) } - bm.CamliType = c.strLocked(bm.CamliType) + bm.CamliType = c.str(bm.CamliType) c.blobs[bm.Ref] = &bm if bm.CamliType != "" { m, ok := c.camBlobs[bm.CamliType] @@ -170,15 +171,27 @@ func (c *Corpus) mergeSignerKeyIdRow(k, v string) error { return nil } -// str returns s, interned. -func (c *Corpus) str(s string) string { - c.mu.Lock() - defer c.mu.Unlock() - return c.strLocked(s) +func (c *Corpus) mergeClaimRow(k, v string) error { + cl, ok := kvClaim(k, v) + if !ok || !cl.Permanode.Valid() { + return fmt.Errorf("bogus claim row: %q -> %q", k, v) + } + cl.Type = c.str(cl.Type) + cl.Attr = c.str(cl.Attr) + cl.Value = c.str(cl.Value) // less likely to intern, but some (tags) do + + pn := cl.Permanode + pm, ok := c.permanodes[pn] + if !ok { + pm = new(PermanodeMeta) + c.permanodes[pn] = pm + } + pm.Claims = append(pm.Claims, cl) + return nil } -// strLocked returns s, interned. -func (c *Corpus) strLocked(s string) string { +// str returns s, interned. +func (c *Corpus) str(s string) string { if s == "" { return "" } @@ -241,3 +254,44 @@ func (c *Corpus) KeyId(signer blob.Ref) (string, error) { } return "", ErrNotFound } + +func (c *Corpus) isDeletedLocked(br blob.Ref) bool { + // TODO: implement + return false +} + +func (c *Corpus) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref, + signerFilter blob.Ref, + attrFilter string) ([]camtypes.Claim, error) { + needSort := false + defer func() { + if needSort { + // TODO: schedule sort of these. It's not + // required by the interface, but we know our + // caller will want to do it, so make their + // job easier and give it to them + // pre-sorted. We do it here rather than + // during on-start scanning to save CPU, to do + // it fewer times per permanode. + } + }() + c.mu.RLock() + defer c.mu.RUnlock() + pm, ok := c.permanodes[permaNode] + if !ok { + return nil, nil + } + for _, cl := range pm.Claims { + if c.isDeletedLocked(cl.BlobRef) { + continue + } + if signerFilter.Valid() && cl.Signer != signerFilter { + continue + } + if attrFilter != "" && cl.Attr != attrFilter { + continue + } + dst = append(dst, cl) + } + return dst, nil +} diff --git a/pkg/index/index.go b/pkg/index/index.go index 58e6d14f6..6994b5ec0 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -494,6 +494,9 @@ func (x *Index) GetRecentPermanodes(dest chan<- camtypes.RecentPermanode, owner func (x *Index) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref, signerFilter blob.Ref, attrFilter string) ([]camtypes.Claim, error) { + if x.corpus != nil { + return x.corpus.AppendClaims(dst, permaNode, signerFilter, attrFilter) + } var ( keyId string err error @@ -509,9 +512,6 @@ func (x *Index) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref, } it = x.queryPrefix(keyPermanodeClaim, permaNode, keyId) } else { - // TODO: for the Signer field below, we'll need a keyId->blob.Ref lookup - // too. For now just bail. - panic("TODO: not implemented. See comment.") it = x.queryPrefix(keyPermanodeClaim, permaNode) } defer closeIterator(it, &err) @@ -529,33 +529,55 @@ func (x *Index) AppendClaims(dst []camtypes.Claim, permaNode blob.Ref, if mustHave != "" && !strings.Contains(val, mustHave) { continue } - keyPart := strings.Split(it.Key(), "|") - valPart := strings.Split(val, "|") - if len(keyPart) < 5 || len(valPart) < 3 { - continue - } - attr := urld(valPart[1]) - if attrFilter != "" && attr != attrFilter { - continue - } - claimRef, ok := blob.Parse(keyPart[4]) + cl, ok := kvClaim(it.Key(), val) if !ok { continue } - date, _ := time.Parse(time.RFC3339, keyPart[3]) - dst = append(dst, camtypes.Claim{ - BlobRef: claimRef, - Signer: signerFilter, - Permanode: permaNode, - Date: date, - Type: urld(valPart[0]), - Attr: attr, - Value: urld(valPart[2]), - }) + if attrFilter != "" && cl.Attr != attrFilter { + continue + } + if signerFilter.Valid() && cl.Signer != signerFilter { + continue + } + dst = append(dst, cl) } return dst, nil } +func kvClaim(k, v string) (c camtypes.Claim, ok bool) { + // TODO(bradfitz): remove the strings.Split calls to reduce allocations. + keyPart := strings.Split(k, "|") + valPart := strings.Split(v, "|") + if len(keyPart) < 5 || len(valPart) < 4 { + return + } + signerRef, ok := blob.Parse(valPart[3]) + if !ok { + return + } + permaNode, ok := blob.Parse(keyPart[1]) + if !ok { + return + } + claimRef, ok := blob.Parse(keyPart[4]) + if !ok { + return + } + date, err := time.Parse(time.RFC3339, keyPart[3]) + if err != nil { + return + } + return camtypes.Claim{ + BlobRef: claimRef, + Signer: signerRef, + Permanode: permaNode, + Date: date, + Type: urld(valPart[0]), + Attr: urld(valPart[1]), + Value: urld(valPart[2]), + }, true +} + func (x *Index) GetBlobMeta(br blob.Ref) (camtypes.BlobMeta, error) { if x.corpus != nil { return x.corpus.GetBlobMeta(br) diff --git a/pkg/index/interface.go b/pkg/index/interface.go index 7c3dccac4..a4d35b9e6 100644 --- a/pkg/index/interface.go +++ b/pkg/index/interface.go @@ -27,6 +27,7 @@ type Interface interface { // they filter the return items to only claims made by the given signer // or claims about the given attribute, respectively. // Deleted claims are never returned. + // The items may be appended in any order. AppendClaims(dst []camtypes.Claim, permaNode blob.Ref, signerFilter blob.Ref, attrFilter string) ([]camtypes.Claim, error) diff --git a/pkg/index/keys.go b/pkg/index/keys.go index 6579796f1..5b3fafefb 100644 --- a/pkg/index/keys.go +++ b/pkg/index/keys.go @@ -24,7 +24,7 @@ import ( // requiredSchemaVersion is incremented every time // an index key type is added, changed, or removed. -const requiredSchemaVersion = 1 +const requiredSchemaVersion = 2 // type of key returns the identifier in k before the first ":" or "|". // (Originally we packed keys by hand and there are a mix of styles) @@ -159,6 +159,11 @@ var ( {"claimType", typeStr}, {"attr", typeStr}, {"value", typeStr}, + // And the signerRef, which seems redundant + // with the signer keyId in the jey, but the + // Claim struct needs this, and there's 1:m + // for keyId:blobRef, so: + {"signerRef", typeBlobRef}, }, } diff --git a/pkg/index/receive.go b/pkg/index/receive.go index 37b7dac4f..40101afbd 100644 --- a/pkg/index/receive.go +++ b/pkg/index/receive.go @@ -344,7 +344,7 @@ func (ix *Index) populateClaim(b *schema.Blob, mm mutationMap) error { mm.Set(recentKey, pnbr.String()) claimKey := keyPermanodeClaim.Key(pnbr, verifiedKeyId, claim.ClaimDateString(), br) - mm.Set(claimKey, keyPermanodeClaim.Val(claim.ClaimType(), attr, value)) + mm.Set(claimKey, keyPermanodeClaim.Val(claim.ClaimType(), attr, value, vr.CamliSigner)) if strings.HasPrefix(attr, "camliPath:") { targetRef, ok := blob.Parse(value) diff --git a/pkg/search/query_test.go b/pkg/search/query_test.go index 9d18ebe4f..04d7b985b 100644 --- a/pkg/search/query_test.go +++ b/pkg/search/query_test.go @@ -425,7 +425,7 @@ func TestQueryPermanodeAttrValueMatches(t *testing.T) { // find permanodes matching a certain file query func TestQueryFileConstraint(t *testing.T) { testQuery(t, testQueryFileConstraint, indexClassic) } func TestQueryFileConstraint_Scan(t *testing.T) { - t.Skip("TODO: claims in memory") + t.Skip("TODO: fileinfo in memory") testQuery(t, testQueryFileConstraint, indexCorpusScan) } func TestQueryFileConstraint_Build(t *testing.T) { diff --git a/pkg/types/camtypes/search.go b/pkg/types/camtypes/search.go index 99b3b7d6f..d22e53f6c 100644 --- a/pkg/types/camtypes/search.go +++ b/pkg/types/camtypes/search.go @@ -38,9 +38,10 @@ func (a RecentPermanode) Equal(b RecentPermanode) bool { a.LastModTime.Equal(b.LastModTime) } -// TODO: document/decide how to represent "multi" claims here. One Claim each? Add Multi in here? -// Move/merge this in with the schema package? type Claim struct { + // TODO: document/decide how to represent "multi" claims here. One Claim each? Add Multi in here? + // Move/merge this in with the schema package? + BlobRef, Signer, Permanode blob.Ref Date time.Time