Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge "index: improve Corpus attr lookup with signer filter"

This commit is contained in:
Mathieu Lonjaret 2016-10-26 20:50:12 +00:00 committed by Gerrit Code Review
parent 32359276ff 0f55bfe980
commit 1a7452561a
2 changed files with 204 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
pkg/index/corpus.go
View File

@ -118,18 +118,44 @@ func (c *Corpus) IsDeleted(br blob.Ref) bool {
}
type PermanodeMeta struct {
// TODO: OwnerKeyId string
Claims []*camtypes.Claim // sorted by camtypes.ClaimsByDate
// latest attributes
Attrs map[string][]string
attr attrValues // attributes from all signers
signer map[blob.Ref]attrValues // attrs per signer
}
type attrValues map[string][]string
// cacheAttrClaim applies attribute changes from cl.
func (m attrValues) cacheAttrClaim(cl *camtypes.Claim) {
switch cl.Type {
case string(schema.SetAttributeClaim):
m[cl.Attr] = []string{cl.Value}
case string(schema.AddAttributeClaim):
m[cl.Attr] = append(m[cl.Attr], cl.Value)
case string(schema.DelAttributeClaim):
if cl.Value == "" {
delete(m, cl.Attr)
} else {
a, i := m[cl.Attr], 0
for _, v := range a {
if v != cl.Value {
a[i] = v
i++
}
}
m[cl.Attr] = a[:i]
}
}
}
// restoreInvariants sorts claims by date and
// recalculates latest attributes.
func (pm *PermanodeMeta) restoreInvariants() {
sort.Sort(camtypes.ClaimPtrsByDate(pm.Claims))
pm.Attrs = make(map[string][]string)
pm.attr = make(attrValues)
pm.signer = make(map[blob.Ref]attrValues)
for _, cl := range pm.Claims {
pm.appendAttrClaim(cl)
}
@ -139,7 +165,7 @@ func (pm *PermanodeMeta) restoreInvariants() {
// that the all but the last element in Claims are sorted by date
// and the last element is the only one not yet included in Attrs.
func (pm *PermanodeMeta) fixupLastClaim() {
if pm.Attrs != nil {
if pm.attr != nil {
n := len(pm.Claims)
if n < 2 || camtypes.ClaimPtrsByDate(pm.Claims).Less(n-2, n-1) {
// already sorted, update Attrs from new Claim
@ -150,41 +176,83 @@ func (pm *PermanodeMeta) fixupLastClaim() {
pm.restoreInvariants()
}
// appendAttrClaim stores permanode attributes
// from cl in pm.attr and pm.signer[cl.Signer].
// The caller of appendAttrClaim is responsible for calling
// it with claims sorted in camtypes.ClaimPtrsByDate order.
func (pm *PermanodeMeta) appendAttrClaim(cl *camtypes.Claim) {
switch cl.Type {
case string(schema.SetAttributeClaim):
pm.Attrs[cl.Attr] = []string{cl.Value}
case string(schema.AddAttributeClaim):
pm.Attrs[cl.Attr] = append(pm.Attrs[cl.Attr], cl.Value)
case string(schema.DelAttributeClaim):
if cl.Value == "" {
pm.Attrs[cl.Attr] = nil
} else {
a, i := pm.Attrs[cl.Attr], 0
for _, v := range a {
if v != cl.Value {
a[i] = v
i++
}
sc, ok := pm.signer[cl.Signer]
if !ok {
// Optimize for the case where cl.Signer of all claims are the same.
// Instead of having two identical attrValues copies in
// pm.attr and pm.signer[cl.Signer],
// use a single attrValues
// until there is at least a second signer.
switch len(pm.signer) {
case 0:
// Set up signer cache to reference
// the existing attrValues.
pm.attr.cacheAttrClaim(cl)
pm.signer[cl.Signer] = pm.attr
return
case 1:
// pm.signer has exactly one other signer,
// and its attrValues entry references pm.attr.
// Make a copy of pm.attr
// for this other signer now.
m := make(attrValues)
for a, v := range pm.attr {
xv := make([]string, len(v))
copy(xv, v)
m[a] = xv
}
for sig := range pm.signer {
pm.signer[sig] = m
break
}
pm.Attrs[cl.Attr] = a[:i]
}
sc = make(attrValues)
pm.signer[cl.Signer] = sc
}
pm.attr.cacheAttrClaim(cl)
// Cache claim in sc only if sc != pm.attr.
if len(pm.signer) > 1 {
sc.cacheAttrClaim(cl)
}
}
// canUseAttrs reports whether pm.Attrs can be used
// to query values for the permanode attributes.
func (pm *PermanodeMeta) canUseAttrs(at time.Time) bool {
if pm.Attrs == nil {
return false
// valuesAtSigner returns an attrValues to query
// permanode attr values at the given time for the signerFilter.
// It returns ok == true if v represents attrValues
// valid for the specified parameters.
// If signerFilter is valid and pm has no attributes for it,
// (nil, true) is returned.
func (pm *PermanodeMeta) valuesAtSigner(at time.Time,
signerFilter blob.Ref) (v attrValues, ok bool) {
if pm.attr == nil {
return nil, false
}
var m attrValues
if signerFilter.Valid() {
m = pm.signer[signerFilter]
if m == nil {
return nil, true
}
} else {
m = pm.attr
}
if at.IsZero() {
return true
return m, true
}
if n := len(pm.Claims); n == 0 || !pm.Claims[n-1].Date.After(at) {
return true
return m, true
}
return false
return nil, false
}
func newCorpus() *Corpus {
@ -1008,12 +1076,12 @@ func (c *Corpus) PermanodeAttrValue(permaNode blob.Ref,
if !ok {
return ""
}
if !signerFilter.Valid() && pm.canUseAttrs(at) {
v := pm.Attrs[attr]
if len(v) != 0 {
return v[0]
if values, ok := pm.valuesAtSigner(at, signerFilter); ok {
v := values[attr]
if len(v) == 0 {
return ""
}
return ""
return v[0]
}
return claimPtrsAttrValue(pm.Claims, attr, at, signerFilter)
}
@ -1034,8 +1102,8 @@ func (c *Corpus) AppendPermanodeAttrValues(dst []string,
if !ok {
return dst
}
if !signerFilter.Valid() && pm.canUseAttrs(at) {
return append(dst, pm.Attrs[attr]...)
if values, ok := pm.valuesAtSigner(at, signerFilter); ok {
return append(dst, values[attr]...)
}
if at.IsZero() {
at = time.Now()
@ -1211,8 +1279,8 @@ func (c *Corpus) PermanodeHasAttrValue(pn blob.Ref, at time.Time, attr, val stri
if !ok {
return false
}
if pm.canUseAttrs(at) {
for _, v := range pm.Attrs[attr] {
if values, ok := pm.valuesAtSigner(at, blob.Ref{}); ok {
for _, v := range values[attr] {
if v == val {
return true
}

143
pkg/index/corpus_test.go
View File

@ -30,80 +30,119 @@ import (
"golang.org/x/net/context"
)
func newTestCorpusWithPermanode() (*index.Corpus, blob.Ref) {
c := index.ExpNewCorpus()
pn := blob.MustParse("abc-123")
func newTestCorpusWithPermanode() (c *index.Corpus, pn, sig1, sig2 blob.Ref) {
c = index.ExpNewCorpus()
pn = blob.MustParse("abc-123")
sig1 = blob.MustParse("abc-456")
sig2 = blob.MustParse("abc-789")
tm := time.Unix(99, 0)
claim := func(verb, attr, val string) *camtypes.Claim {
claim := func(verb, attr, val string, sig blob.Ref) *camtypes.Claim {
tm = tm.Add(time.Second)
return &camtypes.Claim{
Type: verb + "-attribute",
Attr: attr,
Value: val,
Date: tm,
Type: verb + "-attribute",
Attr: attr,
Value: val,
Date: tm,
Signer: sig,
}
}
c.SetClaims(pn, []*camtypes.Claim{
claim("set", "foo", "foov"), // time 100
claim("set", "foo", "foov", sig1), // time 100
claim("add", "tag", "a"), // time 101
claim("add", "tag", "b"), // time 102
claim("del", "tag", ""),
claim("add", "tag", "c"),
claim("add", "tag", "d"),
claim("add", "tag", "e"),
claim("del", "tag", "d"),
claim("add", "tag", "a", sig1), // time 101
claim("add", "tag", "b", sig1), // time 102
claim("del", "tag", "", sig1),
claim("add", "tag", "c", sig1),
claim("add", "tag", "d", sig2),
claim("add", "tag", "e", sig1),
claim("del", "tag", "d", sig2),
claim("add", "tag", "f", sig2),
claim("add", "DelAll", "a"),
claim("add", "DelAll", "b"),
claim("add", "DelAll", "c"),
claim("del", "DelAll", ""),
claim("add", "DelAll", "a", sig1),
claim("add", "DelAll", "b", sig1),
claim("add", "DelAll", "c", sig2),
claim("del", "DelAll", "", sig1),
claim("add", "DelOne", "a"),
claim("add", "DelOne", "b"),
claim("add", "DelOne", "c"),
claim("add", "DelOne", "d"),
claim("del", "DelOne", "d"),
claim("del", "DelOne", "a"),
claim("add", "DelOne", "a", sig1),
claim("add", "DelOne", "b", sig1),
claim("add", "DelOne", "c", sig1),
claim("add", "DelOne", "d", sig2),
claim("add", "DelOne", "e", sig2),
claim("del", "DelOne", "d", sig2),
claim("del", "DelOne", "a", sig1),
claim("add", "SetAfterAdd", "a"),
claim("add", "SetAfterAdd", "b"),
claim("set", "SetAfterAdd", "setv"),
claim("add", "SetAfterAdd", "a", sig1),
claim("add", "SetAfterAdd", "b", sig1),
claim("add", "SetAfterAdd", "c", sig2),
claim("set", "SetAfterAdd", "setv", sig1),
// add an element with fixed time to test
// The claims below help testing
// slow and fast path equivalence
// (lookups based on pm.Claims and pm.Attrs, respectively)
// (lookups based on pm.Claims and cached attrs).
//
// Permanode attr queries at time.Time{} and
// time.Unix(200, 0) should yield the same results
// for the above claims. The difference is that
// they use the cache at time.Time{} and
// the pm.Claims directly (bypassing the cache)
// at time.Unix(200, 0).
{
Type: "set-attribute",
Attr: "CacheTest",
Value: "foo",
Date: time.Unix(201, 0),
Type: "set-attribute",
Attr: "CacheTest",
Value: "foo",
Date: time.Unix(201, 0),
Signer: sig1,
},
{
Type: "set-attribute",
Attr: "CacheTest",
Value: "foo",
Date: time.Unix(202, 0),
Signer: sig2,
},
})
return c, pn
return c, pn, sig1, sig2
}
func TestCorpusAppendPermanodeAttrValues(t *testing.T) {
c, pn := newTestCorpusWithPermanode()
c, pn, sig1, sig2 := newTestCorpusWithPermanode()
s := func(s ...string) []string { return s }
sigMissing := blob.MustParse("xyz-123")
tests := []struct {
attr string
want []string
t time.Time
sig blob.Ref
}{
{attr: "not-exist", want: s()},
{attr: "DelAll", want: s()},
{attr: "DelOne", want: s("b", "c")},
{attr: "DelOne", want: s("b", "c", "e")},
{attr: "foo", want: s("foov")},
{attr: "tag", want: s("c", "e")},
{attr: "tag", want: s("c", "e", "f")},
{attr: "tag", want: s("a", "b"), t: time.Unix(102, 0)},
{attr: "SetAfterAdd", want: s("setv")},
// sig1
{attr: "not-exist", want: s(), sig: sig1},
{attr: "DelAll", want: s(), sig: sig1},
{attr: "DelOne", want: s("b", "c"), sig: sig1},
{attr: "foo", want: s("foov"), sig: sig1},
{attr: "tag", want: s("c", "e"), sig: sig1},
{attr: "tag", want: s("a", "b"), t: time.Unix(102, 0), sig: sig1},
{attr: "SetAfterAdd", want: s("setv"), sig: sig1},
// sig2
{attr: "DelAll", want: s("c"), sig: sig2},
{attr: "DelOne", want: s("e"), sig: sig2},
{attr: "tag", want: s("d"), t: time.Unix(105, 0), sig: sig2},
{attr: "SetAfterAdd", want: s("c"), sig: sig2},
// sigMissing (not present in pn)
{attr: "tag", want: s(), sig: sigMissing},
}
for i, tt := range tests {
got := c.AppendPermanodeAttrValues(nil, pn, tt.attr, tt.t, blob.Ref{})
got := c.AppendPermanodeAttrValues(nil, pn, tt.attr, tt.t, tt.sig)
if len(got) == 0 && len(tt.want) == 0 {
continue
}
@ -116,7 +155,7 @@ func TestCorpusAppendPermanodeAttrValues(t *testing.T) {
// skip equivalence test if specific time was given
continue
}
got = c.AppendPermanodeAttrValues(nil, pn, tt.attr, time.Unix(200, 0), blob.Ref{})
got = c.AppendPermanodeAttrValues(nil, pn, tt.attr, time.Unix(200, 0), tt.sig)
if len(got) == 0 && len(tt.want) == 0 {
continue
}
@ -128,12 +167,13 @@ func TestCorpusAppendPermanodeAttrValues(t *testing.T) {
}
func TestCorpusPermanodeAttrValue(t *testing.T) {
c, pn := newTestCorpusWithPermanode()
c, pn, sig1, sig2 := newTestCorpusWithPermanode()
tests := []struct {
attr string
want string
t time.Time
sig blob.Ref
}{
{attr: "not-exist", want: ""},
{attr: "DelAll", want: ""},
@ -142,9 +182,22 @@ func TestCorpusPermanodeAttrValue(t *testing.T) {
{attr: "tag", want: "c"},
{attr: "tag", want: "a", t: time.Unix(102, 0)},
{attr: "SetAfterAdd", want: "setv"},
// sig1
{attr: "not-exist", want: "", sig: sig1},
{attr: "DelAll", want: "", sig: sig1},
{attr: "DelOne", want: "b", sig: sig1},
{attr: "foo", want: "foov", sig: sig1},
{attr: "tag", want: "c", sig: sig1},
{attr: "SetAfterAdd", want: "setv", sig: sig1},
// sig2
{attr: "DelAll", want: "c", sig: sig2},
{attr: "DelOne", want: "e", sig: sig2},
{attr: "foo", want: "", sig: sig2},
{attr: "tag", want: "f", sig: sig2},
{attr: "SetAfterAdd", want: "c", sig: sig2},
}
for i, tt := range tests {
got := c.PermanodeAttrValue(pn, tt.attr, tt.t, blob.Ref{})
got := c.PermanodeAttrValue(pn, tt.attr, tt.t, tt.sig)
if len(got) == 0 && len(tt.want) == 0 {
continue
}
@ -157,7 +210,7 @@ func TestCorpusPermanodeAttrValue(t *testing.T) {
// skip equivalence test if specific time was given
continue
}
got = c.PermanodeAttrValue(pn, tt.attr, time.Unix(200, 0), blob.Ref{})
got = c.PermanodeAttrValue(pn, tt.attr, time.Unix(200, 0), tt.sig)
if len(got) == 0 && len(tt.want) == 0 {
continue
}
@ -169,7 +222,7 @@ func TestCorpusPermanodeAttrValue(t *testing.T) {
}
func TestCorpusPermanodeHasAttrValue(t *testing.T) {
c, pn := newTestCorpusWithPermanode()
c, pn, _, _ := newTestCorpusWithPermanode()
tests := []struct {
attr string