diff --git a/pkg/search/describe.go b/pkg/search/describe.go index 4ff4f3912..8551f623e 100644 --- a/pkg/search/describe.go +++ b/pkg/search/describe.go @@ -51,7 +51,20 @@ func (sh *Handler) serveDescribe(rw http.ResponseWriter, req *http.Request) { httputil.ReturnJSON(rw, res) } -func (sh *Handler) Describe(dr *DescribeRequest) (*DescribeResponse, error) { +const verboseDescribe = false + +func (sh *Handler) Describe(dr *DescribeRequest) (dres *DescribeResponse, err error) { + if verboseDescribe { + t0 := time.Now() + defer func() { + td := time.Since(t0) + var num int + if dres != nil { + num = len(dres.Meta) + } + log.Printf("Described %d blobs in %v", num, td) + }() + } sh.initDescribeRequest(dr) if dr.BlobRef.Valid() { dr.Describe(dr.BlobRef, dr.depth()) @@ -105,11 +118,13 @@ type DescribeRequest struct { // Internal details, used while loading. // Initialized by sh.initDescribeRequest. - sh *Handler - mu sync.Mutex // protects following: - m MetaMap - done map[blobrefAndDepth]bool // blobref -> true - errs map[string]error // blobref -> error + sh *Handler + mu sync.Mutex // protects following: + m MetaMap + done map[blobrefAndDepth]bool // blobref -> true + errs map[string]error // blobref -> error + resFromRule map[*DescribeRule]map[blob.Ref]bool + flatRuleCache []*DescribeRule // flattened once, by flatRules wg *sync.WaitGroup // for load requests } @@ -119,6 +134,25 @@ type blobrefAndDepth struct { depth int } +// Requires dr.mu is held +func (dr *DescribeRequest) flatRules() []*DescribeRule { + if dr.flatRuleCache == nil { + dr.flatRuleCache = make([]*DescribeRule, 0) + for _, rule := range dr.Rules { + rule.appendToFlatCache(dr) + } + } + return dr.flatRuleCache +} + +func (r *DescribeRule) appendToFlatCache(dr *DescribeRequest) { + dr.flatRuleCache = append(dr.flatRuleCache, r) + for _, rchild := range r.Rules { + rchild.parentRule = r + rchild.appendToFlatCache(dr) + } +} + // Requires dr.mu is held. func (dr *DescribeRequest) foreachResultBlob(fn func(blob.Ref)) { if dr.BlobRef.Valid() { @@ -163,6 +197,11 @@ type DescribeRule struct { // is if the value ends in "*", which matches prefixes // (e.g. "camliPath:*" or "*"). Attrs []string `json:"attrs,omitempty"` + + // Additional rules to run on the described results of Attrs. + Rules []*DescribeRule `json:"rules,omitempty"` + + parentRule *DescribeRule } // DescribeResponse is the JSON response from $searchRoot/camli/search/describe. @@ -665,6 +704,11 @@ func (r *DescribeRule) newMatches(br blob.Ref, dr *DescribeRequest) (brs []blob. return nil } } + if r.parentRule != nil { + if _, ok := dr.resFromRule[r.parentRule][br]; !ok { + return nil + } + } db, ok := dr.m[br.String()] if !ok || db.Permanode == nil { return nil @@ -699,18 +743,33 @@ func (r *DescribeRule) newMatches(br blob.Ref, dr *DescribeRequest) (brs []blob. return brs } +// dr.mu just be locked. +func (dr *DescribeRequest) noteResultFromRule(rule *DescribeRule, br blob.Ref) { + if dr.resFromRule == nil { + dr.resFromRule = make(map[*DescribeRule]map[blob.Ref]bool) + } + m, ok := dr.resFromRule[rule] + if !ok { + m = make(map[blob.Ref]bool) + dr.resFromRule[rule] = m + } + m[br] = true +} + func (dr *DescribeRequest) expandRules() error { loop := true + for loop { loop = false dr.wg.Wait() dr.mu.Lock() len0 := len(dr.m) var new []blob.Ref - for _, rule := range dr.Rules { + for _, rule := range dr.flatRules() { dr.foreachResultBlob(func(br blob.Ref) { for _, nbr := range rule.newMatches(br, dr) { new = append(new, nbr) + dr.noteResultFromRule(rule, nbr) } }) } diff --git a/pkg/search/describe_test.go b/pkg/search/describe_test.go index bce5333ad..60578e3c8 100644 --- a/pkg/search/describe_test.go +++ b/pkg/search/describe_test.go @@ -56,6 +56,35 @@ func searchDescribeSetup(fi *test.FakeIndex) index.Interface { addPermanode(fi, "abc-8881", "name", "leaf8881", ) + + addPermanode(fi, "fourcheckin-0", + "camliNodeType", "foursquare.com:checkin", + "foursquareVenuePermanode", "fourvenue-123", + ) + addPermanode(fi, "fourvenue-123", + "camliNodeType", "foursquare.com:venue", + "camliPath:photos", "venuepicset-123", + ) + addPermanode(fi, "venuepicset-123", + "camliPath:1.jpg", "venuepic-1", + ) + addPermanode(fi, "venuepic-1", + "camliContent", "somevenuepic-0", + ) + addPermanode(fi, "somevenuepic-0", + "foo", "bar", + ) + + addPermanode(fi, "homedir-0", + "camliPath:subdir.1", "homedir-1", + ) + addPermanode(fi, "homedir-1", + "camliPath:subdir.2", "homedir-2", + ) + addPermanode(fi, "homedir-2", + "foo", "bar", + ) + return fi } @@ -133,6 +162,50 @@ var searchDescribeTests = []handlerTest{ }), wantDescribed: []string{"abc-123", "abc-123c", "abc-123cc", "abc-888", "abc-8881"}, }, + + { + name: "foursquare venue photos, but not recursive camliPath explosion", + postBody: marshalJSON(&search.DescribeRequest{ + BlobRefs: []blob.Ref{ + blob.MustParse("homedir-0"), + blob.MustParse("fourcheckin-0"), + }, + Rules: []*search.DescribeRule{ + { + Attrs: []string{"camliContent", "camliContentImage"}, + }, + { + IfCamliNodeType: "foursquare.com:checkin", + Attrs: []string{"foursquareVenuePermanode"}, + }, + { + IfCamliNodeType: "foursquare.com:venue", + Attrs: []string{"camliPath:photos"}, + Rules: []*search.DescribeRule{ + { + Attrs: []string{"camliPath:*"}, + }, + }, + }, + }, + }), + wantDescribed: []string{"homedir-0", "fourcheckin-0", "fourvenue-123", "venuepicset-123", "venuepic-1", "somevenuepic-0"}, + }, + + { + name: "home dirs forever", + postBody: marshalJSON(&search.DescribeRequest{ + BlobRefs: []blob.Ref{ + blob.MustParse("homedir-0"), + }, + Rules: []*search.DescribeRule{ + { + Attrs: []string{"camliPath:*"}, + }, + }, + }), + wantDescribed: []string{"homedir-0", "homedir-1", "homedir-2"}, + }, } func init() {