diff --git a/pkg/index/corpus.go b/pkg/index/corpus.go index 37b71fca6..da8c2e8f5 100644 --- a/pkg/index/corpus.go +++ b/pkg/index/corpus.go @@ -81,6 +81,10 @@ type Corpus struct { imageInfo map[blob.Ref]camtypes.ImageInfo // keyed by fileref (not wholeref) fileWholeRef map[blob.Ref]blob.Ref // fileref -> its wholeref (TODO: multi-valued?) gps map[blob.Ref]latLong // wholeRef -> GPS coordinates + // dirChildren maps a directory to its (direct) children (static-set entries). + dirChildren map[blob.Ref]map[blob.Ref]struct{} + // fileParents maps a file or directory to its (direct) parents. + fileParents map[blob.Ref]map[blob.Ref]struct{} // Lack of edge tracking implementation is issue #707 // (https://github.com/perkeep/perkeep/issues/707) @@ -336,6 +340,8 @@ func newCorpus() *Corpus { deletes: make(map[blob.Ref][]deletion), claimBack: make(map[blob.Ref][]*camtypes.Claim), permanodesSetByNodeType: make(map[string]map[blob.Ref]bool), + dirChildren: make(map[blob.Ref]map[blob.Ref]struct{}), + fileParents: make(map[blob.Ref]map[blob.Ref]struct{}), } c.permanodesByModtime = &lazySortedPermanodes{ c: c, @@ -383,19 +389,20 @@ func (crashStorage) Find(start, end string) sorted.Iterator { // *********** Updating the corpus var corpusMergeFunc = map[string]func(c *Corpus, k, v []byte) error{ - "have": nil, // redundant with "meta" - "recpn": nil, // unneeded. - "meta": (*Corpus).mergeMetaRow, - keySignerKeyID.name: (*Corpus).mergeSignerKeyIdRow, - "claim": (*Corpus).mergeClaimRow, - "fileinfo": (*Corpus).mergeFileInfoRow, - keyFileTimes.name: (*Corpus).mergeFileTimesRow, - "imagesize": (*Corpus).mergeImageSizeRow, - "wholetofile": (*Corpus).mergeWholeToFileRow, - "exifgps": (*Corpus).mergeEXIFGPSRow, - "exiftag": nil, // not using any for now - "signerattrvalue": nil, // ignoring for now - "mediatag": (*Corpus).mergeMediaTag, + "have": nil, // redundant with "meta" + "recpn": nil, // unneeded. + "meta": (*Corpus).mergeMetaRow, + keySignerKeyID.name: (*Corpus).mergeSignerKeyIdRow, + "claim": (*Corpus).mergeClaimRow, + "fileinfo": (*Corpus).mergeFileInfoRow, + keyFileTimes.name: (*Corpus).mergeFileTimesRow, + "imagesize": (*Corpus).mergeImageSizeRow, + "wholetofile": (*Corpus).mergeWholeToFileRow, + "exifgps": (*Corpus).mergeEXIFGPSRow, + "exiftag": nil, // not using any for now + "signerattrvalue": nil, // ignoring for now + "mediatag": (*Corpus).mergeMediaTag, + keyStaticDirChild.name: (*Corpus).mergeStaticDirChildRow, } func memstats() *runtime.MemStats { @@ -420,6 +427,7 @@ var slurpPrefixes = []string{ "wholetofile|", "exifgps|", "mediatag|", + keyStaticDirChild.name + "|", } // Key types (without trailing punctuation) that we slurp to memory at start. @@ -768,6 +776,39 @@ func (c *Corpus) mergeFileInfoRow(k, v []byte) error { return nil } +func (c *Corpus) mergeStaticDirChildRow(k, v []byte) error { + // dirchild|sha1-dir|sha1-child" "1" + // strip the key name + sk := k[len(keyStaticDirChild.name)+1:] + pipe := bytes.IndexByte(sk, '|') + if pipe < 0 { + return fmt.Errorf("invalid dirchild key %q, missing second pipe", k) + } + parent, ok := blob.ParseBytes(sk[:pipe]) + if !ok { + return fmt.Errorf("invalid dirchild parent blobref in key %q", k) + } + child, ok := blob.ParseBytes(sk[pipe+1:]) + if !ok { + return fmt.Errorf("invalid dirchild child blobref in key %q", k) + } + parent = c.br(parent) + child = c.br(child) + children, ok := c.dirChildren[parent] + if !ok { + children = make(map[blob.Ref]struct{}) + } + children[child] = struct{}{} + c.dirChildren[parent] = children + parents, ok := c.fileParents[child] + if !ok { + parents = make(map[blob.Ref]struct{}) + } + parents[parent] = struct{}{} + c.fileParents[child] = parents + return nil +} + func (c *Corpus) mergeFileTimesRow(k, v []byte) error { if len(v) == 0 { return nil @@ -1383,6 +1424,24 @@ func (c *Corpus) GetFileInfo(ctx context.Context, fileRef blob.Ref) (fi camtypes return } +// GetDirChildren returns the direct children (static-set entries) of the directory dirRef. +func (c *Corpus) GetDirChildren(ctx context.Context, dirRef blob.Ref) (map[blob.Ref]struct{}, error) { + children, ok := c.dirChildren[dirRef] + if !ok { + return nil, os.ErrNotExist + } + return children, nil +} + +// GetParentDirs returns the direct parents (directories) of the file or directory childRef. +func (c *Corpus) GetParentDirs(ctx context.Context, childRef blob.Ref) (map[blob.Ref]struct{}, error) { + parents, ok := c.fileParents[childRef] + if !ok { + return nil, os.ErrNotExist + } + return parents, nil +} + func (c *Corpus) GetImageInfo(ctx context.Context, fileRef blob.Ref) (ii camtypes.ImageInfo, err error) { ii, ok := c.imageInfo[fileRef] if !ok { diff --git a/pkg/index/index.go b/pkg/index/index.go index 86d794010..54bc84c19 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -1565,10 +1565,25 @@ func kvEdgeBackward(k, v string) (edge *camtypes.Edge, ok bool) { } // GetDirMembers sends on dest the children of the static directory dir. -func (x *Index) GetDirMembers(dir blob.Ref, dest chan<- blob.Ref, limit int) (err error) { +func (x *Index) GetDirMembers(ctx context.Context, dir blob.Ref, dest chan<- blob.Ref, limit int) (err error) { defer close(dest) sent := 0 + if x.corpus != nil { + children, err := x.corpus.GetDirChildren(ctx, dir) + if err != nil { + return err + } + for child := range children { + dest <- child + sent++ + if sent == limit { + break + } + } + return nil + } + it := x.queryPrefix(keyStaticDirChild, dir.String()) defer closeIterator(it, &err) for it.Next() { diff --git a/pkg/index/indextest/tests.go b/pkg/index/indextest/tests.go index 1d4af20fd..73876971a 100644 --- a/pkg/index/indextest/tests.go +++ b/pkg/index/indextest/tests.go @@ -617,7 +617,7 @@ func Index(t *testing.T, initIdx func() *index.Index) { // GetDirMembers { ch := make(chan blob.Ref, 10) // expect 2 results - err := id.Index.GetDirMembers(imagesDirRef, ch, 50) + err := id.Index.GetDirMembers(ctx, imagesDirRef, ch, 50) if err != nil { t.Fatalf("GetDirMembers = %v", err) } diff --git a/pkg/index/interface.go b/pkg/index/interface.go index e83570d60..5539e309c 100644 --- a/pkg/index/interface.go +++ b/pkg/index/interface.go @@ -103,7 +103,7 @@ type Interface interface { // is nil. // dest must be closed, even when returning an error. // limit <= 0 means unlimited. - GetDirMembers(dirRef blob.Ref, dest chan<- blob.Ref, limit int) error + GetDirMembers(ctx context.Context, dirRef blob.Ref, dest chan<- blob.Ref, limit int) error // Given an owner key, a camliType 'claim', 'attribute' name, // and specific 'value', find the most recent permanode that has diff --git a/pkg/search/describe.go b/pkg/search/describe.go index 983745167..271fa0c4b 100644 --- a/pkg/search/describe.go +++ b/pkg/search/describe.go @@ -119,7 +119,7 @@ type DescribeRequest struct { // Rules specifies a set of rules to instruct how to keep // expanding the described set. All rules are tested and - // matching rules grow the the response set until all rules no + // matching rules grow the response set until all rules no // longer match or internal limits are hit. Rules []*DescribeRule `json:"rules,omitempty"` @@ -883,7 +883,7 @@ func (dr *DescribeRequest) getDirMembers(ctx context.Context, br blob.Ref, depth ch := make(chan blob.Ref) errch := make(chan error) go func() { - errch <- dr.sh.index.GetDirMembers(br, ch, limit) + errch <- dr.sh.index.GetDirMembers(ctx, br, ch, limit) }() var members []blob.Ref diff --git a/pkg/search/query.go b/pkg/search/query.go index 2e60d735d..6bcf754ba 100644 --- a/pkg/search/query.go +++ b/pkg/search/query.go @@ -426,6 +426,10 @@ type FileConstraint struct { // every known hash algorithm. WholeRef blob.Ref `json:"wholeRef,omitempty"` + // ParentDir, if non-nil, constrains the file match based on properties + // of its parent directory. + ParentDir *DirConstraint `json:"parentDir,omitempty"` + // For images: IsImage bool `json:"isImage,omitempty"` EXIF *EXIFConstraint `json:"exif,omitempty"` // TODO: implement @@ -447,22 +451,35 @@ type MediaTagConstraint struct { Int *IntConstraint `json:"int,omitempty"` } +// DirConstraint matches static directories. type DirConstraint struct { // (All non-zero fields must match) - // TODO: implement. mostly need more things in the index. + FileName *StringConstraint `json:"fileName,omitempty"` + BlobRefPrefix string `json:"blobRefPrefix,omitempty"` - FileName *StringConstraint `json:"fileName,omitempty"` + // ParentDir, if non-nil, constrains the directory match based on properties + // of its parent directory. + ParentDir *DirConstraint `json:"parentDir,omitempty"` - TopFileSize, // not recursive - TopFileCount, // not recursive - FileSize, - FileCount *IntConstraint + // TODO: implement. + // FileCount *IntConstraint + // FileSize *IntConstraint - // TODO: these would need thought on how to index efficiently: - // (Also: top-only variants?) - // ContainsFile *FileConstraint - // ContainsDir *DirConstraint + // TopFileCount, if non-nil, constrains the directory match with the directory's + // number of children (non-recursively). + TopFileCount *IntConstraint `json:"topFileCount,omitempty"` + + // RecursiveContains, if non-nil, is like Contains, but applied to all + // the descendants of the directory. It is mutually exclusive with Contains. + RecursiveContains *Constraint `json:"recursiveContains,omitempty"` + + // Contains, if non-nil, constrains the directory match to just those + // directories containing a file matched by Contains. Contains should have a + // BlobPrefix, or a *FileConstraint, or a *DirConstraint, or a *LogicalConstraint + // combination of the aforementioned. It is only applied to the children of the + // directory, in a non-recursive manner. It is mutually exclusive with RecursiveContains. + Contains *Constraint `json:"contains,omitempty"` } // An IntConstraint specifies constraints on an integer. @@ -919,6 +936,34 @@ func (s *search) fileInfo(ctx context.Context, br blob.Ref) (camtypes.FileInfo, return s.h.index.GetFileInfo(ctx, br) } +func (s *search) dirChildren(ctx context.Context, br blob.Ref) (map[blob.Ref]struct{}, error) { + if c := s.h.corpus; c != nil { + return c.GetDirChildren(ctx, br) + } + + ch := make(chan blob.Ref) + errch := make(chan error) + go func() { + errch <- s.h.index.GetDirMembers(ctx, br, ch, s.q.Limit) + }() + children := make(map[blob.Ref]struct{}) + for child := range ch { + children[child] = struct{}{} + } + if err := <-errch; err != nil { + return nil, err + } + return children, nil +} + +func (s *search) parentDirs(ctx context.Context, br blob.Ref) (map[blob.Ref]struct{}, error) { + c := s.h.corpus + if c == nil { + return nil, errors.New("parent directory search not supported without a corpus") + } + return c.GetParentDirs(ctx, br) +} + // optimizePlan returns an optimized version of c which will hopefully // execute faster than executing c literally. func optimizePlan(c *Constraint) *Constraint { @@ -1869,6 +1914,36 @@ func (c *FileConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref return false, nil } } + if pc := c.ParentDir; pc != nil { + parents, err := s.parentDirs(ctx, br) + if err == os.ErrNotExist { + return false, nil + } + if err != nil { + return false, err + } + matches := false + for parent, _ := range parents { + meta, err := s.blobMeta(ctx, parent) + if err != nil { + if os.IsNotExist(err) { + continue + } + return false, err + } + ok, err := pc.blobMatches(ctx, s, parent, meta) + if err != nil { + return false, err + } + if ok { + matches = true + break + } + } + if !matches { + return false, nil + } + } corpus := s.h.corpus if c.WholeRef.Valid() { if corpus == nil { @@ -1976,16 +2051,176 @@ func (c *TimeConstraint) timeMatches(t time.Time) bool { } func (c *DirConstraint) checkValid() error { + if c == nil { + return nil + } + if c.Contains != nil && c.RecursiveContains != nil { + return errors.New("Contains and RecursiveContains in a DirConstraint are mutually exclusive") + } return nil } +func (c *Constraint) isFileOrDirConstraint() bool { + if l := c.Logical; l != nil { + return l.A.isFileOrDirConstraint() && l.B.isFileOrDirConstraint() + } + return c.File != nil || c.Dir != nil +} + +func (c *Constraint) fileOrDirOrLogicalMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) { + if cf := c.File; cf != nil { + return cf.blobMatches(ctx, s, br, bm) + } + if cd := c.Dir; cd != nil { + return cd.blobMatches(ctx, s, br, bm) + } + if l := c.Logical; l != nil { + return l.matcher()(ctx, s, br, bm) + } + return false, nil +} + func (c *DirConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) { if bm.CamliType != "directory" { return false, nil } + // TODO(mpl): I've added c.BlobRefPrefix, so that c.ParentDir can be directly + // matched against a blobRef (instead of e.g. a filename), but I could instead make + // ParentDir be a *Constraint, and logically enforce that it has to "be equivalent" + // to a ParentDir matching or a BlobRefPrefix matching. I think this here below is + // simpler, but not sure it's best in the long run. + if pfx := c.BlobRefPrefix; pfx != "" { + if !br.HasPrefix(pfx) { + return false, nil + } + } + fi, err := s.fileInfo(ctx, br) + if err == os.ErrNotExist { + return false, nil + } + if err != nil { + return false, err + } + if sc := c.FileName; sc != nil && !sc.stringMatches(fi.FileName) { + return false, nil + } + if pc := c.ParentDir; pc != nil { + parents, err := s.parentDirs(ctx, br) + if err == os.ErrNotExist { + return false, nil + } + if err != nil { + return false, err + } + isMatch, err := pc.hasMatchingParent(ctx, s, parents) + if err != nil { + return false, err + } + if !isMatch { + return false, nil + } + } - // TODO: implement - panic("TODO: implement DirConstraint.blobMatches") + // All constraints not pertaining to children must happen above + // this point. + children, err := s.dirChildren(ctx, br) + if err != nil && err != os.ErrNotExist { + return false, err + } + if fc := c.TopFileCount; fc != nil && !fc.intMatches(int64(len(children))) { + return false, nil + } + cc := c.Contains + recursive := false + if cc == nil { + if crc := c.RecursiveContains; crc != nil { + recursive = true + // RecursiveContains implies Contains + cc = crc + } + } + // First test on the direct children + containsMatch := false + if cc != nil { + // Allow directly specifying the fileRef + if cc.BlobRefPrefix != "" { + containsMatch, err = c.hasMatchingChild(ctx, s, children, func(ctx context.Context, s *search, child blob.Ref, bm camtypes.BlobMeta) (bool, error) { + return child.HasPrefix(cc.BlobRefPrefix), nil + }) + } else { + if !cc.isFileOrDirConstraint() { + return false, errors.New("[Recursive]Contains constraint should have a *FileConstraint, or a *DirConstraint, or a *LogicalConstraint combination of the aforementioned.") + } + containsMatch, err = c.hasMatchingChild(ctx, s, children, cc.fileOrDirOrLogicalMatches) + } + if err != nil { + return false, err + } + if !containsMatch && !recursive { + return false, nil + } + } + // Then if needed recurse on the next generation descendants. + if !containsMatch && recursive { + match, err := c.hasMatchingChild(ctx, s, children, c.blobMatches) + if err != nil { + return false, err + } + if !match { + return false, nil + } + } + + // TODO: implement FileCount and FileSize. + + return true, nil +} + +// hasMatchingParent checks all parents against c and returns true as soon as one of +// them matches, or returns false if none of them is a match. +func (c *DirConstraint) hasMatchingParent(ctx context.Context, s *search, parents map[blob.Ref]struct{}) (bool, error) { + for parent := range parents { + meta, err := s.blobMeta(ctx, parent) + if err != nil { + if os.IsNotExist(err) { + continue + } + return false, err + } + ok, err := c.blobMatches(ctx, s, parent, meta) + if err != nil { + return false, err + } + if ok { + return true, nil + } + } + return false, nil +} + +// hasMatchingChild runs matcher against each child and returns true as soon as +// there is a match, of false if none of them is a match. +func (c *DirConstraint) hasMatchingChild(ctx context.Context, s *search, children map[blob.Ref]struct{}, + matcher func(context.Context, *search, blob.Ref, camtypes.BlobMeta) (bool, error)) (bool, error) { + // TODO(mpl): See if we're guaranteed to be CPU-bound (i.e. all resources are in + // corpus), and if not, add some concurrency to spread costly index lookups. + for child, _ := range children { + meta, err := s.blobMeta(ctx, child) + if err != nil { + if os.IsNotExist(err) { + continue + } + return false, err + } + ok, err := matcher(ctx, s, child, meta) + if err != nil { + return false, err + } + if ok { + return true, nil + } + } + return false, nil } type sortSearchResultBlobs struct { diff --git a/pkg/search/query_test.go b/pkg/search/query_test.go index 2d5d2a530..5be69ce97 100644 --- a/pkg/search/query_test.go +++ b/pkg/search/query_test.go @@ -724,6 +724,264 @@ func TestQueryFileConstraint(t *testing.T) { }) } +// find a directory with a name +func TestQueryDirConstraint(t *testing.T) { + testQuery(t, func(qt *queryTest) { + id := qt.id + dirRef := id.UploadDir("somedir", []blob.Ref{}, time.Unix(789, 0)) + qt.t.Logf("dirRef = %q", dirRef) + p1 := id.NewPlannedPermanode("1") + id.SetAttribute(p1, "camliContent", dirRef.String()) + + fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0)) + qt.t.Logf("fileRef3 = %q", fileRef3) + p2 := id.NewPlannedPermanode("2") + id.SetAttribute(p2, "camliContent", fileRef3.String()) + + sq := &SearchQuery{ + Constraint: &Constraint{ + Dir: &DirConstraint{ + FileName: &StringConstraint{ + Contains: "somedir", + }, + }, + }, + } + qt.wantRes(sq, dirRef) + }) +} + +// find permanode with a dir that contains a certain file +func TestQueryDirWithFileConstraint(t *testing.T) { + testQuery(t, func(qt *queryTest) { + id := qt.id + fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0)) + qt.t.Logf("fileRef1 = %q", fileRef1) + fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0)) + qt.t.Logf("fileRef2 = %q", fileRef2) + dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2}, time.Unix(789, 0)) + qt.t.Logf("dirRef = %q", dirRef) + p1 := id.NewPlannedPermanode("1") + id.SetAttribute(p1, "camliContent", dirRef.String()) + + fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0)) + qt.t.Logf("fileRef3 = %q", fileRef3) + p2 := id.NewPlannedPermanode("2") + id.SetAttribute(p2, "camliContent", fileRef3.String()) + + sq := &SearchQuery{ + Constraint: &Constraint{ + Permanode: &PermanodeConstraint{ + Attr: "camliContent", + ValueInSet: &Constraint{ + Dir: &DirConstraint{ + Contains: &Constraint{File: &FileConstraint{ + FileName: &StringConstraint{ + Contains: "some-stuff.txt", + }, + }}, + }, + }, + }, + }, + } + qt.wantRes(sq, p1) + }) +} + +// find permanode with a dir that contains a certain file or dir +func TestQueryDirWithFileOrDirConstraint(t *testing.T) { + testQuery(t, func(qt *queryTest) { + id := qt.id + fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0)) + qt.t.Logf("fileRef1 = %q", fileRef1) + childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0)) + qt.t.Logf("childDirRef = %q", childDirRef) + dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, childDirRef}, time.Unix(789, 0)) + qt.t.Logf("dirRef = %q", dirRef) + p1 := id.NewPlannedPermanode("1") + id.SetAttribute(p1, "camliContent", dirRef.String()) + + fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0)) + qt.t.Logf("fileRef3 = %q", fileRef3) + p2 := id.NewPlannedPermanode("2") + id.SetAttribute(p2, "camliContent", fileRef3.String()) + + sq := &SearchQuery{ + Constraint: &Constraint{ + Permanode: &PermanodeConstraint{ + Attr: "camliContent", + ValueInSet: &Constraint{ + Dir: &DirConstraint{ + Contains: &Constraint{Logical: &LogicalConstraint{ + A: &Constraint{File: &FileConstraint{ + FileName: &StringConstraint{ + Equals: "foobar", + }, + }}, + B: &Constraint{Dir: &DirConstraint{ + FileName: &StringConstraint{ + Equals: "childdir", + }, + }}, + Op: "or", + }}, + }, + }, + }, + }, + } + qt.wantRes(sq, p1) + }) +} + +// find children of a directory, by name. +// in practice, one can also get the children with the proper describe rules, +// but doing so has some limitations that a direct search query has not. +func TestQueryDirChildrenByNameConstraint(t *testing.T) { + testQueryTypes(t, memIndexTypes, func(qt *queryTest) { + id := qt.id + fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0)) + qt.t.Logf("fileRef1 = %q", fileRef1) + fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0)) + qt.t.Logf("fileRef2 = %q", fileRef2) + childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0)) + qt.t.Logf("childDirRef = %q", childDirRef) + dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2, childDirRef}, time.Unix(789, 0)) + qt.t.Logf("dirRef = %q", dirRef) + p1 := id.NewPlannedPermanode("1") + id.SetAttribute(p1, "camliContent", dirRef.String()) + + fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0)) + qt.t.Logf("fileRef3 = %q", fileRef3) + p2 := id.NewPlannedPermanode("2") + id.SetAttribute(p2, "camliContent", fileRef3.String()) + + sq := &SearchQuery{ + Constraint: &Constraint{ + Logical: &LogicalConstraint{ + A: &Constraint{File: &FileConstraint{ + ParentDir: &DirConstraint{ + FileName: &StringConstraint{ + Equals: "somedir", + }, + }, + }}, + B: &Constraint{Dir: &DirConstraint{ + ParentDir: &DirConstraint{ + FileName: &StringConstraint{ + Equals: "somedir", + }, + }, + }}, + Op: "or", + }, + }, + } + qt.wantRes(sq, fileRef1, fileRef2, childDirRef) + }) +} + +// find children of a directory, by blobref. +func TestQueryDirChildrenByRefConstraint(t *testing.T) { + testQueryTypes(t, memIndexTypes, func(qt *queryTest) { + id := qt.id + fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0)) + qt.t.Logf("fileRef1 = %q", fileRef1) + fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0)) + qt.t.Logf("fileRef2 = %q", fileRef2) + childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0)) + qt.t.Logf("childDirRef = %q", childDirRef) + dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2, childDirRef}, time.Unix(789, 0)) + qt.t.Logf("dirRef = %q", dirRef) + p1 := id.NewPlannedPermanode("1") + id.SetAttribute(p1, "camliContent", dirRef.String()) + + fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0)) + qt.t.Logf("fileRef3 = %q", fileRef3) + p2 := id.NewPlannedPermanode("2") + id.SetAttribute(p2, "camliContent", fileRef3.String()) + + sq := &SearchQuery{ + Constraint: &Constraint{ + Logical: &LogicalConstraint{ + A: &Constraint{File: &FileConstraint{ + ParentDir: &DirConstraint{ + BlobRefPrefix: dirRef.String(), + }, + }}, + B: &Constraint{Dir: &DirConstraint{ + ParentDir: &DirConstraint{ + BlobRefPrefix: dirRef.String(), + }, + }}, + Op: "or", + }, + }, + } + qt.wantRes(sq, fileRef1, fileRef2, childDirRef) + }) +} + +// find out if a file is amongst a dir's progeny (grand-children) +func TestQueryDirProgeny(t *testing.T) { + testQuery(t, func(qt *queryTest) { + id := qt.id + grandchild1, _ := id.UploadFile("grandchild1.txt", "hello", time.Unix(123, 0)) + qt.t.Logf("grandchild1 = %q", grandchild1) + grandchild2, _ := id.UploadFile("grandchild2.txt", "world", time.Unix(456, 0)) + qt.t.Logf("grandchild2 = %q", grandchild2) + parentdir := id.UploadDir("parentdir", []blob.Ref{grandchild1, grandchild2}, time.Unix(789, 0)) + qt.t.Logf("parentdir = %q", parentdir) + grandparentdir := id.UploadDir("grandparentdir", []blob.Ref{parentdir}, time.Unix(101112, 0)) + qt.t.Logf("grandparentdir = %q", grandparentdir) + p1 := id.NewPlannedPermanode("1") + id.SetAttribute(p1, "camliContent", grandparentdir.String()) + + p3 := id.NewPlannedPermanode("3") + id.SetAttribute(p3, "camliContent", parentdir.String()) + + // adding an unrelated directory, to make sure we do _not_ find it as well + fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(131415, 0)) + qt.t.Logf("fileRef3 = %q", fileRef3) + otherdir := id.UploadDir("otherdir", []blob.Ref{fileRef3}, time.Unix(161718, 0)) + qt.t.Logf("otherdir = %q", otherdir) + p2 := id.NewPlannedPermanode("2") + id.SetAttribute(p2, "camliContent", otherdir.String()) + + sq := &SearchQuery{ + Constraint: &Constraint{ + Permanode: &PermanodeConstraint{ + Attr: "camliContent", + ValueInSet: &Constraint{ + Dir: &DirConstraint{ + RecursiveContains: &Constraint{File: &FileConstraint{ + FileName: &StringConstraint{ + Contains: "grandchild1.txt", + }, + }}, + }, + }, + }, + }, + } + qt.wantRes(sq, p1, p3) + + // make sure that "Contains" only finds the direct parent, and not the grand-parent as well. + // also this time, skip the permanode layer. + sq = &SearchQuery{ + Constraint: &Constraint{ + Dir: &DirConstraint{ + Contains: &Constraint{ + BlobRefPrefix: grandchild1.String(), + }, + }, + }, + } + qt.wantRes(sq, parentdir) + }) +} + func TestQueryFileConstraint_WholeRef(t *testing.T) { testQueryTypes(t, memIndexTypes, func(qt *queryTest) { id := qt.id