Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge "pkg/search: add some DirConstraint search functionality"

This commit is contained in:
Brad Fitzpatrick 2018-05-03 03:53:55 +00:00 committed by Gerrit Code Review
parent d6fb092e69 03103f00fa
commit b5793c5e1e
7 changed files with 597 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
pkg/index/corpus.go
View File

@ -81,6 +81,10 @@ type Corpus struct {
imageInfo map[blob.Ref]camtypes.ImageInfo // keyed by fileref (not wholeref)
fileWholeRef map[blob.Ref]blob.Ref // fileref -> its wholeref (TODO: multi-valued?)
gps map[blob.Ref]latLong // wholeRef -> GPS coordinates
// dirChildren maps a directory to its (direct) children (static-set entries).
dirChildren map[blob.Ref]map[blob.Ref]struct{}
// fileParents maps a file or directory to its (direct) parents.
fileParents map[blob.Ref]map[blob.Ref]struct{}
// Lack of edge tracking implementation is issue #707
// (https://github.com/perkeep/perkeep/issues/707)
@ -336,6 +340,8 @@ func newCorpus() *Corpus {
deletes: make(map[blob.Ref][]deletion),
claimBack: make(map[blob.Ref][]*camtypes.Claim),
permanodesSetByNodeType: make(map[string]map[blob.Ref]bool),
dirChildren: make(map[blob.Ref]map[blob.Ref]struct{}),
fileParents: make(map[blob.Ref]map[blob.Ref]struct{}),
}
c.permanodesByModtime = &lazySortedPermanodes{
c: c,
@ -383,19 +389,20 @@ func (crashStorage) Find(start, end string) sorted.Iterator {
// *********** Updating the corpus
var corpusMergeFunc = map[string]func(c *Corpus, k, v []byte) error{
"have": nil, // redundant with "meta"
"recpn": nil, // unneeded.
"meta": (*Corpus).mergeMetaRow,
keySignerKeyID.name: (*Corpus).mergeSignerKeyIdRow,
"claim": (*Corpus).mergeClaimRow,
"fileinfo": (*Corpus).mergeFileInfoRow,
keyFileTimes.name: (*Corpus).mergeFileTimesRow,
"imagesize": (*Corpus).mergeImageSizeRow,
"wholetofile": (*Corpus).mergeWholeToFileRow,
"exifgps": (*Corpus).mergeEXIFGPSRow,
"exiftag": nil, // not using any for now
"signerattrvalue": nil, // ignoring for now
"mediatag": (*Corpus).mergeMediaTag,
"have": nil, // redundant with "meta"
"recpn": nil, // unneeded.
"meta": (*Corpus).mergeMetaRow,
keySignerKeyID.name: (*Corpus).mergeSignerKeyIdRow,
"claim": (*Corpus).mergeClaimRow,
"fileinfo": (*Corpus).mergeFileInfoRow,
keyFileTimes.name: (*Corpus).mergeFileTimesRow,
"imagesize": (*Corpus).mergeImageSizeRow,
"wholetofile": (*Corpus).mergeWholeToFileRow,
"exifgps": (*Corpus).mergeEXIFGPSRow,
"exiftag": nil, // not using any for now
"signerattrvalue": nil, // ignoring for now
"mediatag": (*Corpus).mergeMediaTag,
keyStaticDirChild.name: (*Corpus).mergeStaticDirChildRow,
}
func memstats() *runtime.MemStats {
@ -420,6 +427,7 @@ var slurpPrefixes = []string{
"wholetofile|",
"exifgps|",
"mediatag|",
keyStaticDirChild.name + "|",
}
// Key types (without trailing punctuation) that we slurp to memory at start.
@ -768,6 +776,39 @@ func (c *Corpus) mergeFileInfoRow(k, v []byte) error {
return nil
}
func (c *Corpus) mergeStaticDirChildRow(k, v []byte) error {
// dirchild|sha1-dir|sha1-child" "1"
// strip the key name
sk := k[len(keyStaticDirChild.name)+1:]
pipe := bytes.IndexByte(sk, '|')
if pipe < 0 {
return fmt.Errorf("invalid dirchild key %q, missing second pipe", k)
}
parent, ok := blob.ParseBytes(sk[:pipe])
if !ok {
return fmt.Errorf("invalid dirchild parent blobref in key %q", k)
}
child, ok := blob.ParseBytes(sk[pipe+1:])
if !ok {
return fmt.Errorf("invalid dirchild child blobref in key %q", k)
}
parent = c.br(parent)
child = c.br(child)
children, ok := c.dirChildren[parent]
if !ok {
children = make(map[blob.Ref]struct{})
}
children[child] = struct{}{}
c.dirChildren[parent] = children
parents, ok := c.fileParents[child]
if !ok {
parents = make(map[blob.Ref]struct{})
}
parents[parent] = struct{}{}
c.fileParents[child] = parents
return nil
}
func (c *Corpus) mergeFileTimesRow(k, v []byte) error {
if len(v) == 0 {
return nil
@ -1383,6 +1424,24 @@ func (c *Corpus) GetFileInfo(ctx context.Context, fileRef blob.Ref) (fi camtypes
return
}
// GetDirChildren returns the direct children (static-set entries) of the directory dirRef.
func (c *Corpus) GetDirChildren(ctx context.Context, dirRef blob.Ref) (map[blob.Ref]struct{}, error) {
children, ok := c.dirChildren[dirRef]
if !ok {
return nil, os.ErrNotExist
}
return children, nil
}
// GetParentDirs returns the direct parents (directories) of the file or directory childRef.
func (c *Corpus) GetParentDirs(ctx context.Context, childRef blob.Ref) (map[blob.Ref]struct{}, error) {
parents, ok := c.fileParents[childRef]
if !ok {
return nil, os.ErrNotExist
}
return parents, nil
}
func (c *Corpus) GetImageInfo(ctx context.Context, fileRef blob.Ref) (ii camtypes.ImageInfo, err error) {
ii, ok := c.imageInfo[fileRef]
if !ok {

17
pkg/index/index.go
View File

@ -1565,10 +1565,25 @@ func kvEdgeBackward(k, v string) (edge *camtypes.Edge, ok bool) {
}
// GetDirMembers sends on dest the children of the static directory dir.
func (x *Index) GetDirMembers(dir blob.Ref, dest chan<- blob.Ref, limit int) (err error) {
func (x *Index) GetDirMembers(ctx context.Context, dir blob.Ref, dest chan<- blob.Ref, limit int) (err error) {
defer close(dest)
sent := 0
if x.corpus != nil {
children, err := x.corpus.GetDirChildren(ctx, dir)
if err != nil {
return err
}
for child := range children {
dest <- child
sent++
if sent == limit {
break
}
}
return nil
}
it := x.queryPrefix(keyStaticDirChild, dir.String())
defer closeIterator(it, &err)
for it.Next() {

2
pkg/index/indextest/tests.go
View File

@ -617,7 +617,7 @@ func Index(t *testing.T, initIdx func() *index.Index) {
// GetDirMembers
{
ch := make(chan blob.Ref, 10) // expect 2 results
err := id.Index.GetDirMembers(imagesDirRef, ch, 50)
err := id.Index.GetDirMembers(ctx, imagesDirRef, ch, 50)
if err != nil {
t.Fatalf("GetDirMembers = %v", err)
}

2
pkg/index/interface.go
View File

@ -103,7 +103,7 @@ type Interface interface {
// is nil.
// dest must be closed, even when returning an error.
// limit <= 0 means unlimited.
GetDirMembers(dirRef blob.Ref, dest chan<- blob.Ref, limit int) error
GetDirMembers(ctx context.Context, dirRef blob.Ref, dest chan<- blob.Ref, limit int) error
// Given an owner key, a camliType 'claim', 'attribute' name,
// and specific 'value', find the most recent permanode that has

4
pkg/search/describe.go
View File

@ -119,7 +119,7 @@ type DescribeRequest struct {
// Rules specifies a set of rules to instruct how to keep
// expanding the described set. All rules are tested and
// matching rules grow the the response set until all rules no
// matching rules grow the response set until all rules no
// longer match or internal limits are hit.
Rules []*DescribeRule `json:"rules,omitempty"`
@ -883,7 +883,7 @@ func (dr *DescribeRequest) getDirMembers(ctx context.Context, br blob.Ref, depth
ch := make(chan blob.Ref)
errch := make(chan error)
go func() {
errch <- dr.sh.index.GetDirMembers(br, ch, limit)
errch <- dr.sh.index.GetDirMembers(ctx, br, ch, limit)
}()
var members []blob.Ref

259
pkg/search/query.go
View File

@ -426,6 +426,10 @@ type FileConstraint struct {
// every known hash algorithm.
WholeRef blob.Ref `json:"wholeRef,omitempty"`
// ParentDir, if non-nil, constrains the file match based on properties
// of its parent directory.
ParentDir *DirConstraint `json:"parentDir,omitempty"`
// For images:
IsImage bool `json:"isImage,omitempty"`
EXIF *EXIFConstraint `json:"exif,omitempty"` // TODO: implement
@ -447,22 +451,35 @@ type MediaTagConstraint struct {
Int *IntConstraint `json:"int,omitempty"`
}
// DirConstraint matches static directories.
type DirConstraint struct {
// (All non-zero fields must match)
// TODO: implement. mostly need more things in the index.
FileName *StringConstraint `json:"fileName,omitempty"`
BlobRefPrefix string `json:"blobRefPrefix,omitempty"`
FileName *StringConstraint `json:"fileName,omitempty"`
// ParentDir, if non-nil, constrains the directory match based on properties
// of its parent directory.
ParentDir *DirConstraint `json:"parentDir,omitempty"`
TopFileSize, // not recursive
TopFileCount, // not recursive
FileSize,
FileCount *IntConstraint
// TODO: implement.
// FileCount *IntConstraint
// FileSize *IntConstraint
// TODO: these would need thought on how to index efficiently:
// (Also: top-only variants?)
// ContainsFile *FileConstraint
// ContainsDir *DirConstraint
// TopFileCount, if non-nil, constrains the directory match with the directory's
// number of children (non-recursively).
TopFileCount *IntConstraint `json:"topFileCount,omitempty"`
// RecursiveContains, if non-nil, is like Contains, but applied to all
// the descendants of the directory. It is mutually exclusive with Contains.
RecursiveContains *Constraint `json:"recursiveContains,omitempty"`
// Contains, if non-nil, constrains the directory match to just those
// directories containing a file matched by Contains. Contains should have a
// BlobPrefix, or a *FileConstraint, or a *DirConstraint, or a *LogicalConstraint
// combination of the aforementioned. It is only applied to the children of the
// directory, in a non-recursive manner. It is mutually exclusive with RecursiveContains.
Contains *Constraint `json:"contains,omitempty"`
}
// An IntConstraint specifies constraints on an integer.
@ -919,6 +936,34 @@ func (s *search) fileInfo(ctx context.Context, br blob.Ref) (camtypes.FileInfo,
return s.h.index.GetFileInfo(ctx, br)
}
func (s *search) dirChildren(ctx context.Context, br blob.Ref) (map[blob.Ref]struct{}, error) {
if c := s.h.corpus; c != nil {
return c.GetDirChildren(ctx, br)
}
ch := make(chan blob.Ref)
errch := make(chan error)
go func() {
errch <- s.h.index.GetDirMembers(ctx, br, ch, s.q.Limit)
}()
children := make(map[blob.Ref]struct{})
for child := range ch {
children[child] = struct{}{}
}
if err := <-errch; err != nil {
return nil, err
}
return children, nil
}
func (s *search) parentDirs(ctx context.Context, br blob.Ref) (map[blob.Ref]struct{}, error) {
c := s.h.corpus
if c == nil {
return nil, errors.New("parent directory search not supported without a corpus")
}
return c.GetParentDirs(ctx, br)
}
// optimizePlan returns an optimized version of c which will hopefully
// execute faster than executing c literally.
func optimizePlan(c *Constraint) *Constraint {
@ -1869,6 +1914,36 @@ func (c *FileConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref
return false, nil
}
}
if pc := c.ParentDir; pc != nil {
parents, err := s.parentDirs(ctx, br)
if err == os.ErrNotExist {
return false, nil
}
if err != nil {
return false, err
}
matches := false
for parent, _ := range parents {
meta, err := s.blobMeta(ctx, parent)
if err != nil {
if os.IsNotExist(err) {
continue
}
return false, err
}
ok, err := pc.blobMatches(ctx, s, parent, meta)
if err != nil {
return false, err
}
if ok {
matches = true
break
}
}
if !matches {
return false, nil
}
}
corpus := s.h.corpus
if c.WholeRef.Valid() {
if corpus == nil {
@ -1976,16 +2051,176 @@ func (c *TimeConstraint) timeMatches(t time.Time) bool {
}
func (c *DirConstraint) checkValid() error {
if c == nil {
return nil
}
if c.Contains != nil && c.RecursiveContains != nil {
return errors.New("Contains and RecursiveContains in a DirConstraint are mutually exclusive")
}
return nil
}
func (c *Constraint) isFileOrDirConstraint() bool {
if l := c.Logical; l != nil {
return l.A.isFileOrDirConstraint() && l.B.isFileOrDirConstraint()
}
return c.File != nil || c.Dir != nil
}
func (c *Constraint) fileOrDirOrLogicalMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) {
if cf := c.File; cf != nil {
return cf.blobMatches(ctx, s, br, bm)
}
if cd := c.Dir; cd != nil {
return cd.blobMatches(ctx, s, br, bm)
}
if l := c.Logical; l != nil {
return l.matcher()(ctx, s, br, bm)
}
return false, nil
}
func (c *DirConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) {
if bm.CamliType != "directory" {
return false, nil
}
// TODO(mpl): I've added c.BlobRefPrefix, so that c.ParentDir can be directly
// matched against a blobRef (instead of e.g. a filename), but I could instead make
// ParentDir be a *Constraint, and logically enforce that it has to "be equivalent"
// to a ParentDir matching or a BlobRefPrefix matching. I think this here below is
// simpler, but not sure it's best in the long run.
if pfx := c.BlobRefPrefix; pfx != "" {
if !br.HasPrefix(pfx) {
return false, nil
}
}
fi, err := s.fileInfo(ctx, br)
if err == os.ErrNotExist {
return false, nil
}
if err != nil {
return false, err
}
if sc := c.FileName; sc != nil && !sc.stringMatches(fi.FileName) {
return false, nil
}
if pc := c.ParentDir; pc != nil {
parents, err := s.parentDirs(ctx, br)
if err == os.ErrNotExist {
return false, nil
}
if err != nil {
return false, err
}
isMatch, err := pc.hasMatchingParent(ctx, s, parents)
if err != nil {
return false, err
}
if !isMatch {
return false, nil
}
}
// TODO: implement
panic("TODO: implement DirConstraint.blobMatches")
// All constraints not pertaining to children must happen above
// this point.
children, err := s.dirChildren(ctx, br)
if err != nil && err != os.ErrNotExist {
return false, err
}
if fc := c.TopFileCount; fc != nil && !fc.intMatches(int64(len(children))) {
return false, nil
}
cc := c.Contains
recursive := false
if cc == nil {
if crc := c.RecursiveContains; crc != nil {
recursive = true
// RecursiveContains implies Contains
cc = crc
}
}
// First test on the direct children
containsMatch := false
if cc != nil {
// Allow directly specifying the fileRef
if cc.BlobRefPrefix != "" {
containsMatch, err = c.hasMatchingChild(ctx, s, children, func(ctx context.Context, s *search, child blob.Ref, bm camtypes.BlobMeta) (bool, error) {
return child.HasPrefix(cc.BlobRefPrefix), nil
})
} else {
if !cc.isFileOrDirConstraint() {
return false, errors.New("[Recursive]Contains constraint should have a *FileConstraint, or a *DirConstraint, or a *LogicalConstraint combination of the aforementioned.")
}
containsMatch, err = c.hasMatchingChild(ctx, s, children, cc.fileOrDirOrLogicalMatches)
}
if err != nil {
return false, err
}
if !containsMatch && !recursive {
return false, nil
}
}
// Then if needed recurse on the next generation descendants.
if !containsMatch && recursive {
match, err := c.hasMatchingChild(ctx, s, children, c.blobMatches)
if err != nil {
return false, err
}
if !match {
return false, nil
}
}
// TODO: implement FileCount and FileSize.
return true, nil
}
// hasMatchingParent checks all parents against c and returns true as soon as one of
// them matches, or returns false if none of them is a match.
func (c *DirConstraint) hasMatchingParent(ctx context.Context, s *search, parents map[blob.Ref]struct{}) (bool, error) {
for parent := range parents {
meta, err := s.blobMeta(ctx, parent)
if err != nil {
if os.IsNotExist(err) {
continue
}
return false, err
}
ok, err := c.blobMatches(ctx, s, parent, meta)
if err != nil {
return false, err
}
if ok {
return true, nil
}
}
return false, nil
}
// hasMatchingChild runs matcher against each child and returns true as soon as
// there is a match, of false if none of them is a match.
func (c *DirConstraint) hasMatchingChild(ctx context.Context, s *search, children map[blob.Ref]struct{},
matcher func(context.Context, *search, blob.Ref, camtypes.BlobMeta) (bool, error)) (bool, error) {
// TODO(mpl): See if we're guaranteed to be CPU-bound (i.e. all resources are in
// corpus), and if not, add some concurrency to spread costly index lookups.
for child, _ := range children {
meta, err := s.blobMeta(ctx, child)
if err != nil {
if os.IsNotExist(err) {
continue
}
return false, err
}
ok, err := matcher(ctx, s, child, meta)
if err != nil {
return false, err
}
if ok {
return true, nil
}
}
return false, nil
}
type sortSearchResultBlobs struct {

258
pkg/search/query_test.go
View File

@ -724,6 +724,264 @@ func TestQueryFileConstraint(t *testing.T) {
})
}
// find a directory with a name
func TestQueryDirConstraint(t *testing.T) {
testQuery(t, func(qt *queryTest) {
id := qt.id
dirRef := id.UploadDir("somedir", []blob.Ref{}, time.Unix(789, 0))
qt.t.Logf("dirRef = %q", dirRef)
p1 := id.NewPlannedPermanode("1")
id.SetAttribute(p1, "camliContent", dirRef.String())
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
qt.t.Logf("fileRef3 = %q", fileRef3)
p2 := id.NewPlannedPermanode("2")
id.SetAttribute(p2, "camliContent", fileRef3.String())
sq := &SearchQuery{
Constraint: &Constraint{
Dir: &DirConstraint{
FileName: &StringConstraint{
Contains: "somedir",
},
},
},
}
qt.wantRes(sq, dirRef)
})
}
// find permanode with a dir that contains a certain file
func TestQueryDirWithFileConstraint(t *testing.T) {
testQuery(t, func(qt *queryTest) {
id := qt.id
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
qt.t.Logf("fileRef1 = %q", fileRef1)
fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0))
qt.t.Logf("fileRef2 = %q", fileRef2)
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2}, time.Unix(789, 0))
qt.t.Logf("dirRef = %q", dirRef)
p1 := id.NewPlannedPermanode("1")
id.SetAttribute(p1, "camliContent", dirRef.String())
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
qt.t.Logf("fileRef3 = %q", fileRef3)
p2 := id.NewPlannedPermanode("2")
id.SetAttribute(p2, "camliContent", fileRef3.String())
sq := &SearchQuery{
Constraint: &Constraint{
Permanode: &PermanodeConstraint{
Attr: "camliContent",
ValueInSet: &Constraint{
Dir: &DirConstraint{
Contains: &Constraint{File: &FileConstraint{
FileName: &StringConstraint{
Contains: "some-stuff.txt",
},
}},
},
},
},
},
}
qt.wantRes(sq, p1)
})
}
// find permanode with a dir that contains a certain file or dir
func TestQueryDirWithFileOrDirConstraint(t *testing.T) {
testQuery(t, func(qt *queryTest) {
id := qt.id
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
qt.t.Logf("fileRef1 = %q", fileRef1)
childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0))
qt.t.Logf("childDirRef = %q", childDirRef)
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, childDirRef}, time.Unix(789, 0))
qt.t.Logf("dirRef = %q", dirRef)
p1 := id.NewPlannedPermanode("1")
id.SetAttribute(p1, "camliContent", dirRef.String())
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
qt.t.Logf("fileRef3 = %q", fileRef3)
p2 := id.NewPlannedPermanode("2")
id.SetAttribute(p2, "camliContent", fileRef3.String())
sq := &SearchQuery{
Constraint: &Constraint{
Permanode: &PermanodeConstraint{
Attr: "camliContent",
ValueInSet: &Constraint{
Dir: &DirConstraint{
Contains: &Constraint{Logical: &LogicalConstraint{
A: &Constraint{File: &FileConstraint{
FileName: &StringConstraint{
Equals: "foobar",
},
}},
B: &Constraint{Dir: &DirConstraint{
FileName: &StringConstraint{
Equals: "childdir",
},
}},
Op: "or",
}},
},
},
},
},
}
qt.wantRes(sq, p1)
})
}
// find children of a directory, by name.
// in practice, one can also get the children with the proper describe rules,
// but doing so has some limitations that a direct search query has not.
func TestQueryDirChildrenByNameConstraint(t *testing.T) {
testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
id := qt.id
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
qt.t.Logf("fileRef1 = %q", fileRef1)
fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0))
qt.t.Logf("fileRef2 = %q", fileRef2)
childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0))
qt.t.Logf("childDirRef = %q", childDirRef)
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2, childDirRef}, time.Unix(789, 0))
qt.t.Logf("dirRef = %q", dirRef)
p1 := id.NewPlannedPermanode("1")
id.SetAttribute(p1, "camliContent", dirRef.String())
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
qt.t.Logf("fileRef3 = %q", fileRef3)
p2 := id.NewPlannedPermanode("2")
id.SetAttribute(p2, "camliContent", fileRef3.String())
sq := &SearchQuery{
Constraint: &Constraint{
Logical: &LogicalConstraint{
A: &Constraint{File: &FileConstraint{
ParentDir: &DirConstraint{
FileName: &StringConstraint{
Equals: "somedir",
},
},
}},
B: &Constraint{Dir: &DirConstraint{
ParentDir: &DirConstraint{
FileName: &StringConstraint{
Equals: "somedir",
},
},
}},
Op: "or",
},
},
}
qt.wantRes(sq, fileRef1, fileRef2, childDirRef)
})
}
// find children of a directory, by blobref.
func TestQueryDirChildrenByRefConstraint(t *testing.T) {
testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
id := qt.id
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
qt.t.Logf("fileRef1 = %q", fileRef1)
fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0))
qt.t.Logf("fileRef2 = %q", fileRef2)
childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0))
qt.t.Logf("childDirRef = %q", childDirRef)
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2, childDirRef}, time.Unix(789, 0))
qt.t.Logf("dirRef = %q", dirRef)
p1 := id.NewPlannedPermanode("1")
id.SetAttribute(p1, "camliContent", dirRef.String())
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
qt.t.Logf("fileRef3 = %q", fileRef3)
p2 := id.NewPlannedPermanode("2")
id.SetAttribute(p2, "camliContent", fileRef3.String())
sq := &SearchQuery{
Constraint: &Constraint{
Logical: &LogicalConstraint{
A: &Constraint{File: &FileConstraint{
ParentDir: &DirConstraint{
BlobRefPrefix: dirRef.String(),
},
}},
B: &Constraint{Dir: &DirConstraint{
ParentDir: &DirConstraint{
BlobRefPrefix: dirRef.String(),
},
}},
Op: "or",
},
},
}
qt.wantRes(sq, fileRef1, fileRef2, childDirRef)
})
}
// find out if a file is amongst a dir's progeny (grand-children)
func TestQueryDirProgeny(t *testing.T) {
testQuery(t, func(qt *queryTest) {
id := qt.id
grandchild1, _ := id.UploadFile("grandchild1.txt", "hello", time.Unix(123, 0))
qt.t.Logf("grandchild1 = %q", grandchild1)
grandchild2, _ := id.UploadFile("grandchild2.txt", "world", time.Unix(456, 0))
qt.t.Logf("grandchild2 = %q", grandchild2)
parentdir := id.UploadDir("parentdir", []blob.Ref{grandchild1, grandchild2}, time.Unix(789, 0))
qt.t.Logf("parentdir = %q", parentdir)
grandparentdir := id.UploadDir("grandparentdir", []blob.Ref{parentdir}, time.Unix(101112, 0))
qt.t.Logf("grandparentdir = %q", grandparentdir)
p1 := id.NewPlannedPermanode("1")
id.SetAttribute(p1, "camliContent", grandparentdir.String())
p3 := id.NewPlannedPermanode("3")
id.SetAttribute(p3, "camliContent", parentdir.String())
// adding an unrelated directory, to make sure we do _not_ find it as well
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(131415, 0))
qt.t.Logf("fileRef3 = %q", fileRef3)
otherdir := id.UploadDir("otherdir", []blob.Ref{fileRef3}, time.Unix(161718, 0))
qt.t.Logf("otherdir = %q", otherdir)
p2 := id.NewPlannedPermanode("2")
id.SetAttribute(p2, "camliContent", otherdir.String())
sq := &SearchQuery{
Constraint: &Constraint{
Permanode: &PermanodeConstraint{
Attr: "camliContent",
ValueInSet: &Constraint{
Dir: &DirConstraint{
RecursiveContains: &Constraint{File: &FileConstraint{
FileName: &StringConstraint{
Contains: "grandchild1.txt",
},
}},
},
},
},
},
}
qt.wantRes(sq, p1, p3)
// make sure that "Contains" only finds the direct parent, and not the grand-parent as well.
// also this time, skip the permanode layer.
sq = &SearchQuery{
Constraint: &Constraint{
Dir: &DirConstraint{
Contains: &Constraint{
BlobRefPrefix: grandchild1.String(),
},
},
},
}
qt.wantRes(sq, parentdir)
})
}
func TestQueryFileConstraint_WholeRef(t *testing.T) {
testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
id := qt.id