mirror of https://github.com/perkeep/perkeep.git
pkg/search: add some DirConstraint search functionality
The DirConstraint type already existed but wasn't functional as none of the matching for any of its fields was implemented. That functionality is required by two other features: 1) Browsing directories in the publisher requires getting information about the children of the directory. In practice that can be achieved with a search on the directory, accompanied with some describe rules. But that comes with limitations, such as: -no control over the sorting of the described children in the response -max number of children in the response (and no way to overcome it since you can't "continue" on a describe) hence the need for a direct search of a directory's children. This is implemented as DirConstraint/FileConstraint.ParentDir and will be used in https://camlistore-review.googlesource.com/8286 2) Looking for files/directories shared by transitivity. Knowing if an item is the target of a share claim is easy enough once enough of ClaimConstraint is implemented. But in order to find out if an item is effectively shared because one of its ancestors (directory) is the target of a share claim, with transitivity set, we need some sort of search that can do directories traversal. This is implemented as DirConstraint.RecursiveContains and it will be used as a subquery in https://camlistore-review.googlesource.com/9866. Now implemented: DirConstraint.FileName DirConstraint.BlobRefPrefix DirConstraint.ParentDir DirConstraint.TopFileCount DirConstraint.Contains DirConstraint.RecursiveContains ParentDir will also allow us to quit relying on the treeHandler in the web UI, and to use the more generic search queries mechanism instead. Change-Id: I022cb51732ee4271906271fd75c6f737856b6165
This commit is contained in:
mpl
parent
c806ab4bcd
commit
03103f00fa
|
|
@ -81,6 +81,10 @@ type Corpus struct {
|
|||
imageInfo map[blob.Ref]camtypes.ImageInfo // keyed by fileref (not wholeref)
|
||||
fileWholeRef map[blob.Ref]blob.Ref // fileref -> its wholeref (TODO: multi-valued?)
|
||||
gps map[blob.Ref]latLong // wholeRef -> GPS coordinates
|
||||
// dirChildren maps a directory to its (direct) children (static-set entries).
|
||||
dirChildren map[blob.Ref]map[blob.Ref]struct{}
|
||||
// fileParents maps a file or directory to its (direct) parents.
|
||||
fileParents map[blob.Ref]map[blob.Ref]struct{}
|
||||
|
||||
// Lack of edge tracking implementation is issue #707
|
||||
// (https://github.com/perkeep/perkeep/issues/707)
|
||||
|
|
@ -336,6 +340,8 @@ func newCorpus() *Corpus {
|
|||
deletes: make(map[blob.Ref][]deletion),
|
||||
claimBack: make(map[blob.Ref][]*camtypes.Claim),
|
||||
permanodesSetByNodeType: make(map[string]map[blob.Ref]bool),
|
||||
dirChildren: make(map[blob.Ref]map[blob.Ref]struct{}),
|
||||
fileParents: make(map[blob.Ref]map[blob.Ref]struct{}),
|
||||
}
|
||||
c.permanodesByModtime = &lazySortedPermanodes{
|
||||
c: c,
|
||||
|
|
@ -383,19 +389,20 @@ func (crashStorage) Find(start, end string) sorted.Iterator {
|
|||
// *********** Updating the corpus
|
||||
|
||||
var corpusMergeFunc = map[string]func(c *Corpus, k, v []byte) error{
|
||||
"have": nil, // redundant with "meta"
|
||||
"recpn": nil, // unneeded.
|
||||
"meta": (*Corpus).mergeMetaRow,
|
||||
keySignerKeyID.name: (*Corpus).mergeSignerKeyIdRow,
|
||||
"claim": (*Corpus).mergeClaimRow,
|
||||
"fileinfo": (*Corpus).mergeFileInfoRow,
|
||||
keyFileTimes.name: (*Corpus).mergeFileTimesRow,
|
||||
"imagesize": (*Corpus).mergeImageSizeRow,
|
||||
"wholetofile": (*Corpus).mergeWholeToFileRow,
|
||||
"exifgps": (*Corpus).mergeEXIFGPSRow,
|
||||
"exiftag": nil, // not using any for now
|
||||
"signerattrvalue": nil, // ignoring for now
|
||||
"mediatag": (*Corpus).mergeMediaTag,
|
||||
"have": nil, // redundant with "meta"
|
||||
"recpn": nil, // unneeded.
|
||||
"meta": (*Corpus).mergeMetaRow,
|
||||
keySignerKeyID.name: (*Corpus).mergeSignerKeyIdRow,
|
||||
"claim": (*Corpus).mergeClaimRow,
|
||||
"fileinfo": (*Corpus).mergeFileInfoRow,
|
||||
keyFileTimes.name: (*Corpus).mergeFileTimesRow,
|
||||
"imagesize": (*Corpus).mergeImageSizeRow,
|
||||
"wholetofile": (*Corpus).mergeWholeToFileRow,
|
||||
"exifgps": (*Corpus).mergeEXIFGPSRow,
|
||||
"exiftag": nil, // not using any for now
|
||||
"signerattrvalue": nil, // ignoring for now
|
||||
"mediatag": (*Corpus).mergeMediaTag,
|
||||
keyStaticDirChild.name: (*Corpus).mergeStaticDirChildRow,
|
||||
}
|
||||
|
||||
func memstats() *runtime.MemStats {
|
||||
|
|
@ -420,6 +427,7 @@ var slurpPrefixes = []string{
|
|||
"wholetofile|",
|
||||
"exifgps|",
|
||||
"mediatag|",
|
||||
keyStaticDirChild.name + "|",
|
||||
}
|
||||
|
||||
// Key types (without trailing punctuation) that we slurp to memory at start.
|
||||
|
|
@ -768,6 +776,39 @@ func (c *Corpus) mergeFileInfoRow(k, v []byte) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *Corpus) mergeStaticDirChildRow(k, v []byte) error {
|
||||
// dirchild|sha1-dir|sha1-child" "1"
|
||||
// strip the key name
|
||||
sk := k[len(keyStaticDirChild.name)+1:]
|
||||
pipe := bytes.IndexByte(sk, '|')
|
||||
if pipe < 0 {
|
||||
return fmt.Errorf("invalid dirchild key %q, missing second pipe", k)
|
||||
}
|
||||
parent, ok := blob.ParseBytes(sk[:pipe])
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid dirchild parent blobref in key %q", k)
|
||||
}
|
||||
child, ok := blob.ParseBytes(sk[pipe+1:])
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid dirchild child blobref in key %q", k)
|
||||
}
|
||||
parent = c.br(parent)
|
||||
child = c.br(child)
|
||||
children, ok := c.dirChildren[parent]
|
||||
if !ok {
|
||||
children = make(map[blob.Ref]struct{})
|
||||
}
|
||||
children[child] = struct{}{}
|
||||
c.dirChildren[parent] = children
|
||||
parents, ok := c.fileParents[child]
|
||||
if !ok {
|
||||
parents = make(map[blob.Ref]struct{})
|
||||
}
|
||||
parents[parent] = struct{}{}
|
||||
c.fileParents[child] = parents
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Corpus) mergeFileTimesRow(k, v []byte) error {
|
||||
if len(v) == 0 {
|
||||
return nil
|
||||
|
|
@ -1383,6 +1424,24 @@ func (c *Corpus) GetFileInfo(ctx context.Context, fileRef blob.Ref) (fi camtypes
|
|||
return
|
||||
}
|
||||
|
||||
// GetDirChildren returns the direct children (static-set entries) of the directory dirRef.
|
||||
func (c *Corpus) GetDirChildren(ctx context.Context, dirRef blob.Ref) (map[blob.Ref]struct{}, error) {
|
||||
children, ok := c.dirChildren[dirRef]
|
||||
if !ok {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return children, nil
|
||||
}
|
||||
|
||||
// GetParentDirs returns the direct parents (directories) of the file or directory childRef.
|
||||
func (c *Corpus) GetParentDirs(ctx context.Context, childRef blob.Ref) (map[blob.Ref]struct{}, error) {
|
||||
parents, ok := c.fileParents[childRef]
|
||||
if !ok {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return parents, nil
|
||||
}
|
||||
|
||||
func (c *Corpus) GetImageInfo(ctx context.Context, fileRef blob.Ref) (ii camtypes.ImageInfo, err error) {
|
||||
ii, ok := c.imageInfo[fileRef]
|
||||
if !ok {
|
||||
|
|
|
|||
|
|
@ -1565,10 +1565,25 @@ func kvEdgeBackward(k, v string) (edge *camtypes.Edge, ok bool) {
|
|||
}
|
||||
|
||||
// GetDirMembers sends on dest the children of the static directory dir.
|
||||
func (x *Index) GetDirMembers(dir blob.Ref, dest chan<- blob.Ref, limit int) (err error) {
|
||||
func (x *Index) GetDirMembers(ctx context.Context, dir blob.Ref, dest chan<- blob.Ref, limit int) (err error) {
|
||||
defer close(dest)
|
||||
|
||||
sent := 0
|
||||
if x.corpus != nil {
|
||||
children, err := x.corpus.GetDirChildren(ctx, dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for child := range children {
|
||||
dest <- child
|
||||
sent++
|
||||
if sent == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
it := x.queryPrefix(keyStaticDirChild, dir.String())
|
||||
defer closeIterator(it, &err)
|
||||
for it.Next() {
|
||||
|
|
|
|||
|
|
@ -617,7 +617,7 @@ func Index(t *testing.T, initIdx func() *index.Index) {
|
|||
// GetDirMembers
|
||||
{
|
||||
ch := make(chan blob.Ref, 10) // expect 2 results
|
||||
err := id.Index.GetDirMembers(imagesDirRef, ch, 50)
|
||||
err := id.Index.GetDirMembers(ctx, imagesDirRef, ch, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("GetDirMembers = %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ type Interface interface {
|
|||
// is nil.
|
||||
// dest must be closed, even when returning an error.
|
||||
// limit <= 0 means unlimited.
|
||||
GetDirMembers(dirRef blob.Ref, dest chan<- blob.Ref, limit int) error
|
||||
GetDirMembers(ctx context.Context, dirRef blob.Ref, dest chan<- blob.Ref, limit int) error
|
||||
|
||||
// Given an owner key, a camliType 'claim', 'attribute' name,
|
||||
// and specific 'value', find the most recent permanode that has
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ type DescribeRequest struct {
|
|||
|
||||
// Rules specifies a set of rules to instruct how to keep
|
||||
// expanding the described set. All rules are tested and
|
||||
// matching rules grow the the response set until all rules no
|
||||
// matching rules grow the response set until all rules no
|
||||
// longer match or internal limits are hit.
|
||||
Rules []*DescribeRule `json:"rules,omitempty"`
|
||||
|
||||
|
|
@ -883,7 +883,7 @@ func (dr *DescribeRequest) getDirMembers(ctx context.Context, br blob.Ref, depth
|
|||
ch := make(chan blob.Ref)
|
||||
errch := make(chan error)
|
||||
go func() {
|
||||
errch <- dr.sh.index.GetDirMembers(br, ch, limit)
|
||||
errch <- dr.sh.index.GetDirMembers(ctx, br, ch, limit)
|
||||
}()
|
||||
|
||||
var members []blob.Ref
|
||||
|
|
|
|||
|
|
@ -426,6 +426,10 @@ type FileConstraint struct {
|
|||
// every known hash algorithm.
|
||||
WholeRef blob.Ref `json:"wholeRef,omitempty"`
|
||||
|
||||
// ParentDir, if non-nil, constrains the file match based on properties
|
||||
// of its parent directory.
|
||||
ParentDir *DirConstraint `json:"parentDir,omitempty"`
|
||||
|
||||
// For images:
|
||||
IsImage bool `json:"isImage,omitempty"`
|
||||
EXIF *EXIFConstraint `json:"exif,omitempty"` // TODO: implement
|
||||
|
|
@ -447,22 +451,35 @@ type MediaTagConstraint struct {
|
|||
Int *IntConstraint `json:"int,omitempty"`
|
||||
}
|
||||
|
||||
// DirConstraint matches static directories.
|
||||
type DirConstraint struct {
|
||||
// (All non-zero fields must match)
|
||||
|
||||
// TODO: implement. mostly need more things in the index.
|
||||
FileName *StringConstraint `json:"fileName,omitempty"`
|
||||
BlobRefPrefix string `json:"blobRefPrefix,omitempty"`
|
||||
|
||||
FileName *StringConstraint `json:"fileName,omitempty"`
|
||||
// ParentDir, if non-nil, constrains the directory match based on properties
|
||||
// of its parent directory.
|
||||
ParentDir *DirConstraint `json:"parentDir,omitempty"`
|
||||
|
||||
TopFileSize, // not recursive
|
||||
TopFileCount, // not recursive
|
||||
FileSize,
|
||||
FileCount *IntConstraint
|
||||
// TODO: implement.
|
||||
// FileCount *IntConstraint
|
||||
// FileSize *IntConstraint
|
||||
|
||||
// TODO: these would need thought on how to index efficiently:
|
||||
// (Also: top-only variants?)
|
||||
// ContainsFile *FileConstraint
|
||||
// ContainsDir *DirConstraint
|
||||
// TopFileCount, if non-nil, constrains the directory match with the directory's
|
||||
// number of children (non-recursively).
|
||||
TopFileCount *IntConstraint `json:"topFileCount,omitempty"`
|
||||
|
||||
// RecursiveContains, if non-nil, is like Contains, but applied to all
|
||||
// the descendants of the directory. It is mutually exclusive with Contains.
|
||||
RecursiveContains *Constraint `json:"recursiveContains,omitempty"`
|
||||
|
||||
// Contains, if non-nil, constrains the directory match to just those
|
||||
// directories containing a file matched by Contains. Contains should have a
|
||||
// BlobPrefix, or a *FileConstraint, or a *DirConstraint, or a *LogicalConstraint
|
||||
// combination of the aforementioned. It is only applied to the children of the
|
||||
// directory, in a non-recursive manner. It is mutually exclusive with RecursiveContains.
|
||||
Contains *Constraint `json:"contains,omitempty"`
|
||||
}
|
||||
|
||||
// An IntConstraint specifies constraints on an integer.
|
||||
|
|
@ -919,6 +936,34 @@ func (s *search) fileInfo(ctx context.Context, br blob.Ref) (camtypes.FileInfo,
|
|||
return s.h.index.GetFileInfo(ctx, br)
|
||||
}
|
||||
|
||||
func (s *search) dirChildren(ctx context.Context, br blob.Ref) (map[blob.Ref]struct{}, error) {
|
||||
if c := s.h.corpus; c != nil {
|
||||
return c.GetDirChildren(ctx, br)
|
||||
}
|
||||
|
||||
ch := make(chan blob.Ref)
|
||||
errch := make(chan error)
|
||||
go func() {
|
||||
errch <- s.h.index.GetDirMembers(ctx, br, ch, s.q.Limit)
|
||||
}()
|
||||
children := make(map[blob.Ref]struct{})
|
||||
for child := range ch {
|
||||
children[child] = struct{}{}
|
||||
}
|
||||
if err := <-errch; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return children, nil
|
||||
}
|
||||
|
||||
func (s *search) parentDirs(ctx context.Context, br blob.Ref) (map[blob.Ref]struct{}, error) {
|
||||
c := s.h.corpus
|
||||
if c == nil {
|
||||
return nil, errors.New("parent directory search not supported without a corpus")
|
||||
}
|
||||
return c.GetParentDirs(ctx, br)
|
||||
}
|
||||
|
||||
// optimizePlan returns an optimized version of c which will hopefully
|
||||
// execute faster than executing c literally.
|
||||
func optimizePlan(c *Constraint) *Constraint {
|
||||
|
|
@ -1869,6 +1914,36 @@ func (c *FileConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref
|
|||
return false, nil
|
||||
}
|
||||
}
|
||||
if pc := c.ParentDir; pc != nil {
|
||||
parents, err := s.parentDirs(ctx, br)
|
||||
if err == os.ErrNotExist {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
matches := false
|
||||
for parent, _ := range parents {
|
||||
meta, err := s.blobMeta(ctx, parent)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
ok, err := pc.blobMatches(ctx, s, parent, meta)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if ok {
|
||||
matches = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matches {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
corpus := s.h.corpus
|
||||
if c.WholeRef.Valid() {
|
||||
if corpus == nil {
|
||||
|
|
@ -1976,16 +2051,176 @@ func (c *TimeConstraint) timeMatches(t time.Time) bool {
|
|||
}
|
||||
|
||||
func (c *DirConstraint) checkValid() error {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
if c.Contains != nil && c.RecursiveContains != nil {
|
||||
return errors.New("Contains and RecursiveContains in a DirConstraint are mutually exclusive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Constraint) isFileOrDirConstraint() bool {
|
||||
if l := c.Logical; l != nil {
|
||||
return l.A.isFileOrDirConstraint() && l.B.isFileOrDirConstraint()
|
||||
}
|
||||
return c.File != nil || c.Dir != nil
|
||||
}
|
||||
|
||||
func (c *Constraint) fileOrDirOrLogicalMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) {
|
||||
if cf := c.File; cf != nil {
|
||||
return cf.blobMatches(ctx, s, br, bm)
|
||||
}
|
||||
if cd := c.Dir; cd != nil {
|
||||
return cd.blobMatches(ctx, s, br, bm)
|
||||
}
|
||||
if l := c.Logical; l != nil {
|
||||
return l.matcher()(ctx, s, br, bm)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *DirConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) {
|
||||
if bm.CamliType != "directory" {
|
||||
return false, nil
|
||||
}
|
||||
// TODO(mpl): I've added c.BlobRefPrefix, so that c.ParentDir can be directly
|
||||
// matched against a blobRef (instead of e.g. a filename), but I could instead make
|
||||
// ParentDir be a *Constraint, and logically enforce that it has to "be equivalent"
|
||||
// to a ParentDir matching or a BlobRefPrefix matching. I think this here below is
|
||||
// simpler, but not sure it's best in the long run.
|
||||
if pfx := c.BlobRefPrefix; pfx != "" {
|
||||
if !br.HasPrefix(pfx) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
fi, err := s.fileInfo(ctx, br)
|
||||
if err == os.ErrNotExist {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if sc := c.FileName; sc != nil && !sc.stringMatches(fi.FileName) {
|
||||
return false, nil
|
||||
}
|
||||
if pc := c.ParentDir; pc != nil {
|
||||
parents, err := s.parentDirs(ctx, br)
|
||||
if err == os.ErrNotExist {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
isMatch, err := pc.hasMatchingParent(ctx, s, parents)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !isMatch {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: implement
|
||||
panic("TODO: implement DirConstraint.blobMatches")
|
||||
// All constraints not pertaining to children must happen above
|
||||
// this point.
|
||||
children, err := s.dirChildren(ctx, br)
|
||||
if err != nil && err != os.ErrNotExist {
|
||||
return false, err
|
||||
}
|
||||
if fc := c.TopFileCount; fc != nil && !fc.intMatches(int64(len(children))) {
|
||||
return false, nil
|
||||
}
|
||||
cc := c.Contains
|
||||
recursive := false
|
||||
if cc == nil {
|
||||
if crc := c.RecursiveContains; crc != nil {
|
||||
recursive = true
|
||||
// RecursiveContains implies Contains
|
||||
cc = crc
|
||||
}
|
||||
}
|
||||
// First test on the direct children
|
||||
containsMatch := false
|
||||
if cc != nil {
|
||||
// Allow directly specifying the fileRef
|
||||
if cc.BlobRefPrefix != "" {
|
||||
containsMatch, err = c.hasMatchingChild(ctx, s, children, func(ctx context.Context, s *search, child blob.Ref, bm camtypes.BlobMeta) (bool, error) {
|
||||
return child.HasPrefix(cc.BlobRefPrefix), nil
|
||||
})
|
||||
} else {
|
||||
if !cc.isFileOrDirConstraint() {
|
||||
return false, errors.New("[Recursive]Contains constraint should have a *FileConstraint, or a *DirConstraint, or a *LogicalConstraint combination of the aforementioned.")
|
||||
}
|
||||
containsMatch, err = c.hasMatchingChild(ctx, s, children, cc.fileOrDirOrLogicalMatches)
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !containsMatch && !recursive {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
// Then if needed recurse on the next generation descendants.
|
||||
if !containsMatch && recursive {
|
||||
match, err := c.hasMatchingChild(ctx, s, children, c.blobMatches)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !match {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: implement FileCount and FileSize.
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// hasMatchingParent checks all parents against c and returns true as soon as one of
|
||||
// them matches, or returns false if none of them is a match.
|
||||
func (c *DirConstraint) hasMatchingParent(ctx context.Context, s *search, parents map[blob.Ref]struct{}) (bool, error) {
|
||||
for parent := range parents {
|
||||
meta, err := s.blobMeta(ctx, parent)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
ok, err := c.blobMatches(ctx, s, parent, meta)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if ok {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// hasMatchingChild runs matcher against each child and returns true as soon as
|
||||
// there is a match, of false if none of them is a match.
|
||||
func (c *DirConstraint) hasMatchingChild(ctx context.Context, s *search, children map[blob.Ref]struct{},
|
||||
matcher func(context.Context, *search, blob.Ref, camtypes.BlobMeta) (bool, error)) (bool, error) {
|
||||
// TODO(mpl): See if we're guaranteed to be CPU-bound (i.e. all resources are in
|
||||
// corpus), and if not, add some concurrency to spread costly index lookups.
|
||||
for child, _ := range children {
|
||||
meta, err := s.blobMeta(ctx, child)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
ok, err := matcher(ctx, s, child, meta)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if ok {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type sortSearchResultBlobs struct {
|
||||
|
|
|
|||
|
|
@ -724,6 +724,264 @@ func TestQueryFileConstraint(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// find a directory with a name
|
||||
func TestQueryDirConstraint(t *testing.T) {
|
||||
testQuery(t, func(qt *queryTest) {
|
||||
id := qt.id
|
||||
dirRef := id.UploadDir("somedir", []blob.Ref{}, time.Unix(789, 0))
|
||||
qt.t.Logf("dirRef = %q", dirRef)
|
||||
p1 := id.NewPlannedPermanode("1")
|
||||
id.SetAttribute(p1, "camliContent", dirRef.String())
|
||||
|
||||
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
|
||||
qt.t.Logf("fileRef3 = %q", fileRef3)
|
||||
p2 := id.NewPlannedPermanode("2")
|
||||
id.SetAttribute(p2, "camliContent", fileRef3.String())
|
||||
|
||||
sq := &SearchQuery{
|
||||
Constraint: &Constraint{
|
||||
Dir: &DirConstraint{
|
||||
FileName: &StringConstraint{
|
||||
Contains: "somedir",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.wantRes(sq, dirRef)
|
||||
})
|
||||
}
|
||||
|
||||
// find permanode with a dir that contains a certain file
|
||||
func TestQueryDirWithFileConstraint(t *testing.T) {
|
||||
testQuery(t, func(qt *queryTest) {
|
||||
id := qt.id
|
||||
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
|
||||
qt.t.Logf("fileRef1 = %q", fileRef1)
|
||||
fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0))
|
||||
qt.t.Logf("fileRef2 = %q", fileRef2)
|
||||
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2}, time.Unix(789, 0))
|
||||
qt.t.Logf("dirRef = %q", dirRef)
|
||||
p1 := id.NewPlannedPermanode("1")
|
||||
id.SetAttribute(p1, "camliContent", dirRef.String())
|
||||
|
||||
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
|
||||
qt.t.Logf("fileRef3 = %q", fileRef3)
|
||||
p2 := id.NewPlannedPermanode("2")
|
||||
id.SetAttribute(p2, "camliContent", fileRef3.String())
|
||||
|
||||
sq := &SearchQuery{
|
||||
Constraint: &Constraint{
|
||||
Permanode: &PermanodeConstraint{
|
||||
Attr: "camliContent",
|
||||
ValueInSet: &Constraint{
|
||||
Dir: &DirConstraint{
|
||||
Contains: &Constraint{File: &FileConstraint{
|
||||
FileName: &StringConstraint{
|
||||
Contains: "some-stuff.txt",
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.wantRes(sq, p1)
|
||||
})
|
||||
}
|
||||
|
||||
// find permanode with a dir that contains a certain file or dir
|
||||
func TestQueryDirWithFileOrDirConstraint(t *testing.T) {
|
||||
testQuery(t, func(qt *queryTest) {
|
||||
id := qt.id
|
||||
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
|
||||
qt.t.Logf("fileRef1 = %q", fileRef1)
|
||||
childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0))
|
||||
qt.t.Logf("childDirRef = %q", childDirRef)
|
||||
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, childDirRef}, time.Unix(789, 0))
|
||||
qt.t.Logf("dirRef = %q", dirRef)
|
||||
p1 := id.NewPlannedPermanode("1")
|
||||
id.SetAttribute(p1, "camliContent", dirRef.String())
|
||||
|
||||
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
|
||||
qt.t.Logf("fileRef3 = %q", fileRef3)
|
||||
p2 := id.NewPlannedPermanode("2")
|
||||
id.SetAttribute(p2, "camliContent", fileRef3.String())
|
||||
|
||||
sq := &SearchQuery{
|
||||
Constraint: &Constraint{
|
||||
Permanode: &PermanodeConstraint{
|
||||
Attr: "camliContent",
|
||||
ValueInSet: &Constraint{
|
||||
Dir: &DirConstraint{
|
||||
Contains: &Constraint{Logical: &LogicalConstraint{
|
||||
A: &Constraint{File: &FileConstraint{
|
||||
FileName: &StringConstraint{
|
||||
Equals: "foobar",
|
||||
},
|
||||
}},
|
||||
B: &Constraint{Dir: &DirConstraint{
|
||||
FileName: &StringConstraint{
|
||||
Equals: "childdir",
|
||||
},
|
||||
}},
|
||||
Op: "or",
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.wantRes(sq, p1)
|
||||
})
|
||||
}
|
||||
|
||||
// find children of a directory, by name.
|
||||
// in practice, one can also get the children with the proper describe rules,
|
||||
// but doing so has some limitations that a direct search query has not.
|
||||
func TestQueryDirChildrenByNameConstraint(t *testing.T) {
|
||||
testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
|
||||
id := qt.id
|
||||
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
|
||||
qt.t.Logf("fileRef1 = %q", fileRef1)
|
||||
fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0))
|
||||
qt.t.Logf("fileRef2 = %q", fileRef2)
|
||||
childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0))
|
||||
qt.t.Logf("childDirRef = %q", childDirRef)
|
||||
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2, childDirRef}, time.Unix(789, 0))
|
||||
qt.t.Logf("dirRef = %q", dirRef)
|
||||
p1 := id.NewPlannedPermanode("1")
|
||||
id.SetAttribute(p1, "camliContent", dirRef.String())
|
||||
|
||||
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
|
||||
qt.t.Logf("fileRef3 = %q", fileRef3)
|
||||
p2 := id.NewPlannedPermanode("2")
|
||||
id.SetAttribute(p2, "camliContent", fileRef3.String())
|
||||
|
||||
sq := &SearchQuery{
|
||||
Constraint: &Constraint{
|
||||
Logical: &LogicalConstraint{
|
||||
A: &Constraint{File: &FileConstraint{
|
||||
ParentDir: &DirConstraint{
|
||||
FileName: &StringConstraint{
|
||||
Equals: "somedir",
|
||||
},
|
||||
},
|
||||
}},
|
||||
B: &Constraint{Dir: &DirConstraint{
|
||||
ParentDir: &DirConstraint{
|
||||
FileName: &StringConstraint{
|
||||
Equals: "somedir",
|
||||
},
|
||||
},
|
||||
}},
|
||||
Op: "or",
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.wantRes(sq, fileRef1, fileRef2, childDirRef)
|
||||
})
|
||||
}
|
||||
|
||||
// find children of a directory, by blobref.
|
||||
func TestQueryDirChildrenByRefConstraint(t *testing.T) {
|
||||
testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
|
||||
id := qt.id
|
||||
fileRef1, _ := id.UploadFile("some-stuff.txt", "hello", time.Unix(123, 0))
|
||||
qt.t.Logf("fileRef1 = %q", fileRef1)
|
||||
fileRef2, _ := id.UploadFile("more-stuff.txt", "world", time.Unix(456, 0))
|
||||
qt.t.Logf("fileRef2 = %q", fileRef2)
|
||||
childDirRef := id.UploadDir("childdir", []blob.Ref{}, time.Unix(457, 0))
|
||||
qt.t.Logf("childDirRef = %q", childDirRef)
|
||||
dirRef := id.UploadDir("somedir", []blob.Ref{fileRef1, fileRef2, childDirRef}, time.Unix(789, 0))
|
||||
qt.t.Logf("dirRef = %q", dirRef)
|
||||
p1 := id.NewPlannedPermanode("1")
|
||||
id.SetAttribute(p1, "camliContent", dirRef.String())
|
||||
|
||||
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(101112, 0))
|
||||
qt.t.Logf("fileRef3 = %q", fileRef3)
|
||||
p2 := id.NewPlannedPermanode("2")
|
||||
id.SetAttribute(p2, "camliContent", fileRef3.String())
|
||||
|
||||
sq := &SearchQuery{
|
||||
Constraint: &Constraint{
|
||||
Logical: &LogicalConstraint{
|
||||
A: &Constraint{File: &FileConstraint{
|
||||
ParentDir: &DirConstraint{
|
||||
BlobRefPrefix: dirRef.String(),
|
||||
},
|
||||
}},
|
||||
B: &Constraint{Dir: &DirConstraint{
|
||||
ParentDir: &DirConstraint{
|
||||
BlobRefPrefix: dirRef.String(),
|
||||
},
|
||||
}},
|
||||
Op: "or",
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.wantRes(sq, fileRef1, fileRef2, childDirRef)
|
||||
})
|
||||
}
|
||||
|
||||
// find out if a file is amongst a dir's progeny (grand-children)
|
||||
func TestQueryDirProgeny(t *testing.T) {
|
||||
testQuery(t, func(qt *queryTest) {
|
||||
id := qt.id
|
||||
grandchild1, _ := id.UploadFile("grandchild1.txt", "hello", time.Unix(123, 0))
|
||||
qt.t.Logf("grandchild1 = %q", grandchild1)
|
||||
grandchild2, _ := id.UploadFile("grandchild2.txt", "world", time.Unix(456, 0))
|
||||
qt.t.Logf("grandchild2 = %q", grandchild2)
|
||||
parentdir := id.UploadDir("parentdir", []blob.Ref{grandchild1, grandchild2}, time.Unix(789, 0))
|
||||
qt.t.Logf("parentdir = %q", parentdir)
|
||||
grandparentdir := id.UploadDir("grandparentdir", []blob.Ref{parentdir}, time.Unix(101112, 0))
|
||||
qt.t.Logf("grandparentdir = %q", grandparentdir)
|
||||
p1 := id.NewPlannedPermanode("1")
|
||||
id.SetAttribute(p1, "camliContent", grandparentdir.String())
|
||||
|
||||
p3 := id.NewPlannedPermanode("3")
|
||||
id.SetAttribute(p3, "camliContent", parentdir.String())
|
||||
|
||||
// adding an unrelated directory, to make sure we do _not_ find it as well
|
||||
fileRef3, _ := id.UploadFile("other-file", "hellooooo", time.Unix(131415, 0))
|
||||
qt.t.Logf("fileRef3 = %q", fileRef3)
|
||||
otherdir := id.UploadDir("otherdir", []blob.Ref{fileRef3}, time.Unix(161718, 0))
|
||||
qt.t.Logf("otherdir = %q", otherdir)
|
||||
p2 := id.NewPlannedPermanode("2")
|
||||
id.SetAttribute(p2, "camliContent", otherdir.String())
|
||||
|
||||
sq := &SearchQuery{
|
||||
Constraint: &Constraint{
|
||||
Permanode: &PermanodeConstraint{
|
||||
Attr: "camliContent",
|
||||
ValueInSet: &Constraint{
|
||||
Dir: &DirConstraint{
|
||||
RecursiveContains: &Constraint{File: &FileConstraint{
|
||||
FileName: &StringConstraint{
|
||||
Contains: "grandchild1.txt",
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.wantRes(sq, p1, p3)
|
||||
|
||||
// make sure that "Contains" only finds the direct parent, and not the grand-parent as well.
|
||||
// also this time, skip the permanode layer.
|
||||
sq = &SearchQuery{
|
||||
Constraint: &Constraint{
|
||||
Dir: &DirConstraint{
|
||||
Contains: &Constraint{
|
||||
BlobRefPrefix: grandchild1.String(),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.wantRes(sq, parentdir)
|
||||
})
|
||||
}
|
||||
|
||||
func TestQueryFileConstraint_WholeRef(t *testing.T) {
|
||||
testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
|
||||
id := qt.id
|
||||
|
|
|
|||
Loading…
Reference in New Issue