diff --git a/pkg/index/index.go b/pkg/index/index.go index aee42a3b2..a085f3007 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -677,6 +677,31 @@ func (x *Index) EdgesTo(ref blob.Ref, opts *search.EdgesToOpts) (edges []*search return edges, nil } +// GetDirMembers sends on dest the children of the static directory dir. +func (x *Index) GetDirMembers(dir blob.Ref, dest chan<- blob.Ref, limit int) error { + defer close(dest) + + sent := 0 + it := x.queryPrefix(keyStaticDirChild, dir.String()) + for it.Next() { + keyPart := strings.Split(it.Key(), "|") + if len(keyPart) != 3 { + return fmt.Errorf("index: bogus key keyStaticDirChild = %q", it.Key()) + } + + child, ok := blob.Parse(keyPart[2]) + if !ok { + continue + } + dest <- child + sent++ + if sent == limit { + break + } + } + return nil +} + // Storage returns the index's underlying Storage implementation. func (x *Index) Storage() Storage { return x.s } diff --git a/pkg/index/indextest/tests.go b/pkg/index/indextest/tests.go index 2a1a3f1c3..7453332a4 100644 --- a/pkg/index/indextest/tests.go +++ b/pkg/index/indextest/tests.go @@ -181,6 +181,40 @@ func (id *IndexDeps) UploadFile(fileName string, contents string, modTime time.T return } +// If modTime is zero, it's not used. +func (id *IndexDeps) UploadDir(dirName string, children []blob.Ref, modTime time.Time) blob.Ref { + // static-set entries blob + ss := new(schema.StaticSet) + for _, child := range children { + ss.Add(child) + } + ssjson := ss.Blob().JSON() + ssb := &test.Blob{Contents: ssjson} + id.BlobSource.AddBlob(ssb) + _, err := id.Index.ReceiveBlob(ssb.BlobRef(), ssb.Reader()) + if err != nil { + id.Fatalf("UploadDir.ReceiveBlob: %v", err) + } + + // directory blob + bb := schema.NewDirMap(dirName) + bb.PopulateDirectoryMap(ssb.BlobRef()) + if !modTime.IsZero() { + bb.SetModTime(modTime) + } + dirjson, err := bb.JSON() + if err != nil { + id.Fatalf("UploadDir.JSON: %v", err) + } + dirb := &test.Blob{Contents: dirjson} + id.BlobSource.AddBlob(dirb) + _, err = id.Index.ReceiveBlob(dirb.BlobRef(), dirb.Reader()) + if err != nil { + id.Fatalf("UploadDir.ReceiveBlob: %v", err) + } + return dirb.BlobRef() +} + // NewIndexDeps returns an IndexDeps helper for populating and working // with the provided index for tests. func NewIndexDeps(index *index.Index) *IndexDeps { @@ -283,6 +317,13 @@ func Index(t *testing.T, initIdx func() *index.Index) { exifFileRef = uploadFile("dude-exif.jpg", time.Unix(1361248796, 0)) } + // Upload the dir containing the two previous images + imagesDirRef := id.UploadDir( + "testdata", + []blob.Ref{jpegFileRef, exifFileRef}, + time.Now(), + ) + lastPermanodeMutation := id.lastTime() id.dumpIndex(t) @@ -442,6 +483,36 @@ func Index(t *testing.T, initIdx func() *index.Index) { } } + // GetDirMembers + { + ch := make(chan blob.Ref, 10) // expect 2 results + err := id.Index.GetDirMembers(imagesDirRef, ch, 50) + if err != nil { + t.Fatalf("GetDirMembers = %v", err) + } + got := []blob.Ref{} + for r := range ch { + got = append(got, r) + } + want := []blob.Ref{jpegFileRef, exifFileRef} + if len(got) != len(want) { + t.Errorf("GetDirMembers results differ.\n got: %v\nwant: %v", + got, want) + } + for _, w := range want { + found := false + for _, g := range got { + if w.String() == g.String() { + found = true + break + } + } + if !found { + t.Errorf("GetDirMembers: %v was not found.", w) + } + } + } + // GetBlobMIMEType { mime, size, err := id.Index.GetBlobMIMEType(pn) diff --git a/pkg/index/keys.go b/pkg/index/keys.go index 26d79905f..736e35a19 100644 --- a/pkg/index/keys.go +++ b/pkg/index/keys.go @@ -241,6 +241,18 @@ var ( }, } + // child of a directory + keyStaticDirChild = &keyType{ + "dirchild", + []part{ + {"dirref", typeBlobRef}, // blobref of "directory" schema blob + {"child", typeStr}, // blobref of the child + }, + []part{ + {"1", typeStr}, + }, + } + // Audio attributes (e.g., ID3 tags). Uses generic terms like // "artist", "title", "album", etc. keyAudioTag = &keyType{ diff --git a/pkg/index/receive.go b/pkg/index/receive.go index 1d3f973f9..69774d230 100644 --- a/pkg/index/receive.go +++ b/pkg/index/receive.go @@ -285,6 +285,9 @@ func (ix *Index) populateDir(b *schema.Blob, bm BatchMutation) error { } bm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(len(sts), b.FileName(), "")) + for _, br := range sts { + bm.Set(keyStaticDirChild.Key(blobRef, br.String()), "1") + } return nil } diff --git a/pkg/index/sqlite/sqlite_test.go b/pkg/index/sqlite/sqlite_test.go index 4fa1b9d78..709fa760e 100644 --- a/pkg/index/sqlite/sqlite_test.go +++ b/pkg/index/sqlite/sqlite_test.go @@ -118,8 +118,8 @@ func TestConcurrency(t *testing.T) { i := i go func() { bm := s.BeginBatch() - bm.Set("keyA-" + fmt.Sprint(i), fmt.Sprintf("valA=%d", i)) - bm.Set("keyB-" + fmt.Sprint(i), fmt.Sprintf("valB=%d", i)) + bm.Set("keyA-"+fmt.Sprint(i), fmt.Sprintf("valA=%d", i)) + bm.Set("keyB-"+fmt.Sprint(i), fmt.Sprintf("valB=%d", i)) ch <- s.CommitBatch(bm) }() } diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 1811c36e3..dab9b06ba 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -571,6 +571,11 @@ func NewFileMap(fileName string) *Builder { return newCommonFilenameMap(fileName).SetType("file") } +// NewDirMap returns a new builder of a type "directory" schema for the provided fileName. +func NewDirMap(fileName string) *Builder { + return newCommonFilenameMap(fileName).SetType("directory") +} + func newCommonFilenameMap(fileName string) *Builder { bb := base(1, "" /* no type yet */) if fileName != "" { diff --git a/pkg/search/handler.go b/pkg/search/handler.go index 5c6a31724..6b23cb584 100644 --- a/pkg/search/handler.go +++ b/pkg/search/handler.go @@ -600,6 +600,10 @@ type DescribeRequest struct { // Depth is the optional traversal depth to describe from the // root BlobRef. If zero, a default is used. Depth int + // MaxDirChildren is the requested optional limit to the number + // of children that should be fetched when describing a static + // directory. If zero, a default is used. + MaxDirChildren int // Internal details, used while loading. // Initialized by sh.initDescribeRequest. @@ -614,7 +618,7 @@ type DescribeRequest struct { func (r *DescribeRequest) URLSuffix() string { var buf bytes.Buffer - fmt.Fprintf(&buf, "camli/search/describe?depth=%d", r.Depth) + fmt.Fprintf(&buf, "camli/search/describe?depth=%d&maxdirchildren=%d", r.depth(), r.maxDirChildren()) for _, br := range r.BlobRefs { buf.WriteString("&blobref=") buf.WriteString(br.String()) @@ -641,6 +645,7 @@ func (r *DescribeRequest) fromHTTP(req *http.Request) { r.BlobRef = httputil.MustGetBlobRef(req, "blobref") } r.Depth = httputil.OptionalInt(req, "depth") + r.MaxDirChildren = httputil.OptionalInt(req, "maxdirchildren") } type DescribedBlob struct { @@ -660,6 +665,8 @@ type DescribedBlob struct { Dir *FileInfo `json:"dir,omitempty"` // if camliType "file", and File.IsImage() Image *ImageInfo `json:"image,omitempty"` + // if camliType "directory" + DirChildren []blob.Ref `json:"dirChildren,omitempty"` Thumbnail string `json:"thumbnailSrc,omitempty"` ThumbnailWidth int `json:"thumbnailWidth,omitempty"` @@ -754,6 +761,18 @@ func (b *DescribedBlob) Members() []*DescribedBlob { return m } +func (b *DescribedBlob) DirMembers() []*DescribedBlob { + if b == nil || b.Dir == nil || len(b.DirChildren) == 0 { + return nil + } + + m := make([]*DescribedBlob, 0) + for _, br := range b.DirChildren { + m = append(m, b.PeerBlob(br)) + } + return m +} + func (b *DescribedBlob) ContentRef() (br blob.Ref, ok bool) { if b != nil && b.Permanode != nil { if cref := b.Permanode.Attr.Get("camliContent"); cref != "" { @@ -930,7 +949,8 @@ func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob. return blob.Ref{}, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent) } if des.Permanode != nil { - if cr, ok := des.ContentRef(); ok && strings.HasPrefix(cr.Digest(), prefix) { + cr, ok := des.ContentRef() + if ok && strings.HasPrefix(cr.Digest(), prefix) { return cr, nil } for _, member := range des.Members() { @@ -938,6 +958,19 @@ func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob. return member.BlobRef, nil } } + _, err := dr.DescribeSync(cr) + if err != nil { + return blob.Ref{}, fmt.Errorf("Failed to describe content %q of parent %q", cr, parent) + } + if _, _, ok := des.PermanodeDir(); ok { + return sh.ResolvePrefixHop(cr, prefix) + } + } else if des.Dir != nil { + for _, child := range des.DirChildren { + if strings.HasPrefix(child.Digest(), prefix) { + return child, nil + } + } } return blob.Ref{}, fmt.Errorf("Member prefix %q not found in %q", prefix, parent) } @@ -973,6 +1006,10 @@ func (dr *DescribeRequest) depth() int { return 4 } +func (dr *DescribeRequest) maxDirChildren() int { + return sanitizeNumResults(dr.MaxDirChildren) +} + func (dr *DescribeRequest) metaMap() (map[string]*DescribedBlob, error) { return dr.metaMapThumbs(0) } @@ -1103,7 +1140,18 @@ func (dr *DescribeRequest) describeReally(br blob.Ref, depth int) { } else { dr.addError(br, err) } + return } + members, err := dr.getDirMembers(br, depth) + if err != nil { + if os.IsNotExist(err) { + log.Printf("index.GetDirMembers(directory %s) failed; index stale?", br) + } else { + dr.addError(br, err) + } + return + } + des.DirChildren = members } } @@ -1234,6 +1282,25 @@ claimLoop: } } +func (dr *DescribeRequest) getDirMembers(br blob.Ref, depth int) ([]blob.Ref, error) { + limit := dr.maxDirChildren() + ch := make(chan blob.Ref) + errch := make(chan error) + go func() { + errch <- dr.sh.index.GetDirMembers(br, ch, limit) + }() + + var members []blob.Ref + for child := range ch { + dr.Describe(child, depth) + members = append(members, child) + } + if err := <-errch; err != nil { + return nil, err + } + return members, nil +} + // SignerAttrValueResponse is the JSON response to $search/camli/search/signerattrvalue type SignerAttrValueResponse struct { Permanode blob.Ref `json:"permanode"` diff --git a/pkg/search/search.go b/pkg/search/search.go index 0dcf22e86..2a3e6d36a 100644 --- a/pkg/search/search.go +++ b/pkg/search/search.go @@ -30,7 +30,7 @@ import ( type Result struct { BlobRef blob.Ref Signer blob.Ref // may be nil - LastModTime int64 // seconds since epoch; TODO: time.Time? + LastModTime int64 // seconds since epoch; TODO: time.Time? } // Results exists mostly for debugging, to provide a String method on @@ -170,7 +170,7 @@ func (e *Edge) String() string { type Index interface { // dest must be closed, even when returning an error. - // limit is <= 0 for default. smallest possible default is 0 + // limit <= 0 means unlimited. GetRecentPermanodes(dest chan *Result, owner blob.Ref, limit int) error @@ -221,6 +221,13 @@ type Index interface { // Should return os.ErrNotExist if not found. GetImageInfo(fileRef blob.Ref) (*ImageInfo, error) + // GetDirMembers sends on dest the children of the static + // directory dirRef. It returns os.ErrNotExist if dirRef + // is nil. + // dest must be closed, even when returning an error. + // limit <= 0 means unlimited. + GetDirMembers(dirRef blob.Ref, dest chan<- blob.Ref, limit int) error + // Given an owner key, a camliType 'claim', 'attribute' name, // and specific 'value', find the most recent permanode that has // a corresponding 'set-attribute' claim attached. diff --git a/pkg/test/fakeindex.go b/pkg/test/fakeindex.go index b898b6ae8..b096dd73d 100644 --- a/pkg/test/fakeindex.go +++ b/pkg/test/fakeindex.go @@ -149,6 +149,10 @@ func (fi *FakeIndex) GetImageInfo(fileRef blob.Ref) (*search.ImageInfo, error) { panic("NOIMPL") } +func (fi *FakeIndex) GetDirMembers(dir blob.Ref, dest chan<- blob.Ref, limit int) error { + panic("NOIMPL") +} + func (fi *FakeIndex) PermanodeOfSignerAttrValue(signer blob.Ref, attr, val string) (blob.Ref, error) { fi.lk.Lock() defer fi.lk.Unlock()