search: support for static directory children

This change introduces a new index entry
to help with finding the children of a static directory.
It also fixes ResolvePrefixHop so that it takes
into account static directories, and not only collections.

This is the first step to support publishing static directories.

http://camlistore.org/issue/179

Change-Id: I5666e5caa6c782004054ae4c19a6b6119d4fda8b

pkg/index/index.go

@@ -677,6 +677,31 @@ func (x *Index) EdgesTo(ref blob.Ref, opts *search.EdgesToOpts) (edges []*search
 	return edges, nil
 }
 
+// GetDirMembers sends on dest the children of the static directory dir.
+func (x *Index) GetDirMembers(dir blob.Ref, dest chan<- blob.Ref, limit int) error {
+	defer close(dest)
+
+	sent := 0
+	it := x.queryPrefix(keyStaticDirChild, dir.String())
+	for it.Next() {
+		keyPart := strings.Split(it.Key(), "|")
+		if len(keyPart) != 3 {
+			return fmt.Errorf("index: bogus key keyStaticDirChild = %q", it.Key())
+		}
+
+		child, ok := blob.Parse(keyPart[2])
+		if !ok {
+			continue
+		}
+		dest <- child
+		sent++
+		if sent == limit {
+			break
+		}
+	}
+	return nil
+}
+
 // Storage returns the index's underlying Storage implementation.
 func (x *Index) Storage() Storage { return x.s }
 

pkg/index/indextest/tests.go

@@ -181,6 +181,40 @@ func (id *IndexDeps) UploadFile(fileName string, contents string, modTime time.T
 	return
 }
 
+// If modTime is zero, it's not used.
+func (id *IndexDeps) UploadDir(dirName string, children []blob.Ref, modTime time.Time) blob.Ref {
+	// static-set entries blob
+	ss := new(schema.StaticSet)
+	for _, child := range children {
+		ss.Add(child)
+	}
+	ssjson := ss.Blob().JSON()
+	ssb := &test.Blob{Contents: ssjson}
+	id.BlobSource.AddBlob(ssb)
+	_, err := id.Index.ReceiveBlob(ssb.BlobRef(), ssb.Reader())
+	if err != nil {
+		id.Fatalf("UploadDir.ReceiveBlob: %v", err)
+	}
+
+	// directory blob
+	bb := schema.NewDirMap(dirName)
+	bb.PopulateDirectoryMap(ssb.BlobRef())
+	if !modTime.IsZero() {
+		bb.SetModTime(modTime)
+	}
+	dirjson, err := bb.JSON()
+	if err != nil {
+		id.Fatalf("UploadDir.JSON: %v", err)
+	}
+	dirb := &test.Blob{Contents: dirjson}
+	id.BlobSource.AddBlob(dirb)
+	_, err = id.Index.ReceiveBlob(dirb.BlobRef(), dirb.Reader())
+	if err != nil {
+		id.Fatalf("UploadDir.ReceiveBlob: %v", err)
+	}
+	return dirb.BlobRef()
+}
+
 // NewIndexDeps returns an IndexDeps helper for populating and working
 // with the provided index for tests.
 func NewIndexDeps(index *index.Index) *IndexDeps {
@@ -283,6 +317,13 @@ func Index(t *testing.T, initIdx func() *index.Index) {
 		exifFileRef = uploadFile("dude-exif.jpg", time.Unix(1361248796, 0))
 	}
 
+	// Upload the dir containing the two previous images
+	imagesDirRef := id.UploadDir(
+		"testdata",
+		[]blob.Ref{jpegFileRef, exifFileRef},
+		time.Now(),
+	)
+
 	lastPermanodeMutation := id.lastTime()
 	id.dumpIndex(t)
 
@@ -442,6 +483,36 @@ func Index(t *testing.T, initIdx func() *index.Index) {
 		}
 	}
 
+	// GetDirMembers
+	{
+		ch := make(chan blob.Ref, 10) // expect 2 results
+		err := id.Index.GetDirMembers(imagesDirRef, ch, 50)
+		if err != nil {
+			t.Fatalf("GetDirMembers = %v", err)
+		}
+		got := []blob.Ref{}
+		for r := range ch {
+			got = append(got, r)
+		}
+		want := []blob.Ref{jpegFileRef, exifFileRef}
+		if len(got) != len(want) {
+			t.Errorf("GetDirMembers results differ.\n got: %v\nwant: %v",
+				got, want)
+		}
+		for _, w := range want {
+			found := false
+			for _, g := range got {
+				if w.String() == g.String() {
+					found = true
+					break
+				}
+			}
+			if !found {
+				t.Errorf("GetDirMembers: %v was not found.", w)
+			}
+		}
+	}
+
 	// GetBlobMIMEType
 	{
 		mime, size, err := id.Index.GetBlobMIMEType(pn)

pkg/index/keys.go

@@ -241,6 +241,18 @@ var (
 		},
 	}
 
+	// child of a directory
+	keyStaticDirChild = &keyType{
+		"dirchild",
+		[]part{
+			{"dirref", typeBlobRef}, // blobref of "directory" schema blob
+			{"child", typeStr},      // blobref of the child
+		},
+		[]part{
+			{"1", typeStr},
+		},
+	}
+
 	// Audio attributes (e.g., ID3 tags). Uses generic terms like
 	// "artist", "title", "album", etc.
 	keyAudioTag = &keyType{

pkg/index/receive.go

@@ -285,6 +285,9 @@ func (ix *Index) populateDir(b *schema.Blob, bm BatchMutation) error {
 	}
 
 	bm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(len(sts), b.FileName(), ""))
+	for _, br := range sts {
+		bm.Set(keyStaticDirChild.Key(blobRef, br.String()), "1")
+	}
 	return nil
 }
 

pkg/index/sqlite/sqlite_test.go

@@ -118,8 +118,8 @@ func TestConcurrency(t *testing.T) {
 		i := i
 		go func() {
 			bm := s.BeginBatch()
-			bm.Set("keyA-" + fmt.Sprint(i), fmt.Sprintf("valA=%d", i))
-			bm.Set("keyB-" + fmt.Sprint(i), fmt.Sprintf("valB=%d", i))
+			bm.Set("keyA-"+fmt.Sprint(i), fmt.Sprintf("valA=%d", i))
+			bm.Set("keyB-"+fmt.Sprint(i), fmt.Sprintf("valB=%d", i))
 			ch <- s.CommitBatch(bm)
 		}()
 	}

pkg/schema/schema.go

@@ -571,6 +571,11 @@ func NewFileMap(fileName string) *Builder {
 	return newCommonFilenameMap(fileName).SetType("file")
 }
 
+// NewDirMap returns a new builder of a type "directory" schema for the provided fileName.
+func NewDirMap(fileName string) *Builder {
+	return newCommonFilenameMap(fileName).SetType("directory")
+}
+
 func newCommonFilenameMap(fileName string) *Builder {
 	bb := base(1, "" /* no type yet */)
 	if fileName != "" {

pkg/search/handler.go

@@ -600,6 +600,10 @@ type DescribeRequest struct {
 	// Depth is the optional traversal depth to describe from the
 	// root BlobRef. If zero, a default is used.
 	Depth int
+	// MaxDirChildren is the requested optional limit to the number
+	// of children that should be fetched when describing a static
+	// directory. If zero, a default is used.
+	MaxDirChildren int
 
 	// Internal details, used while loading.
 	// Initialized by sh.initDescribeRequest.
@@ -614,7 +618,7 @@ type DescribeRequest struct {
 
 func (r *DescribeRequest) URLSuffix() string {
 	var buf bytes.Buffer
-	fmt.Fprintf(&buf, "camli/search/describe?depth=%d", r.Depth)
+	fmt.Fprintf(&buf, "camli/search/describe?depth=%d&maxdirchildren=%d", r.depth(), r.maxDirChildren())
 	for _, br := range r.BlobRefs {
 		buf.WriteString("&blobref=")
 		buf.WriteString(br.String())
@@ -641,6 +645,7 @@ func (r *DescribeRequest) fromHTTP(req *http.Request) {
 		r.BlobRef = httputil.MustGetBlobRef(req, "blobref")
 	}
 	r.Depth = httputil.OptionalInt(req, "depth")
+	r.MaxDirChildren = httputil.OptionalInt(req, "maxdirchildren")
 }
 
 type DescribedBlob struct {
@@ -660,6 +665,8 @@ type DescribedBlob struct {
 	Dir *FileInfo `json:"dir,omitempty"`
 	// if camliType "file", and File.IsImage()
 	Image *ImageInfo `json:"image,omitempty"`
+	// if camliType "directory"
+	DirChildren []blob.Ref `json:"dirChildren,omitempty"`
 
 	Thumbnail       string `json:"thumbnailSrc,omitempty"`
 	ThumbnailWidth  int    `json:"thumbnailWidth,omitempty"`
@@ -754,6 +761,18 @@ func (b *DescribedBlob) Members() []*DescribedBlob {
 	return m
 }
 
+func (b *DescribedBlob) DirMembers() []*DescribedBlob {
+	if b == nil || b.Dir == nil || len(b.DirChildren) == 0 {
+		return nil
+	}
+
+	m := make([]*DescribedBlob, 0)
+	for _, br := range b.DirChildren {
+		m = append(m, b.PeerBlob(br))
+	}
+	return m
+}
+
 func (b *DescribedBlob) ContentRef() (br blob.Ref, ok bool) {
 	if b != nil && b.Permanode != nil {
 		if cref := b.Permanode.Attr.Get("camliContent"); cref != "" {
@@ -930,7 +949,8 @@ func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob.
 		return blob.Ref{}, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent)
 	}
 	if des.Permanode != nil {
-		if cr, ok := des.ContentRef(); ok && strings.HasPrefix(cr.Digest(), prefix) {
+		cr, ok := des.ContentRef()
+		if ok && strings.HasPrefix(cr.Digest(), prefix) {
 			return cr, nil
 		}
 		for _, member := range des.Members() {
@@ -938,6 +958,19 @@ func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob.
 				return member.BlobRef, nil
 			}
 		}
+		_, err := dr.DescribeSync(cr)
+		if err != nil {
+			return blob.Ref{}, fmt.Errorf("Failed to describe content %q of parent %q", cr, parent)
+		}
+		if _, _, ok := des.PermanodeDir(); ok {
+			return sh.ResolvePrefixHop(cr, prefix)
+		}
+	} else if des.Dir != nil {
+		for _, child := range des.DirChildren {
+			if strings.HasPrefix(child.Digest(), prefix) {
+				return child, nil
+			}
+		}
 	}
 	return blob.Ref{}, fmt.Errorf("Member prefix %q not found in %q", prefix, parent)
 }
@@ -973,6 +1006,10 @@ func (dr *DescribeRequest) depth() int {
 	return 4
 }
 
+func (dr *DescribeRequest) maxDirChildren() int {
+	return sanitizeNumResults(dr.MaxDirChildren)
+}
+
 func (dr *DescribeRequest) metaMap() (map[string]*DescribedBlob, error) {
 	return dr.metaMapThumbs(0)
 }
@@ -1103,7 +1140,18 @@ func (dr *DescribeRequest) describeReally(br blob.Ref, depth int) {
 			} else {
 				dr.addError(br, err)
 			}
+			return
 		}
+		members, err := dr.getDirMembers(br, depth)
+		if err != nil {
+			if os.IsNotExist(err) {
+				log.Printf("index.GetDirMembers(directory %s) failed; index stale?", br)
+			} else {
+				dr.addError(br, err)
+			}
+			return
+		}
+		des.DirChildren = members
 	}
 }
 
@@ -1234,6 +1282,25 @@ claimLoop:
 	}
 }
 
+func (dr *DescribeRequest) getDirMembers(br blob.Ref, depth int) ([]blob.Ref, error) {
+	limit := dr.maxDirChildren()
+	ch := make(chan blob.Ref)
+	errch := make(chan error)
+	go func() {
+		errch <- dr.sh.index.GetDirMembers(br, ch, limit)
+	}()
+
+	var members []blob.Ref
+	for child := range ch {
+		dr.Describe(child, depth)
+		members = append(members, child)
+	}
+	if err := <-errch; err != nil {
+		return nil, err
+	}
+	return members, nil
+}
+
 // SignerAttrValueResponse is the JSON response to $search/camli/search/signerattrvalue
 type SignerAttrValueResponse struct {
 	Permanode blob.Ref `json:"permanode"`

pkg/search/search.go

@@ -30,7 +30,7 @@ import (
 type Result struct {
 	BlobRef     blob.Ref
 	Signer      blob.Ref // may be nil
-	LastModTime int64            // seconds since epoch; TODO: time.Time?
+	LastModTime int64    // seconds since epoch; TODO: time.Time?
 }
 
 // Results exists mostly for debugging, to provide a String method on
@@ -170,7 +170,7 @@ func (e *Edge) String() string {
 
 type Index interface {
 	// dest must be closed, even when returning an error.
-	// limit is <= 0 for default.  smallest possible default is 0
+	// limit <= 0 means unlimited.
 	GetRecentPermanodes(dest chan *Result,
 		owner blob.Ref,
 		limit int) error
@@ -221,6 +221,13 @@ type Index interface {
 	// Should return os.ErrNotExist if not found.
 	GetImageInfo(fileRef blob.Ref) (*ImageInfo, error)
 
+	// GetDirMembers sends on dest the children of the static
+	// directory dirRef. It returns os.ErrNotExist if dirRef
+	// is nil.
+	// dest must be closed, even when returning an error.
+	// limit <= 0 means unlimited.
+	GetDirMembers(dirRef blob.Ref, dest chan<- blob.Ref, limit int) error
+
 	// Given an owner key, a camliType 'claim', 'attribute' name,
 	// and specific 'value', find the most recent permanode that has
 	// a corresponding 'set-attribute' claim attached.

pkg/test/fakeindex.go

@@ -149,6 +149,10 @@ func (fi *FakeIndex) GetImageInfo(fileRef blob.Ref) (*search.ImageInfo, error) {
 	panic("NOIMPL")
 }
 
+func (fi *FakeIndex) GetDirMembers(dir blob.Ref, dest chan<- blob.Ref, limit int) error {
+	panic("NOIMPL")
+}
+
 func (fi *FakeIndex) PermanodeOfSignerAttrValue(signer blob.Ref, attr, val string) (blob.Ref, error) {
 	fi.lk.Lock()
 	defer fi.lk.Unlock()