search: new DescribeRule describe support

Step towards getting rid of the the integer describe depth API.

Now, after the integer describe depth, all the DescribeRules will also
run to expand things.

Change-Id: I5cfe7e6058be51328e529a2299e13a6a2ba5f869

pkg/search/describe.go

@@ -0,0 +1,909 @@
+/*
+Copyright 2014 The Camlistore Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package search
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"camlistore.org/pkg/blob"
+	"camlistore.org/pkg/httputil"
+	"camlistore.org/pkg/images"
+	"camlistore.org/pkg/syncutil"
+	"camlistore.org/pkg/types"
+	"camlistore.org/pkg/types/camtypes"
+)
+
+func (sh *Handler) serveDescribe(rw http.ResponseWriter, req *http.Request) {
+	defer httputil.RecoverJSON(rw, req)
+	var dr DescribeRequest
+	dr.fromHTTP(req)
+
+	res, err := sh.Describe(&dr)
+	if err != nil {
+		httputil.ServeJSONError(rw, err)
+		return
+	}
+	httputil.ReturnJSON(rw, res)
+}
+
+func (sh *Handler) Describe(dr *DescribeRequest) (*DescribeResponse, error) {
+	sh.initDescribeRequest(dr)
+	if dr.BlobRef.Valid() {
+		dr.Describe(dr.BlobRef, dr.depth())
+	}
+	for _, br := range dr.BlobRefs {
+		dr.Describe(br, dr.depth())
+	}
+	if err := dr.expandRules(); err != nil {
+		return nil, err
+	}
+	metaMap, err := dr.metaMapThumbs(dr.ThumbnailSize)
+	if err != nil {
+		return nil, err
+	}
+	return &DescribeResponse{metaMap}, nil
+}
+
+type DescribeRequest struct {
+	// BlobRefs are the blobs to describe. If length zero, BlobRef
+	// is used.
+	BlobRefs []blob.Ref `json:"blobrefs,omitempty"`
+
+	// BlobRef is the blob to describe.
+	BlobRef blob.Ref `json:"blobref,omitempty"`
+
+	// Depth is the optional traversal depth to describe from the
+	// root BlobRef. If zero, a default is used.
+	// Depth is deprecated and will be removed. Use Rules instead.
+	Depth int `json:"depth,omitempty"`
+
+	// MaxDirChildren is the requested optional limit to the number
+	// of children that should be fetched when describing a static
+	// directory. If zero, a default is used.
+	MaxDirChildren int `json:"maxDirChildren,omitempty"`
+
+	// At specifies the time which we wish to see the state of
+	// this blob.  If zero (unspecified), all claims will be
+	// considered, otherwise, any claims after this date will not
+	// be considered.
+	At types.Time3339 `json:"at"`
+
+	// ThumbnailSize sets the max dimension for the thumbnail ULR generated,
+	// or zero for none
+	ThumbnailSize int `json:"thumbnailSize,omitempty"`
+
+	// Rules specifies a set of rules to instruct how to keep
+	// expanding the described set. All rules are tested and
+	// matching rules grow the the response set until all rules no
+	// longer match or internal limits are hit.
+	Rules []*DescribeRule `json:"rules,omitempty"`
+
+	// Internal details, used while loading.
+	// Initialized by sh.initDescribeRequest.
+	sh   *Handler
+	mu   sync.Mutex // protects following:
+	m    MetaMap
+	done map[blobrefAndDepth]bool // blobref -> true
+	errs map[string]error         // blobref -> error
+
+	wg *sync.WaitGroup // for load requests
+}
+
+type blobrefAndDepth struct {
+	br    blob.Ref
+	depth int
+}
+
+// Requires dr.mu is held.
+func (dr *DescribeRequest) foreachResultBlob(fn func(blob.Ref)) {
+	if dr.BlobRef.Valid() {
+		fn(dr.BlobRef)
+	}
+	for _, br := range dr.BlobRefs {
+		fn(br)
+	}
+	for brStr := range dr.m {
+		if br, ok := blob.Parse(brStr); ok {
+			fn(br)
+		}
+	}
+}
+
+// Requires dr.mu is held.
+func (dr *DescribeRequest) blobInitiallyRequested(br blob.Ref) bool {
+	if dr.BlobRef.Valid() && dr.BlobRef == br {
+		return true
+	}
+	for _, br1 := range dr.BlobRefs {
+		if br == br1 {
+			return true
+		}
+	}
+	return false
+}
+
+type DescribeRule struct {
+	// All non-zero 'If*' fields in the following set must match
+	// for the rule to match:
+
+	// IsResultRoot, if true, only matches if the blob was part of
+	// the original search results, not a blob expanded later.
+	IfResultRoot bool `json:"ifResultRoot,omitempty"`
+
+	// IfCamliNodeType matches if the "camliNodeType" attribute
+	// equals this value.
+	IfCamliNodeType string `json:"ifCamliNodeType,omitempty"`
+
+	// Attrs lists attributes to describe. A special case
+	// is if the value ends in "*", which matches prefixes
+	// (e.g. "camliPath:*" or "*").
+	Attrs []string `json:"attrs,omitempty"`
+}
+
+// DescribeResponse is the JSON response from $searchRoot/camli/search/describe.
+type DescribeResponse struct {
+	Meta MetaMap `json:"meta"`
+}
+
+// A MetaMap is a map from blobref to a DescribedBlob.
+type MetaMap map[string]*DescribedBlob
+
+type DescribedBlob struct {
+	Request *DescribeRequest `json:"-"`
+
+	BlobRef   blob.Ref `json:"blobRef"`
+	CamliType string   `json:"camliType,omitempty"`
+	Size      int64    `json:"size,"`
+
+	// if camliType "permanode"
+	Permanode *DescribedPermanode `json:"permanode,omitempty"`
+
+	// if camliType "file"
+	File *camtypes.FileInfo `json:"file,omitempty"`
+	// if camliType "directory"
+	Dir *camtypes.FileInfo `json:"dir,omitempty"`
+	// if camliType "file", and File.IsImage()
+	Image *camtypes.ImageInfo `json:"image,omitempty"`
+	// if camliType "file" and media file
+	MediaTags map[string]string `json:"mediaTags,omitempty"`
+
+	// if camliType "directory"
+	DirChildren []blob.Ref `json:"dirChildren,omitempty"`
+
+	Thumbnail       string `json:"thumbnailSrc,omitempty"`
+	ThumbnailWidth  int    `json:"thumbnailWidth,omitempty"`
+	ThumbnailHeight int    `json:"thumbnailHeight,omitempty"`
+
+	// Stub is set if this is not loaded, but referenced.
+	Stub bool `json:"-"`
+}
+
+func (m MetaMap) Get(br blob.Ref) *DescribedBlob {
+	if !br.Valid() {
+		return nil
+	}
+	return m[br.String()]
+}
+
+func (r *DescribeRequest) URLSuffix() string {
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "camli/search/describe?depth=%d&maxdirchildren=%d",
+		r.depth(), r.maxDirChildren())
+	for _, br := range r.BlobRefs {
+		buf.WriteString("&blobref=")
+		buf.WriteString(br.String())
+	}
+	if len(r.BlobRefs) == 0 && r.BlobRef.Valid() {
+		buf.WriteString("&blobref=")
+		buf.WriteString(r.BlobRef.String())
+	}
+	if !r.At.IsZero() {
+		buf.WriteString("&at=")
+		buf.WriteString(r.At.String())
+	}
+	return buf.String()
+}
+
+// fromHTTP panics with an httputil value on failure
+func (r *DescribeRequest) fromHTTP(req *http.Request) {
+	switch {
+	case httputil.IsGet(req):
+		r.fromHTTPGet(req)
+	case req.Method == "POST":
+		r.fromHTTPPost(req)
+	default:
+		panic("Unsupported method")
+	}
+}
+
+func (r *DescribeRequest) fromHTTPPost(req *http.Request) {
+	err := json.NewDecoder(req.Body).Decode(r)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (r *DescribeRequest) fromHTTPGet(req *http.Request) {
+	req.ParseForm()
+	if vv := req.Form["blobref"]; len(vv) > 1 {
+		for _, brs := range vv {
+			if br, ok := blob.Parse(brs); ok {
+				r.BlobRefs = append(r.BlobRefs, br)
+			} else {
+				panic(httputil.InvalidParameterError("blobref"))
+			}
+		}
+	} else {
+		r.BlobRef = httputil.MustGetBlobRef(req, "blobref")
+	}
+	r.Depth = httputil.OptionalInt(req, "depth")
+	r.MaxDirChildren = httputil.OptionalInt(req, "maxdirchildren")
+	r.ThumbnailSize = thumbnailSize(req)
+	r.At = types.ParseTime3339OrZero(req.FormValue("at"))
+}
+
+// PermanodeFile returns in path the blobref of the described permanode
+// and the blobref of its File camliContent.
+// If b isn't a permanode, or doesn't have a camliContent that
+// is a file blob, ok is false.
+func (b *DescribedBlob) PermanodeFile() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
+	if b == nil || b.Permanode == nil {
+		return
+	}
+	if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
+		if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.File != nil {
+			return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.File, true
+		}
+	}
+	return
+}
+
+// PermanodeDir returns in path the blobref of the described permanode
+// and the blobref of its Directory camliContent.
+// If b isn't a permanode, or doesn't have a camliContent that
+// is a directory blob, ok is false.
+func (b *DescribedBlob) PermanodeDir() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
+	if b == nil || b.Permanode == nil {
+		return
+	}
+	if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
+		if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.Dir != nil {
+			return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.Dir, true
+		}
+	}
+	return
+}
+
+func (b *DescribedBlob) DomID() string {
+	if b == nil {
+		return ""
+	}
+	return b.BlobRef.DomID()
+}
+
+func (b *DescribedBlob) Title() string {
+	if b == nil {
+		return ""
+	}
+	if b.Permanode != nil {
+		if t := b.Permanode.Attr.Get("title"); t != "" {
+			return t
+		}
+		if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
+			return b.Request.DescribedBlobStr(contentRef).Title()
+		}
+	}
+	if b.File != nil {
+		return b.File.FileName
+	}
+	if b.Dir != nil {
+		return b.Dir.FileName
+	}
+	return ""
+}
+
+func (b *DescribedBlob) Description() string {
+	if b == nil {
+		return ""
+	}
+	if b.Permanode != nil {
+		return b.Permanode.Attr.Get("description")
+	}
+	return ""
+}
+
+// Members returns all of b's children, as given by b's camliMember and camliPath:*
+// attributes. Only the first entry for a given camliPath attribute is used.
+func (b *DescribedBlob) Members() []*DescribedBlob {
+	if b == nil {
+		return nil
+	}
+	m := make([]*DescribedBlob, 0)
+	if b.Permanode != nil {
+		for _, bstr := range b.Permanode.Attr["camliMember"] {
+			if br, ok := blob.Parse(bstr); ok {
+				m = append(m, b.PeerBlob(br))
+			}
+		}
+		for k, bstrs := range b.Permanode.Attr {
+			if strings.HasPrefix(k, "camliPath:") && len(bstrs) > 0 {
+				if br, ok := blob.Parse(bstrs[0]); ok {
+					m = append(m, b.PeerBlob(br))
+				}
+			}
+		}
+	}
+	return m
+}
+
+func (b *DescribedBlob) DirMembers() []*DescribedBlob {
+	if b == nil || b.Dir == nil || len(b.DirChildren) == 0 {
+		return nil
+	}
+
+	m := make([]*DescribedBlob, 0)
+	for _, br := range b.DirChildren {
+		m = append(m, b.PeerBlob(br))
+	}
+	return m
+}
+
+func (b *DescribedBlob) ContentRef() (br blob.Ref, ok bool) {
+	if b != nil && b.Permanode != nil {
+		if cref := b.Permanode.Attr.Get("camliContent"); cref != "" {
+			return blob.Parse(cref)
+		}
+	}
+	return
+}
+
+// Given a blobref string returns a Description or nil.
+// dr may be nil itself.
+func (dr *DescribeRequest) DescribedBlobStr(blobstr string) *DescribedBlob {
+	if dr == nil {
+		return nil
+	}
+	dr.mu.Lock()
+	defer dr.mu.Unlock()
+	return dr.m[blobstr]
+}
+
+// PeerBlob returns a DescribedBlob for the provided blobref.
+//
+// Unlike DescribedBlobStr, the returned DescribedBlob is never nil.
+//
+// If the blob was never loaded along with the the receiver (or if the
+// receiver is nil), a stub DescribedBlob is returned with its Stub
+// field set true.
+func (b *DescribedBlob) PeerBlob(br blob.Ref) *DescribedBlob {
+	if b.Request == nil {
+		return &DescribedBlob{BlobRef: br, Stub: true}
+	}
+	b.Request.mu.Lock()
+	defer b.Request.mu.Unlock()
+	return b.peerBlob(br)
+}
+
+// version of PeerBlob when b.Request.mu is already held.
+func (b *DescribedBlob) peerBlob(br blob.Ref) *DescribedBlob {
+	if peer, ok := b.Request.m[br.String()]; ok {
+		return peer
+	}
+	return &DescribedBlob{Request: b.Request, BlobRef: br, Stub: true}
+}
+
+// HasSecureLinkTo returns true if there's a valid link from this blob
+// to the other blob. This is used in access control (hence the
+// somewhat redundant "Secure" in the name) and should be paranoid
+// against e.g. random user/attacker-control attributes making links
+// to other blobs.
+//
+// TODO: don't linear scan here.  rewrite this in terms of ResolvePrefixHop,
+// passing down some policy perhaps?  or maybe that's enough.
+func (b *DescribedBlob) HasSecureLinkTo(other blob.Ref) bool {
+	if b == nil || !other.Valid() {
+		return false
+	}
+	ostr := other.String()
+	if b.Permanode != nil {
+		if b.Permanode.Attr.Get("camliContent") == ostr {
+			return true
+		}
+		for _, mstr := range b.Permanode.Attr["camliMember"] {
+			if mstr == ostr {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (b *DescribedBlob) isPermanode() bool {
+	return b.Permanode != nil
+}
+
+// returns a path relative to the UI handler.
+//
+// Locking: requires that DescribedRequest is done loading or that
+// Request.mu is held (as it is from metaMap)
+func (b *DescribedBlob) thumbnail(thumbSize int) (path string, width, height int, ok bool) {
+	if thumbSize <= 0 || !b.isPermanode() {
+		return
+	}
+	if b.Stub {
+		return "node.png", thumbSize, thumbSize, true
+	}
+
+	if b.Permanode.IsContainer() {
+		return "folder.png", thumbSize, thumbSize, true
+	}
+
+	if content, ok := b.ContentRef(); ok {
+		peer := b.peerBlob(content)
+		if peer.File != nil {
+			ii := peer.Image
+			if peer.File.IsImage() && ii != nil && ii.Height > 0 && ii.Width > 0 {
+				image := fmt.Sprintf("thumbnail/%s/%s?mh=%d&tv=%s", peer.BlobRef,
+					url.QueryEscape(peer.File.FileName), thumbSize, images.ThumbnailVersion())
+				mw, mh := images.ScaledDimensions(
+					int(ii.Width), int(ii.Height),
+					MaxImageSize, thumbSize)
+				return image, mw, mh, true
+			}
+
+			// TODO: different thumbnails based on peer.File.MIMEType.
+			const fileIconAspectRatio = 260.0 / 300.0
+			var width = int(math.Floor(float64(thumbSize)*fileIconAspectRatio + 0.5))
+			return "file.png", width, thumbSize, true
+		}
+		if peer.Dir != nil {
+			return "folder.png", thumbSize, thumbSize, true
+		}
+	}
+
+	return "node.png", thumbSize, thumbSize, true
+}
+
+type DescribedPermanode struct {
+	Attr    url.Values `json:"attr"` // a map[string][]string
+	ModTime time.Time  `json:"modtime,omitempty"`
+}
+
+// IsContainer returns whether the permanode has either named ("camliPath:"-prefixed) or unnamed
+// ("camliMember") member attributes.
+func (dp *DescribedPermanode) IsContainer() bool {
+	if members := dp.Attr["camliMember"]; len(members) > 0 {
+		return true
+	}
+	for k := range dp.Attr {
+		if strings.HasPrefix(k, "camliPath:") {
+			return true
+		}
+	}
+	return false
+}
+
+func (dp *DescribedPermanode) jsonMap() map[string]interface{} {
+	m := jsonMap()
+
+	am := jsonMap()
+	m["attr"] = am
+	for k, vv := range dp.Attr {
+		if len(vv) > 0 {
+			vl := make([]string, len(vv))
+			copy(vl[:], vv[:])
+			am[k] = vl
+		}
+	}
+	return m
+}
+
+// NewDescribeRequest returns a new DescribeRequest holding the state
+// of blobs and their summarized descriptions.  Use DescribeBlob
+// one or more times before calling Result.
+func (sh *Handler) NewDescribeRequest() *DescribeRequest {
+	dr := new(DescribeRequest)
+	sh.initDescribeRequest(dr)
+	return dr
+}
+
+func (sh *Handler) initDescribeRequest(req *DescribeRequest) {
+	if req.sh != nil {
+		panic("already initialized")
+	}
+	req.sh = sh
+	req.m = make(MetaMap)
+	req.errs = make(map[string]error)
+	req.wg = new(sync.WaitGroup)
+}
+
+type DescribeError map[string]error
+
+func (de DescribeError) Error() string {
+	var buf bytes.Buffer
+	for b, err := range de {
+		fmt.Fprintf(&buf, "%s: %v; ", b, err)
+	}
+	return fmt.Sprintf("Errors (%d) describing blobs: %s", len(de), buf.String())
+}
+
+// Result waits for all outstanding lookups to complete and
+// returns the map of blobref (strings) to their described
+// results. The returned error is non-nil if any errors
+// occured, and will be of type DescribeError.
+func (dr *DescribeRequest) Result() (desmap map[string]*DescribedBlob, err error) {
+	dr.wg.Wait()
+	// TODO: set "done" / locked flag, so no more DescribeBlob can
+	// be called.
+	if len(dr.errs) > 0 {
+		return dr.m, DescribeError(dr.errs)
+	}
+	return dr.m, nil
+}
+
+func (dr *DescribeRequest) depth() int {
+	if dr.Depth > 0 {
+		return dr.Depth
+	}
+	return 4
+}
+
+func (dr *DescribeRequest) maxDirChildren() int {
+	return sanitizeNumResults(dr.MaxDirChildren)
+}
+
+func (dr *DescribeRequest) metaMap() (map[string]*DescribedBlob, error) {
+	return dr.metaMapThumbs(0)
+}
+
+func (dr *DescribeRequest) metaMapThumbs(thumbSize int) (map[string]*DescribedBlob, error) {
+	// thumbSize of zero means to not include the thumbnails.
+	dr.wg.Wait()
+	dr.mu.Lock()
+	defer dr.mu.Unlock()
+	for k, err := range dr.errs {
+		// TODO: include all?
+		return nil, fmt.Errorf("error populating %s: %v", k, err)
+	}
+	m := make(map[string]*DescribedBlob)
+	for k, desb := range dr.m {
+		m[k] = desb
+		if src, w, h, ok := desb.thumbnail(thumbSize); ok {
+			desb.Thumbnail = src
+			desb.ThumbnailWidth = w
+			desb.ThumbnailHeight = h
+		}
+	}
+	return m, nil
+}
+
+func (dr *DescribeRequest) describedBlob(b blob.Ref) *DescribedBlob {
+	dr.mu.Lock()
+	defer dr.mu.Unlock()
+	bs := b.String()
+	if des, ok := dr.m[bs]; ok {
+		return des
+	}
+	des := &DescribedBlob{Request: dr, BlobRef: b}
+	dr.m[bs] = des
+	return des
+}
+
+func (dr *DescribeRequest) DescribeSync(br blob.Ref) (*DescribedBlob, error) {
+	dr.Describe(br, 1)
+	res, err := dr.Result()
+	if err != nil {
+		return nil, err
+	}
+	return res[br.String()], nil
+}
+
+// Describe starts a lookup of br, down to the provided depth.
+// It returns immediately.
+func (dr *DescribeRequest) Describe(br blob.Ref, depth int) {
+	if depth <= 0 {
+		return
+	}
+	dr.mu.Lock()
+	defer dr.mu.Unlock()
+	if dr.done == nil {
+		dr.done = make(map[blobrefAndDepth]bool)
+	}
+	doneKey := blobrefAndDepth{br, depth}
+	if dr.done[doneKey] {
+		return
+	}
+	dr.done[doneKey] = true
+	dr.wg.Add(1)
+	go func() {
+		defer dr.wg.Done()
+		dr.describeReally(br, depth)
+	}()
+}
+
+// requires dr.mu is held
+func (dr *DescribeRequest) isDescribedOrError(br blob.Ref) bool {
+	brs := br.String()
+	if _, ok := dr.m[brs]; ok {
+		return true
+	}
+	if _, ok := dr.errs[brs]; ok {
+		return true
+	}
+	return false
+}
+
+// requires dr.mu be held.
+func (r *DescribeRule) newMatches(br blob.Ref, dr *DescribeRequest) (brs []blob.Ref) {
+	if r.IfResultRoot {
+		if !dr.blobInitiallyRequested(br) {
+			return nil
+		}
+	}
+	db, ok := dr.m[br.String()]
+	if !ok || db.Permanode == nil {
+		return nil
+	}
+	if t := r.IfCamliNodeType; t != "" {
+		gotType := db.Permanode.Attr.Get("camliNodeType")
+		if gotType != t {
+			return nil
+		}
+	}
+	for attr, vv := range db.Permanode.Attr {
+		matches := false
+		for _, matchAttr := range r.Attrs {
+			if attr == matchAttr {
+				matches = true
+				break
+			}
+			if strings.HasSuffix(matchAttr, "*") && strings.HasPrefix(attr, strings.TrimSuffix(matchAttr, "*")) {
+				matches = true
+				break
+			}
+		}
+		if !matches {
+			continue
+		}
+		for _, v := range vv {
+			if br, ok := blob.Parse(v); ok {
+				brs = append(brs, br)
+			}
+		}
+	}
+	return brs
+}
+
+func (dr *DescribeRequest) expandRules() error {
+	loop := true
+	for loop {
+		loop = false
+		dr.wg.Wait()
+		dr.mu.Lock()
+		len0 := len(dr.m)
+		var new []blob.Ref
+		for _, rule := range dr.Rules {
+			dr.foreachResultBlob(func(br blob.Ref) {
+				for _, nbr := range rule.newMatches(br, dr) {
+					new = append(new, nbr)
+				}
+			})
+		}
+		dr.mu.Unlock()
+		for _, br := range new {
+			dr.Describe(br, 1)
+		}
+		dr.wg.Wait()
+		dr.mu.Lock()
+		len1 := len(dr.m)
+		dr.mu.Unlock()
+		loop = len0 != len1
+	}
+	return nil
+}
+
+func (dr *DescribeRequest) addError(br blob.Ref, err error) {
+	if err == nil {
+		return
+	}
+	dr.mu.Lock()
+	defer dr.mu.Unlock()
+	// TODO: append? meh.
+	dr.errs[br.String()] = err
+}
+
+func (dr *DescribeRequest) describeReally(br blob.Ref, depth int) {
+	meta, err := dr.sh.index.GetBlobMeta(br)
+	if err == os.ErrNotExist {
+		return
+	}
+	if err != nil {
+		dr.addError(br, err)
+		return
+	}
+
+	// TODO: convert all this in terms of
+	// DescribedBlob/DescribedPermanode/DescribedFile, not json
+	// maps.  Then add JSON marhsallers to those types. Add tests.
+	des := dr.describedBlob(br)
+	if meta.CamliType != "" {
+		des.setMIMEType("application/json; camliType=" + meta.CamliType)
+	}
+	des.Size = int64(meta.Size)
+
+	switch des.CamliType {
+	case "permanode":
+		des.Permanode = new(DescribedPermanode)
+		dr.populatePermanodeFields(des.Permanode, br, dr.sh.owner, depth)
+	case "file":
+		fi, err := dr.sh.index.GetFileInfo(br)
+		if err != nil {
+			if os.IsNotExist(err) {
+				log.Printf("index.GetFileInfo(file %s) failed; index stale?", br)
+			} else {
+				dr.addError(br, err)
+			}
+			return
+		}
+		des.File = &fi
+		if des.File.IsImage() && !skipImageInfoLookup(des.File) {
+			imgInfo, err := dr.sh.index.GetImageInfo(br)
+			if err != nil {
+				if os.IsNotExist(err) {
+					log.Printf("index.GetImageInfo(file %s) failed; index stale?", br)
+				} else {
+					dr.addError(br, err)
+				}
+			} else {
+				des.Image = &imgInfo
+			}
+		}
+		if mediaTags, err := dr.sh.index.GetMediaTags(br); err == nil {
+			des.MediaTags = mediaTags
+		}
+	case "directory":
+		var g syncutil.Group
+		g.Go(func() (err error) {
+			fi, err := dr.sh.index.GetFileInfo(br)
+			if os.IsNotExist(err) {
+				log.Printf("index.GetFileInfo(directory %s) failed; index stale?", br)
+			}
+			if err == nil {
+				des.Dir = &fi
+			}
+			return
+		})
+		g.Go(func() (err error) {
+			des.DirChildren, err = dr.getDirMembers(br, depth)
+			return
+		})
+		if err := g.Err(); err != nil {
+			dr.addError(br, err)
+		}
+	}
+}
+
+func (dr *DescribeRequest) populatePermanodeFields(pi *DescribedPermanode, pn, signer blob.Ref, depth int) {
+	pi.Attr = make(url.Values)
+	attr := pi.Attr
+
+	claims, err := dr.sh.index.AppendClaims(nil, pn, signer, "")
+	if err != nil {
+		log.Printf("Error getting claims of %s: %v", pn.String(), err)
+		dr.addError(pn, fmt.Errorf("Error getting claims of %s: %v", pn.String(), err))
+		return
+	}
+
+	sort.Sort(camtypes.ClaimsByDate(claims))
+claimLoop:
+	for _, cl := range claims {
+		if !dr.At.IsZero() {
+			if cl.Date.After(dr.At.Time()) {
+				continue
+			}
+		}
+		switch cl.Type {
+		default:
+			continue
+		case "del-attribute":
+			if cl.Value == "" {
+				delete(attr, cl.Attr)
+			} else {
+				sl := attr[cl.Attr]
+				filtered := make([]string, 0, len(sl))
+				for _, val := range sl {
+					if val != cl.Value {
+						filtered = append(filtered, val)
+					}
+				}
+				attr[cl.Attr] = filtered
+			}
+		case "set-attribute":
+			delete(attr, cl.Attr)
+			fallthrough
+		case "add-attribute":
+			if cl.Value == "" {
+				continue
+			}
+			sl, ok := attr[cl.Attr]
+			if ok {
+				for _, exist := range sl {
+					if exist == cl.Value {
+						continue claimLoop
+					}
+				}
+			} else {
+				sl = make([]string, 0, 1)
+				attr[cl.Attr] = sl
+			}
+			attr[cl.Attr] = append(sl, cl.Value)
+		}
+		pi.ModTime = cl.Date
+	}
+
+	// Descend into any references in current attributes.
+	for key, vals := range attr {
+		dr.describeRefs(key, depth)
+		for _, v := range vals {
+			dr.describeRefs(v, depth)
+		}
+	}
+}
+
+func (dr *DescribeRequest) getDirMembers(br blob.Ref, depth int) ([]blob.Ref, error) {
+	limit := dr.maxDirChildren()
+	ch := make(chan blob.Ref)
+	errch := make(chan error)
+	go func() {
+		errch <- dr.sh.index.GetDirMembers(br, ch, limit)
+	}()
+
+	var members []blob.Ref
+	for child := range ch {
+		dr.Describe(child, depth)
+		members = append(members, child)
+	}
+	if err := <-errch; err != nil {
+		return nil, err
+	}
+	return members, nil
+}
+
+func (dr *DescribeRequest) describeRefs(str string, depth int) {
+	for _, match := range blobRefPattern.FindAllString(str, -1) {
+		if ref, ok := blob.ParseKnown(match); ok {
+			dr.Describe(ref, depth-1)
+		}
+	}
+}
+
+func (d *DescribedBlob) setMIMEType(mime string) {
+	if strings.HasPrefix(mime, camliTypePrefix) {
+		d.CamliType = strings.TrimPrefix(mime, camliTypePrefix)
+	}
+}

pkg/search/describe_test.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2014 The Camlistore Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package search_test
+
+import (
+	"testing"
+
+	"camlistore.org/pkg/blob"
+	"camlistore.org/pkg/index"
+	"camlistore.org/pkg/search"
+	"camlistore.org/pkg/test"
+)
+
+func addPermanode(fi *test.FakeIndex, pnStr string, attrs ...string) {
+	pn := blob.MustParse(pnStr)
+	fi.AddMeta(pn, "permanode", 123)
+	for len(attrs) > 0 {
+		k, v := attrs[0], attrs[1]
+		attrs = attrs[2:]
+		fi.AddClaim(owner, pn, "add-attribute", k, v)
+	}
+}
+
+func searchDescribeSetup(fi *test.FakeIndex) index.Interface {
+	addPermanode(fi, "abc-123",
+		"camliContent", "abc-123c",
+		"camliImageContent", "abc-888",
+	)
+	addPermanode(fi, "abc-123c",
+		"camliContent", "abc-123cc",
+		"camliImageContent", "abc-123c1",
+	)
+	addPermanode(fi, "abc-123c1",
+		"some", "image",
+	)
+	addPermanode(fi, "abc-123cc",
+		"name", "leaf",
+	)
+	addPermanode(fi, "abc-888",
+		"camliContent", "abc-8881",
+	)
+	addPermanode(fi, "abc-8881",
+		"name", "leaf8881",
+	)
+	return fi
+}
+
+var searchDescribeTests = []handlerTest{
+	{
+		name:     "null",
+		postBody: marshalJSON(&search.DescribeRequest{}),
+		want: jmap(&search.DescribeResponse{
+			Meta: search.MetaMap{},
+		}),
+	},
+
+	{
+		name: "single",
+		postBody: marshalJSON(&search.DescribeRequest{
+			BlobRef: blob.MustParse("abc-123"),
+		}),
+		wantDescribed: []string{"abc-123"},
+	},
+
+	{
+		name: "follow all camliContent",
+		postBody: marshalJSON(&search.DescribeRequest{
+			BlobRef: blob.MustParse("abc-123"),
+			Rules: []*search.DescribeRule{
+				{
+					Attrs: []string{"camliContent"},
+				},
+			},
+		}),
+		wantDescribed: []string{"abc-123", "abc-123c", "abc-123cc"},
+	},
+
+	{
+		name: "follow only root camliContent",
+		postBody: marshalJSON(&search.DescribeRequest{
+			BlobRef: blob.MustParse("abc-123"),
+			Rules: []*search.DescribeRule{
+				{
+					IfResultRoot: true,
+					Attrs:        []string{"camliContent"},
+				},
+			},
+		}),
+		wantDescribed: []string{"abc-123", "abc-123c"},
+	},
+
+	{
+		name: "follow all root, substring",
+		postBody: marshalJSON(&search.DescribeRequest{
+			BlobRef: blob.MustParse("abc-123"),
+			Rules: []*search.DescribeRule{
+				{
+					IfResultRoot: true,
+					Attrs:        []string{"camli*"},
+				},
+			},
+		}),
+		wantDescribed: []string{"abc-123", "abc-123c", "abc-888"},
+	},
+
+	{
+		name: "two rules, two attrs",
+		postBody: marshalJSON(&search.DescribeRequest{
+			BlobRef: blob.MustParse("abc-123"),
+			Rules: []*search.DescribeRule{
+				{
+					IfResultRoot: true,
+					Attrs:        []string{"camliContent", "camliImageContent"},
+				},
+				{
+					Attrs: []string{"camliContent"},
+				},
+			},
+		}),
+		wantDescribed: []string{"abc-123", "abc-123c", "abc-123cc", "abc-888", "abc-8881"},
+	},
+}
+
+func init() {
+	checkNoDups("searchDescribeTests", searchDescribeTests)
+}
+
+func TestSearchDescribe(t *testing.T) {
+	for _, ht := range searchDescribeTests {
+		if ht.setup == nil {
+			ht.setup = searchDescribeSetup
+		}
+		if ht.query == "" {
+			ht.query = "describe"
+		}
+		ht.test(t)
+	}
+}

pkg/search/handler.go

@@ -20,25 +20,19 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"log"
-	"math"
 	"net/http"
 	"net/url"
-	"os"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"camlistore.org/pkg/blob"
 	"camlistore.org/pkg/blobserver"
 	"camlistore.org/pkg/httputil"
-	"camlistore.org/pkg/images"
 	"camlistore.org/pkg/index"
 	"camlistore.org/pkg/jsonconfig"
-	"camlistore.org/pkg/syncutil"
 	"camlistore.org/pkg/types"
 	"camlistore.org/pkg/types/camtypes"
 )
@@ -181,26 +175,31 @@ var getHandler = map[string]func(*Handler, http.ResponseWriter, *http.Request){
 	"edgesto":         (*Handler).serveEdgesTo,
 }
 
+var postHandler = map[string]func(*Handler, http.ResponseWriter, *http.Request){
+	"describe": (*Handler).serveDescribe,
+	"query":    (*Handler).serveQuery,
+}
+
 func (sh *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	ret := jsonMap()
 	suffix := httputil.PathSuffix(req)
 
-	if httputil.IsGet(req) {
+	handlers := getHandler
-		fn := getHandler[strings.TrimPrefix(suffix, "camli/search/")]
+	switch {
-		if fn != nil {
+	case httputil.IsGet(req):
-			fn(sh, rw, req)
+		// use default from above
-			return
+	case req.Method == "POST":
-		}
+		handlers = postHandler
+	default:
+		handlers = nil
 	}
-	if req.Method == "POST" {
+	fn := handlers[strings.TrimPrefix(suffix, "camli/search/")]
-		switch suffix {
+	if fn != nil {
-		case "camli/search/query":
+		fn(sh, rw, req)
-			sh.serveQuery(rw, req)
+		return
-			return
-		}
 	}
 
 	// TODO: discovery for the endpoints & better error message with link to discovery info
+	ret := jsonMap()
 	ret["error"] = "Unsupported search path or method"
 	ret["errorType"] = "input"
 	httputil.ReturnJSON(rw, ret)
@@ -365,16 +364,6 @@ func (r *EdgesRequest) fromHTTP(req *http.Request) {
 	r.ToRef = httputil.MustGetBlobRef(req, "blobref")
 }
 
-// A MetaMap is a map from blobref to a DescribedBlob.
-type MetaMap map[string]*DescribedBlob
-
-func (m MetaMap) Get(br blob.Ref) *DescribedBlob {
-	if !br.Valid() {
-		return nil
-	}
-	return m[br.String()]
-}
-
 // TODO(mpl): it looks like we never populate RecentResponse.Error*, shouldn't we remove them?
 // Same for WithAttrResponse. I suppose it doesn't matter much if we end up removing GetRecentPermanodes anyway...
 
@@ -397,11 +386,6 @@ func (r *RecentResponse) Err() error {
 	return nil
 }
 
-// DescribeResponse is the JSON response from $searchRoot/camli/search/describe.
-type DescribeResponse struct {
-	Meta MetaMap `json:"meta"`
-}
-
 // WithAttrResponse is the JSON response from $searchRoot/camli/search/permanodeattr.
 type WithAttrResponse struct {
 	WithAttr []*WithAttrItem `json:"withAttr"`
@@ -655,387 +639,6 @@ func (sh *Handler) serveClaims(rw http.ResponseWriter, req *http.Request) {
 	httputil.ReturnJSON(rw, res)
 }
 
-type DescribeRequest struct {
-	// BlobRefs are the blobs to describe. If length zero, BlobRef
-	// is used.
-	BlobRefs []blob.Ref `json:"blobrefs,omitempty"`
-
-	// BlobRef is the blob to describe.
-	BlobRef blob.Ref `json:"blobref,omitempty"`
-
-	// Depth is the optional traversal depth to describe from the
-	// root BlobRef. If zero, a default is used.
-	Depth int `json:"depth,omitempty"`
-	// MaxDirChildren is the requested optional limit to the number
-	// of children that should be fetched when describing a static
-	// directory. If zero, a default is used.
-	MaxDirChildren int `json:"maxDirChildren,omitempty"`
-
-	// At specifies the time which we wish to see the state of
-	// this blob.  If zero (unspecified), all claims will be
-	// considered, otherwise, any claims after this date will not
-	// be considered.
-	At types.Time3339 `json:"at"`
-
-	// ThumbnailSize sets the max dimension for the thumbnail ULR generated,
-	// or zero for none
-	ThumbnailSize int `json:"thumbnailSize,omitempty"`
-
-	// Internal details, used while loading.
-	// Initialized by sh.initDescribeRequest.
-	sh   *Handler
-	mu   sync.Mutex // protects following:
-	m    MetaMap
-	done map[string]bool  // blobref -> described
-	errs map[string]error // blobref -> error
-
-	wg *sync.WaitGroup // for load requests
-}
-
-func (r *DescribeRequest) URLSuffix() string {
-	var buf bytes.Buffer
-	fmt.Fprintf(&buf, "camli/search/describe?depth=%d&maxdirchildren=%d",
-		r.depth(), r.maxDirChildren())
-	for _, br := range r.BlobRefs {
-		buf.WriteString("&blobref=")
-		buf.WriteString(br.String())
-	}
-	if len(r.BlobRefs) == 0 && r.BlobRef.Valid() {
-		buf.WriteString("&blobref=")
-		buf.WriteString(r.BlobRef.String())
-	}
-	if !r.At.IsZero() {
-		buf.WriteString("&at=")
-		buf.WriteString(r.At.String())
-	}
-	return buf.String()
-}
-
-// fromHTTP panics with an httputil value on failure
-func (r *DescribeRequest) fromHTTP(req *http.Request) {
-	req.ParseForm()
-	if vv := req.Form["blobref"]; len(vv) > 1 {
-		for _, brs := range vv {
-			if br, ok := blob.Parse(brs); ok {
-				r.BlobRefs = append(r.BlobRefs, br)
-			} else {
-				panic(httputil.InvalidParameterError("blobref"))
-			}
-		}
-	} else {
-		r.BlobRef = httputil.MustGetBlobRef(req, "blobref")
-	}
-	r.Depth = httputil.OptionalInt(req, "depth")
-	r.MaxDirChildren = httputil.OptionalInt(req, "maxdirchildren")
-	r.ThumbnailSize = thumbnailSize(req)
-	r.At = types.ParseTime3339OrZero(req.FormValue("at"))
-}
-
-type DescribedBlob struct {
-	Request *DescribeRequest `json:"-"`
-
-	BlobRef   blob.Ref `json:"blobRef"`
-	CamliType string   `json:"camliType,omitempty"`
-	Size      int64    `json:"size,"`
-
-	// if camliType "permanode"
-	Permanode *DescribedPermanode `json:"permanode,omitempty"`
-
-	// if camliType "file"
-	File *camtypes.FileInfo `json:"file,omitempty"`
-	// if camliType "directory"
-	Dir *camtypes.FileInfo `json:"dir,omitempty"`
-	// if camliType "file", and File.IsImage()
-	Image *camtypes.ImageInfo `json:"image,omitempty"`
-	// if camliType "file" and media file
-	MediaTags map[string]string `json:"mediaTags,omitempty"`
-
-	// if camliType "directory"
-	DirChildren []blob.Ref `json:"dirChildren,omitempty"`
-
-	Thumbnail       string `json:"thumbnailSrc,omitempty"`
-	ThumbnailWidth  int    `json:"thumbnailWidth,omitempty"`
-	ThumbnailHeight int    `json:"thumbnailHeight,omitempty"`
-
-	// Stub is set if this is not loaded, but referenced.
-	Stub bool `json:"-"`
-}
-
-// PermanodeFile returns in path the blobref of the described permanode
-// and the blobref of its File camliContent.
-// If b isn't a permanode, or doesn't have a camliContent that
-// is a file blob, ok is false.
-func (b *DescribedBlob) PermanodeFile() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
-	if b == nil || b.Permanode == nil {
-		return
-	}
-	if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
-		if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.File != nil {
-			return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.File, true
-		}
-	}
-	return
-}
-
-// PermanodeDir returns in path the blobref of the described permanode
-// and the blobref of its Directory camliContent.
-// If b isn't a permanode, or doesn't have a camliContent that
-// is a directory blob, ok is false.
-func (b *DescribedBlob) PermanodeDir() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
-	if b == nil || b.Permanode == nil {
-		return
-	}
-	if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
-		if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.Dir != nil {
-			return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.Dir, true
-		}
-	}
-	return
-}
-
-func (b *DescribedBlob) DomID() string {
-	if b == nil {
-		return ""
-	}
-	return b.BlobRef.DomID()
-}
-
-func (b *DescribedBlob) Title() string {
-	if b == nil {
-		return ""
-	}
-	if b.Permanode != nil {
-		if t := b.Permanode.Attr.Get("title"); t != "" {
-			return t
-		}
-		if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
-			return b.Request.DescribedBlobStr(contentRef).Title()
-		}
-	}
-	if b.File != nil {
-		return b.File.FileName
-	}
-	if b.Dir != nil {
-		return b.Dir.FileName
-	}
-	return ""
-}
-
-func (b *DescribedBlob) Description() string {
-	if b == nil {
-		return ""
-	}
-	if b.Permanode != nil {
-		return b.Permanode.Attr.Get("description")
-	}
-	return ""
-}
-
-// Members returns all of b's children, as given by b's camliMember and camliPath:*
-// attributes. Only the first entry for a given camliPath attribute is used.
-func (b *DescribedBlob) Members() []*DescribedBlob {
-	if b == nil {
-		return nil
-	}
-	m := make([]*DescribedBlob, 0)
-	if b.Permanode != nil {
-		for _, bstr := range b.Permanode.Attr["camliMember"] {
-			if br, ok := blob.Parse(bstr); ok {
-				m = append(m, b.PeerBlob(br))
-			}
-		}
-		for k, bstrs := range b.Permanode.Attr {
-			if strings.HasPrefix(k, "camliPath:") && len(bstrs) > 0 {
-				if br, ok := blob.Parse(bstrs[0]); ok {
-					m = append(m, b.PeerBlob(br))
-				}
-			}
-		}
-	}
-	return m
-}
-
-func (b *DescribedBlob) DirMembers() []*DescribedBlob {
-	if b == nil || b.Dir == nil || len(b.DirChildren) == 0 {
-		return nil
-	}
-
-	m := make([]*DescribedBlob, 0)
-	for _, br := range b.DirChildren {
-		m = append(m, b.PeerBlob(br))
-	}
-	return m
-}
-
-func (b *DescribedBlob) ContentRef() (br blob.Ref, ok bool) {
-	if b != nil && b.Permanode != nil {
-		if cref := b.Permanode.Attr.Get("camliContent"); cref != "" {
-			return blob.Parse(cref)
-		}
-	}
-	return
-}
-
-// Given a blobref string returns a Description or nil.
-// dr may be nil itself.
-func (dr *DescribeRequest) DescribedBlobStr(blobstr string) *DescribedBlob {
-	if dr == nil {
-		return nil
-	}
-	dr.mu.Lock()
-	defer dr.mu.Unlock()
-	return dr.m[blobstr]
-}
-
-// PeerBlob returns a DescribedBlob for the provided blobref.
-//
-// Unlike DescribedBlobStr, the returned DescribedBlob is never nil.
-//
-// If the blob was never loaded along with the the receiver (or if the
-// receiver is nil), a stub DescribedBlob is returned with its Stub
-// field set true.
-func (b *DescribedBlob) PeerBlob(br blob.Ref) *DescribedBlob {
-	if b.Request == nil {
-		return &DescribedBlob{BlobRef: br, Stub: true}
-	}
-	b.Request.mu.Lock()
-	defer b.Request.mu.Unlock()
-	return b.peerBlob(br)
-}
-
-// version of PeerBlob when b.Request.mu is already held.
-func (b *DescribedBlob) peerBlob(br blob.Ref) *DescribedBlob {
-	if peer, ok := b.Request.m[br.String()]; ok {
-		return peer
-	}
-	return &DescribedBlob{Request: b.Request, BlobRef: br, Stub: true}
-}
-
-// HasSecureLinkTo returns true if there's a valid link from this blob
-// to the other blob. This is used in access control (hence the
-// somewhat redundant "Secure" in the name) and should be paranoid
-// against e.g. random user/attacker-control attributes making links
-// to other blobs.
-//
-// TODO: don't linear scan here.  rewrite this in terms of ResolvePrefixHop,
-// passing down some policy perhaps?  or maybe that's enough.
-func (b *DescribedBlob) HasSecureLinkTo(other blob.Ref) bool {
-	if b == nil || !other.Valid() {
-		return false
-	}
-	ostr := other.String()
-	if b.Permanode != nil {
-		if b.Permanode.Attr.Get("camliContent") == ostr {
-			return true
-		}
-		for _, mstr := range b.Permanode.Attr["camliMember"] {
-			if mstr == ostr {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-func (b *DescribedBlob) isPermanode() bool {
-	return b.Permanode != nil
-}
-
-// returns a path relative to the UI handler.
-//
-// Locking: requires that DescribedRequest is done loading or that
-// Request.mu is held (as it is from metaMap)
-func (b *DescribedBlob) thumbnail(thumbSize int) (path string, width, height int, ok bool) {
-	if thumbSize <= 0 || !b.isPermanode() {
-		return
-	}
-	if b.Stub {
-		return "node.png", thumbSize, thumbSize, true
-	}
-
-	if b.Permanode.IsContainer() {
-		return "folder.png", thumbSize, thumbSize, true
-	}
-
-	if content, ok := b.ContentRef(); ok {
-		peer := b.peerBlob(content)
-		if peer.File != nil {
-			ii := peer.Image
-			if peer.File.IsImage() && ii != nil && ii.Height > 0 && ii.Width > 0 {
-				image := fmt.Sprintf("thumbnail/%s/%s?mh=%d&tv=%s", peer.BlobRef,
-					url.QueryEscape(peer.File.FileName), thumbSize, images.ThumbnailVersion())
-				mw, mh := images.ScaledDimensions(
-					int(ii.Width), int(ii.Height),
-					MaxImageSize, thumbSize)
-				return image, mw, mh, true
-			}
-
-			// TODO: different thumbnails based on peer.File.MIMEType.
-			const fileIconAspectRatio = 260.0 / 300.0
-			var width = int(math.Floor(float64(thumbSize)*fileIconAspectRatio + 0.5))
-			return "file.png", width, thumbSize, true
-		}
-		if peer.Dir != nil {
-			return "folder.png", thumbSize, thumbSize, true
-		}
-	}
-
-	return "node.png", thumbSize, thumbSize, true
-}
-
-type DescribedPermanode struct {
-	Attr    url.Values `json:"attr"` // a map[string][]string
-	ModTime time.Time  `json:"modtime,omitempty"`
-}
-
-// IsContainer returns whether the permanode has either named ("camliPath:"-prefixed) or unnamed
-// ("camliMember") member attributes.
-func (dp *DescribedPermanode) IsContainer() bool {
-	if members := dp.Attr["camliMember"]; len(members) > 0 {
-		return true
-	}
-	for k := range dp.Attr {
-		if strings.HasPrefix(k, "camliPath:") {
-			return true
-		}
-	}
-	return false
-}
-
-func (dp *DescribedPermanode) jsonMap() map[string]interface{} {
-	m := jsonMap()
-
-	am := jsonMap()
-	m["attr"] = am
-	for k, vv := range dp.Attr {
-		if len(vv) > 0 {
-			vl := make([]string, len(vv))
-			copy(vl[:], vv[:])
-			am[k] = vl
-		}
-	}
-	return m
-}
-
-// NewDescribeRequest returns a new DescribeRequest holding the state
-// of blobs and their summarized descriptions.  Use DescribeBlob
-// one or more times before calling Result.
-func (sh *Handler) NewDescribeRequest() *DescribeRequest {
-	dr := new(DescribeRequest)
-	sh.initDescribeRequest(dr)
-	return dr
-}
-
-func (sh *Handler) initDescribeRequest(req *DescribeRequest) {
-	if req.sh != nil {
-		panic("already initialized")
-	}
-	req.sh = sh
-	req.m = make(MetaMap)
-	req.errs = make(map[string]error)
-	req.wg = new(sync.WaitGroup)
-}
-
 // Given a blobref and a few hex characters of the digest of the next hop, return the complete
 // blobref of the prefix, if that's a valid next hop.
 func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob.Ref, err error) {
@@ -1083,219 +686,6 @@ func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob.
 	return blob.Ref{}, fmt.Errorf("Member prefix %q not found in %q", prefix, parent)
 }
 
-type DescribeError map[string]error
-
-func (de DescribeError) Error() string {
-	var buf bytes.Buffer
-	for b, err := range de {
-		fmt.Fprintf(&buf, "%s: %v; ", b, err)
-	}
-	return fmt.Sprintf("Errors (%d) describing blobs: %s", len(de), buf.String())
-}
-
-// Result waits for all outstanding lookups to complete and
-// returns the map of blobref (strings) to their described
-// results. The returned error is non-nil if any errors
-// occured, and will be of type DescribeError.
-func (dr *DescribeRequest) Result() (desmap map[string]*DescribedBlob, err error) {
-	dr.wg.Wait()
-	// TODO: set "done" / locked flag, so no more DescribeBlob can
-	// be called.
-	if len(dr.errs) > 0 {
-		return dr.m, DescribeError(dr.errs)
-	}
-	return dr.m, nil
-}
-
-func (dr *DescribeRequest) depth() int {
-	if dr.Depth > 0 {
-		return dr.Depth
-	}
-	return 4
-}
-
-func (dr *DescribeRequest) maxDirChildren() int {
-	return sanitizeNumResults(dr.MaxDirChildren)
-}
-
-func (dr *DescribeRequest) metaMap() (map[string]*DescribedBlob, error) {
-	return dr.metaMapThumbs(0)
-}
-
-func (dr *DescribeRequest) metaMapThumbs(thumbSize int) (map[string]*DescribedBlob, error) {
-	// thumbSize of zero means to not include the thumbnails.
-	dr.wg.Wait()
-	dr.mu.Lock()
-	defer dr.mu.Unlock()
-	for k, err := range dr.errs {
-		// TODO: include all?
-		return nil, fmt.Errorf("error populating %s: %v", k, err)
-	}
-	m := make(map[string]*DescribedBlob)
-	for k, desb := range dr.m {
-		m[k] = desb
-		if src, w, h, ok := desb.thumbnail(thumbSize); ok {
-			desb.Thumbnail = src
-			desb.ThumbnailWidth = w
-			desb.ThumbnailHeight = h
-		}
-	}
-	return m, nil
-}
-
-func (dr *DescribeRequest) describedBlob(b blob.Ref) *DescribedBlob {
-	dr.mu.Lock()
-	defer dr.mu.Unlock()
-	bs := b.String()
-	if des, ok := dr.m[bs]; ok {
-		return des
-	}
-	des := &DescribedBlob{Request: dr, BlobRef: b}
-	dr.m[bs] = des
-	return des
-}
-
-func (dr *DescribeRequest) DescribeSync(br blob.Ref) (*DescribedBlob, error) {
-	dr.Describe(br, 1)
-	res, err := dr.Result()
-	if err != nil {
-		return nil, err
-	}
-	return res[br.String()], nil
-}
-
-// Describe starts a lookup of br, down to the provided depth.
-// It returns immediately.
-func (dr *DescribeRequest) Describe(br blob.Ref, depth int) {
-	if depth <= 0 {
-		return
-	}
-	dr.mu.Lock()
-	defer dr.mu.Unlock()
-	if dr.done == nil {
-		dr.done = make(map[string]bool)
-	}
-	brefAndDepth := fmt.Sprintf("%s-%d", br, depth)
-	if dr.done[brefAndDepth] {
-		return
-	}
-	dr.done[brefAndDepth] = true
-	dr.wg.Add(1)
-	go func() {
-		defer dr.wg.Done()
-		dr.describeReally(br, depth)
-	}()
-}
-
-func (dr *DescribeRequest) addError(br blob.Ref, err error) {
-	if err == nil {
-		return
-	}
-	dr.mu.Lock()
-	defer dr.mu.Unlock()
-	// TODO: append? meh.
-	dr.errs[br.String()] = err
-}
-
-func (dr *DescribeRequest) describeReally(br blob.Ref, depth int) {
-	meta, err := dr.sh.index.GetBlobMeta(br)
-	if err == os.ErrNotExist {
-		return
-	}
-	if err != nil {
-		dr.addError(br, err)
-		return
-	}
-
-	// TODO: convert all this in terms of
-	// DescribedBlob/DescribedPermanode/DescribedFile, not json
-	// maps.  Then add JSON marhsallers to those types. Add tests.
-	des := dr.describedBlob(br)
-	if meta.CamliType != "" {
-		des.setMIMEType("application/json; camliType=" + meta.CamliType)
-	}
-	des.Size = int64(meta.Size)
-
-	switch des.CamliType {
-	case "permanode":
-		des.Permanode = new(DescribedPermanode)
-		dr.populatePermanodeFields(des.Permanode, br, dr.sh.owner, depth)
-	case "file":
-		fi, err := dr.sh.index.GetFileInfo(br)
-		if err != nil {
-			if os.IsNotExist(err) {
-				log.Printf("index.GetFileInfo(file %s) failed; index stale?", br)
-			} else {
-				dr.addError(br, err)
-			}
-			return
-		}
-		des.File = &fi
-		if des.File.IsImage() && !skipImageInfoLookup(des.File) {
-			imgInfo, err := dr.sh.index.GetImageInfo(br)
-			if err != nil {
-				if os.IsNotExist(err) {
-					log.Printf("index.GetImageInfo(file %s) failed; index stale?", br)
-				} else {
-					dr.addError(br, err)
-				}
-			} else {
-				des.Image = &imgInfo
-			}
-		}
-		if mediaTags, err := dr.sh.index.GetMediaTags(br); err == nil {
-			des.MediaTags = mediaTags
-		}
-	case "directory":
-		var g syncutil.Group
-		g.Go(func() (err error) {
-			fi, err := dr.sh.index.GetFileInfo(br)
-			if os.IsNotExist(err) {
-				log.Printf("index.GetFileInfo(directory %s) failed; index stale?", br)
-			}
-			if err == nil {
-				des.Dir = &fi
-			}
-			return
-		})
-		g.Go(func() (err error) {
-			des.DirChildren, err = dr.getDirMembers(br, depth)
-			return
-		})
-		if err := g.Err(); err != nil {
-			dr.addError(br, err)
-		}
-	}
-}
-
-func (sh *Handler) Describe(dr *DescribeRequest) (*DescribeResponse, error) {
-	sh.initDescribeRequest(dr)
-	if dr.BlobRef.Valid() {
-		dr.Describe(dr.BlobRef, dr.depth())
-	}
-	for _, br := range dr.BlobRefs {
-		dr.Describe(br, dr.depth())
-	}
-	metaMap, err := dr.metaMapThumbs(dr.ThumbnailSize)
-	if err != nil {
-		return nil, err
-	}
-	return &DescribeResponse{metaMap}, nil
-}
-
-func (sh *Handler) serveDescribe(rw http.ResponseWriter, req *http.Request) {
-	defer httputil.RecoverJSON(rw, req)
-	var dr DescribeRequest
-	dr.fromHTTP(req)
-
-	res, err := sh.Describe(&dr)
-	if err != nil {
-		httputil.ServeJSONError(rw, err)
-		return
-	}
-	httputil.ReturnJSON(rw, res)
-}
-
 func (sh *Handler) serveFiles(rw http.ResponseWriter, req *http.Request) {
 	ret := jsonMap()
 	defer httputil.ReturnJSON(rw, ret)
@@ -1322,100 +712,6 @@ func (sh *Handler) serveFiles(rw http.ResponseWriter, req *http.Request) {
 	return
 }
 
-func (dr *DescribeRequest) populatePermanodeFields(pi *DescribedPermanode, pn, signer blob.Ref, depth int) {
-	pi.Attr = make(url.Values)
-	attr := pi.Attr
-
-	claims, err := dr.sh.index.AppendClaims(nil, pn, signer, "")
-	if err != nil {
-		log.Printf("Error getting claims of %s: %v", pn.String(), err)
-		dr.addError(pn, fmt.Errorf("Error getting claims of %s: %v", pn.String(), err))
-		return
-	}
-
-	sort.Sort(camtypes.ClaimsByDate(claims))
-claimLoop:
-	for _, cl := range claims {
-		if !dr.At.IsZero() {
-			if cl.Date.After(dr.At.Time()) {
-				continue
-			}
-		}
-		switch cl.Type {
-		default:
-			continue
-		case "del-attribute":
-			if cl.Value == "" {
-				delete(attr, cl.Attr)
-			} else {
-				sl := attr[cl.Attr]
-				filtered := make([]string, 0, len(sl))
-				for _, val := range sl {
-					if val != cl.Value {
-						filtered = append(filtered, val)
-					}
-				}
-				attr[cl.Attr] = filtered
-			}
-		case "set-attribute":
-			delete(attr, cl.Attr)
-			fallthrough
-		case "add-attribute":
-			if cl.Value == "" {
-				continue
-			}
-			sl, ok := attr[cl.Attr]
-			if ok {
-				for _, exist := range sl {
-					if exist == cl.Value {
-						continue claimLoop
-					}
-				}
-			} else {
-				sl = make([]string, 0, 1)
-				attr[cl.Attr] = sl
-			}
-			attr[cl.Attr] = append(sl, cl.Value)
-		}
-		pi.ModTime = cl.Date
-	}
-
-	// Descend into any references in current attributes.
-	for key, vals := range attr {
-		dr.describeRefs(key, depth)
-		for _, v := range vals {
-			dr.describeRefs(v, depth)
-		}
-	}
-}
-
-func (dr *DescribeRequest) getDirMembers(br blob.Ref, depth int) ([]blob.Ref, error) {
-	limit := dr.maxDirChildren()
-	ch := make(chan blob.Ref)
-	errch := make(chan error)
-	go func() {
-		errch <- dr.sh.index.GetDirMembers(br, ch, limit)
-	}()
-
-	var members []blob.Ref
-	for child := range ch {
-		dr.Describe(child, depth)
-		members = append(members, child)
-	}
-	if err := <-errch; err != nil {
-		return nil, err
-	}
-	return members, nil
-}
-
-func (dr *DescribeRequest) describeRefs(str string, depth int) {
-	for _, match := range blobRefPattern.FindAllString(str, -1) {
-		if ref, ok := blob.ParseKnown(match); ok {
-			dr.Describe(ref, depth-1)
-		}
-	}
-}
-
 // SignerAttrValueResponse is the JSON response to $search/camli/search/signerattrvalue
 type SignerAttrValueResponse struct {
 	Permanode blob.Ref `json:"permanode"`
@@ -1605,12 +901,6 @@ func (sh *Handler) serveSignerPaths(rw http.ResponseWriter, req *http.Request) {
 
 const camliTypePrefix = "application/json; camliType="
 
-func (d *DescribedBlob) setMIMEType(mime string) {
-	if strings.HasPrefix(mime, camliTypePrefix) {
-		d.CamliType = strings.TrimPrefix(mime, camliTypePrefix)
-	}
-}
-
 func skipImageInfoLookup(fi *camtypes.FileInfo) bool {
 	// psd photoshop files are not currently indexed (no width/height info available),
 	// so we don't even try to hit the index, which would just log an error.

pkg/search/handler_test.go

@@ -17,14 +17,17 @@ limitations under the License.
 package search_test
 
 import (
-	. "camlistore.org/pkg/search"
-
 	"bytes"
 	"encoding/json"
+	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -33,6 +36,7 @@ import (
 	"camlistore.org/pkg/index"
 	"camlistore.org/pkg/index/indextest"
 	"camlistore.org/pkg/osutil"
+	. "camlistore.org/pkg/search"
 	"camlistore.org/pkg/test"
 )
 
@@ -61,10 +65,15 @@ type handlerTest struct {
 	// FakeIndex is not used.
 	setup func(fi *test.FakeIndex) index.Interface
 
-	name  string // test name
+	name     string // test name
-	query string // the HTTP path + optional query suffix after "camli/search/"
+	query    string // the HTTP path + optional query suffix after "camli/search/"
+	postBody string // if non-nil, a POST request
 
 	want map[string]interface{}
+	// wantDescribed is a list of blobref strings that should've been
+	// described in meta. If want is nil and this is non-zero length,
+	// want is ignored.
+	wantDescribed []string
 }
 
 var owner = blob.MustParse("abcown-123")
@@ -614,57 +623,112 @@ var handlerTests = []handlerTest{
 	},
 }
 
+func marshalJSON(v interface{}) string {
+	b, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+func jmap(v interface{}) map[string]interface{} {
+	m := make(map[string]interface{})
+	if err := json.NewDecoder(strings.NewReader(marshalJSON(v))).Decode(&m); err != nil {
+		panic(err)
+	}
+	return m
+}
+
+func checkNoDups(sliceName string, tests []handlerTest) {
+	seen := map[string]bool{}
+	for _, tt := range tests {
+		if seen[tt.name] {
+			panic(fmt.Sprintf("duplicate handlerTest named %q in var %s", tt.name, sliceName))
+		}
+		seen[tt.name] = true
+	}
+}
+
+func init() {
+	checkNoDups("handlerTests", handlerTests)
+}
+
+func (ht handlerTest) test(t *testing.T) {
+	SetTestHookBug121(func() {})
+
+	fakeIndex := test.NewFakeIndex()
+	idx := ht.setup(fakeIndex)
+
+	indexOwner := owner
+	if io, ok := idx.(indexOwnerer); ok {
+		indexOwner = io.IndexOwner()
+	}
+	h := NewHandler(idx, indexOwner)
+
+	var body io.Reader
+	var method = "GET"
+	if ht.postBody != "" {
+		method = "POST"
+		body = strings.NewReader(ht.postBody)
+	}
+	req, err := http.NewRequest(method, "/camli/search/"+ht.query, body)
+	if err != nil {
+		t.Fatalf("%s: bad query: %v", ht.name, err)
+	}
+	req.Header.Set(httputil.PathSuffixHeader, req.URL.Path[1:])
+
+	rr := httptest.NewRecorder()
+	rr.Body = new(bytes.Buffer)
+
+	h.ServeHTTP(rr, req)
+	got := rr.Body.Bytes()
+
+	if len(ht.wantDescribed) > 0 {
+		dr := new(DescribeResponse)
+		if err := json.NewDecoder(bytes.NewReader(got)).Decode(dr); err != nil {
+			t.Fatalf("On test %s: Non-JSON response: %s", ht.name, got)
+		}
+		var gotDesc []string
+		for k := range dr.Meta {
+			gotDesc = append(gotDesc, k)
+		}
+		sort.Strings(ht.wantDescribed)
+		sort.Strings(gotDesc)
+		if !reflect.DeepEqual(gotDesc, ht.wantDescribed) {
+			t.Errorf("On test %s: described blobs:\n%v\nwant:\n%v\n",
+				ht.name, gotDesc, ht.wantDescribed)
+		}
+		if ht.want == nil {
+			return
+		}
+	}
+
+	want, _ := json.MarshalIndent(ht.want, "", "  ")
+	trim := bytes.TrimSpace
+
+	if bytes.Equal(trim(got), trim(want)) {
+		return
+	}
+
+	// Try with re-encoded got, since the JSON ordering doesn't matter
+	// to the test,
+	gotj := parseJSON(string(got))
+	got2, _ := json.MarshalIndent(gotj, "", "  ")
+	if bytes.Equal(got2, want) {
+		return
+	}
+	diff := test.Diff(want, got2)
+
+	t.Errorf("test %s:\nwant: %s\n got: %s\ndiff:\n%s", ht.name, want, got, diff)
+}
+
 func TestHandler(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 		return
 	}
-	seen := map[string]bool{}
 	defer SetTestHookBug121(func() {})
-	for _, tt := range handlerTests {
+	for _, ht := range handlerTests {
-		if seen[tt.name] {
+		ht.test(t)
-			t.Fatalf("duplicate test named %q", tt.name)
-		}
-		seen[tt.name] = true
-		SetTestHookBug121(func() {})
-
-		fakeIndex := test.NewFakeIndex()
-		idx := tt.setup(fakeIndex)
-
-		indexOwner := owner
-		if io, ok := idx.(indexOwnerer); ok {
-			indexOwner = io.IndexOwner()
-		}
-		h := NewHandler(idx, indexOwner)
-
-		req, err := http.NewRequest("GET", "/camli/search/"+tt.query, nil)
-		if err != nil {
-			t.Fatalf("%s: bad query: %v", tt.name, err)
-		}
-		req.Header.Set(httputil.PathSuffixHeader, req.URL.Path[1:])
-
-		rr := httptest.NewRecorder()
-		rr.Body = new(bytes.Buffer)
-
-		h.ServeHTTP(rr, req)
-
-		got := rr.Body.Bytes()
-		want, _ := json.MarshalIndent(tt.want, "", "  ")
-		trim := bytes.TrimSpace
-
-		if bytes.Equal(trim(got), trim(want)) {
-			continue
-		}
-
-		// Try with re-encoded got, since the JSON ordering doesn't matter
-		// to the test,
-		gotj := parseJSON(string(got))
-		got2, _ := json.MarshalIndent(gotj, "", "  ")
-		if bytes.Equal(got2, want) {
-			continue
-		}
-		diff := test.Diff(want, got2)
-
-		t.Errorf("test %s:\nwant: %s\n got: %s\ndiff:\n%s", tt.name, want, got, diff)
 	}
 }

pkg/test/fakeindex.go

@@ -26,7 +26,6 @@ import (
 
 	"camlistore.org/pkg/blob"
 	"camlistore.org/pkg/context"
-	"camlistore.org/pkg/types"
 	"camlistore.org/pkg/types/camtypes"
 )
 
@@ -80,12 +79,12 @@ func camliTypeFromMime(mime string) string {
 	return ""
 }
 
-func (fi *FakeIndex) AddMeta(br blob.Ref, camliType string, size int64) {
+func (fi *FakeIndex) AddMeta(br blob.Ref, camliType string, size uint32) {
 	fi.lk.Lock()
 	defer fi.lk.Unlock()
 	fi.meta[br] = camtypes.BlobMeta{
 		Ref:       br,
-		Size:      types.U32(size),
+		Size:      size,
 		CamliType: camliType,
 	}
 }