publish: use generic queries

Use generic queries instead of specialized index queries. This is a step
towards the publisher app, because its client will have to use generic
queries.

Context: http://camlistore.org/issue/365

Change-Id: I2781a345e024174e3bea8511b6cdc6f342d5a7c1

pkg/index/corpus.go

@@ -854,6 +854,8 @@ func (c *Corpus) PermanodeModtimeLocked(pn blob.Ref) (t time.Time, ok bool) {
 	return t, !t.IsZero()
 }
 
+// AppendPermanodeAttrValues appends to dst all the values for the attribute
+// attr set on permaNode.
 // signerFilter is optional.
 // dst must start with length 0 (laziness, mostly)
 func (c *Corpus) AppendPermanodeAttrValues(dst []string,

pkg/search/handler.go

@@ -366,6 +366,9 @@ func (m MetaMap) Get(br blob.Ref) *DescribedBlob {
 	return m[br.String()]
 }
 
+// TODO(mpl): it looks like we never populate RecentResponse.Error*, shouldn't we remove them?
+// Same for WithAttrResponse. I suppose it doesn't matter much if we end up removing GetRecentPermanodes anyway...
+
 // RecentResponse is the JSON response from $searchRoot/camli/search/recent.
 type RecentResponse struct {
 	Recent []*RecentItem `json:"recent"`

pkg/search/query.go

@@ -77,7 +77,7 @@ func (t *SortType) UnmarshalJSON(v []byte) error {
 
 type SearchQuery struct {
 	// Exactly one of Expression or Contraint must be set.
-	// If an Expression is set, it's compiled to an Constraint.
+	// If an Expression is set, it's compiled to a Constraint.
 
 	// Expression is a textual search query in minimal form,
 	// e.g. "hawaii before:2008" or "tag:foo" or "foo" or "location:portland"
@@ -1235,6 +1235,8 @@ func (c *PermanodeConstraint) blobMatches(s *search, br blob.Ref, bm camtypes.Bl
 	return true, nil
 }
 
+// permanodeMatchesAttrVals checks that the values in vals - all of them, if c.ValueAll is set -
+// match the values for c.Attr.
 // vals are the current permanode values of c.Attr.
 func (c *PermanodeConstraint) permanodeMatchesAttrVals(s *search, vals []string) (bool, error) {
 	if c.NumValue != nil && !c.NumValue.intMatches(int64(len(vals))) {

pkg/server/publish.go

@@ -33,7 +33,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"time"
 
 	"camlistore.org/pkg/auth"
 	"camlistore.org/pkg/blob"
@@ -239,31 +238,62 @@ func (ph *PublishHandler) makeClosureHandler(root string) (http.Handler, error)
 	return makeClosureHandler(root, "publish")
 }
 
+func (ph *PublishHandler) camliRootQuery() (*search.SearchResult, error) {
+	// TODO(mpl): I've voluntarily omitted the owner because it's not clear to
+	// that we actually care about that. Same for signer in lookupPathTarget.
+	return ph.Search.Query(&search.SearchQuery{
+		Limit: 1,
+		Constraint: &search.Constraint{
+			Permanode: &search.PermanodeConstraint{
+				Attr:  "camliRoot",
+				Value: ph.RootName,
+			},
+		},
+	})
+}
+
 func (ph *PublishHandler) rootPermanode() (blob.Ref, error) {
 	// TODO: caching, but this can change over time (though
 	// probably rare). might be worth a 5 second cache or
 	// something in-memory? better invalidation story first would
 	// be nice.
-	br, err := ph.Search.Index().PermanodeOfSignerAttrValue(ph.Search.Owner(), "camliRoot", ph.RootName)
+	result, err := ph.camliRootQuery()
 	if err != nil {
-		log.Printf("Error: publish handler at serving root name %q has no configured permanode: %v",
-			ph.RootName, err)
+		return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.RootName, err)
 	}
-	return br, err
+	if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() {
+		return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.RootName, os.ErrNotExist)
+	}
+	return result.Blobs[0].Blob, nil
 }
 
 func (ph *PublishHandler) lookupPathTarget(root blob.Ref, suffix string) (blob.Ref, error) {
 	if suffix == "" {
 		return root, nil
 	}
-	path, err := ph.Search.Index().PathLookup(ph.Search.Owner(), root, suffix, time.Time{})
+	// TODO: verify it's optimized: http://camlistore.org/issue/405
+	result, err := ph.Search.Query(&search.SearchQuery{
+		Limit: 1,
+		Constraint: &search.Constraint{
+			Permanode: &search.PermanodeConstraint{
+				SkipHidden: true,
+				Relation: &search.RelationConstraint{
+					Relation: "parent",
+					EdgeType: "camliPath:" + suffix,
+					Any: &search.Constraint{
+						BlobRefPrefix: root.String(),
+					},
+				},
+			},
+		},
+	})
 	if err != nil {
 		return blob.Ref{}, err
 	}
-	if !path.Target.Valid() {
+	if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() {
 		return blob.Ref{}, os.ErrNotExist
 	}
-	return path.Target, nil
+	return result.Blobs[0].Blob, nil
 }
 
 func (ph *PublishHandler) serveDiscovery(rw http.ResponseWriter, req *http.Request) {
@@ -612,6 +642,7 @@ func (pr *publishRequest) parent() (parentPath string, parentBlobRef blob.Ref, e
 	if pr.subjectBasePath == "" {
 		return "", blob.Ref{}, errors.New("subjectBasePath not set")
 	}
+	// TODO(mpl): this fails when the parent is the root. fix it.
 	hops := publishedPath(pr.subjectBasePath).splitHops()
 	if len(hops) == 0 {
 		return "", blob.Ref{}, errors.New("No subresource digest in subjectBasePath")
@@ -1026,12 +1057,13 @@ func (ph *PublishHandler) setRootNode(jsonSign *signhandler.Handler, pn blob.Ref
 }
 
 func (ph *PublishHandler) bootstrapPermanode(jsonSign *signhandler.Handler) (err error) {
-	if pn, err := ph.Search.Index().PermanodeOfSignerAttrValue(ph.Search.Owner(), "camliRoot", ph.RootName); err == nil {
-		log.Printf("Publish root %q using existing permanode %s", ph.RootName, pn)
+	result, err := ph.camliRootQuery()
+	if err == nil && len(result.Blobs) > 0 && result.Blobs[0].Blob.Valid() {
+		log.Printf("Publish root %q using existing permanode %s", ph.RootName, result.Blobs[0].Blob)
 		return nil
 	}
-	log.Printf("Publish root %q needs a permanode + claim", ph.RootName)
 
+	log.Printf("Publish root %q needs a permanode + claim", ph.RootName)
 	pn, err := ph.signUpload(jsonSign, "permanode", schema.NewUnsignedPermanode())
 	if err != nil {
 		return err

pkg/server/publish_test.go

@@ -21,11 +21,12 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
-	"camlistore.org/pkg/blob"
 	"camlistore.org/pkg/httputil"
+	"camlistore.org/pkg/index"
+	"camlistore.org/pkg/index/indextest"
 	"camlistore.org/pkg/search"
-	"camlistore.org/pkg/test"
 )
 
 type publishURLTest struct {
@@ -33,104 +34,100 @@ type publishURLTest struct {
 	subject, subres string // expected
 }
 
-var publishURLTests = []publishURLTest{
-	// URL to a single picture permanoe (returning its HTML wrapper page)
-	{
-		path:    "/pics/singlepic",
-		subject: "picpn-1234",
-	},
+var publishURLTests []publishURLTest
 
-	// URL to a gallery permanode (returning its HTML wrapper page)
-	{
-		path:    "/pics/camping",
-		subject: "gal-1234",
-	},
+func setupContent(rootName string) *indextest.IndexDeps {
+	idx := index.NewMemoryIndex()
+	idxd := indextest.NewIndexDeps(idx)
 
-	// URL to a picture permanode within a gallery (following one hop, returning HTML)
-	{
-		path:    "/pics/camping/-/h9876543210",
-		subject: "picpn-9876543210",
-	},
+	picNode := idxd.NewPlannedPermanode("picpn-1234")                                             // sha1-f5e90fcc50a79caa8b22a4aa63ba92e436cab9ec
+	galRef := idxd.NewPlannedPermanode("gal-1234")                                                // sha1-2bdf2053922c3dfa70b01a4827168fce1c1df691
+	rootRef := idxd.NewPlannedPermanode("root-abcd")                                              // sha1-dbb3e5f28c7e01536d43ce194f3dd7b921b8460d
+	camp0 := idxd.NewPlannedPermanode("picpn-9876543210")                                         // sha1-2d473e07ca760231dd82edeef4019d5b7d0ccb42
+	camp1 := idxd.NewPlannedPermanode("picpn-9876543211")                                         // sha1-961b700536d5151fc1f3920955cc92767572a064
+	camp0f, _ := idxd.UploadFile("picfile-f00ff00f00a5.jpg", "picfile-f00ff00f00a5", time.Time{}) // sha1-01dbcb193fc789033fb2d08ed22abe7105b48640
+	camp1f, _ := idxd.UploadFile("picfile-f00ff00f00b6.jpg", "picfile-f00ff00f00b6", time.Time{}) // sha1-1213ec17a42cc51bdeb95ff91ac1b5fc5157740f
 
-	// URL to a gallery -> picture permanode -> its file
-	// (following two hops, returning HTML)
-	{
-		path:    "/pics/camping/-/h9876543210/hf00ff00f00a",
-		subject: "picfile-f00ff00f00a5",
-	},
+	idxd.SetAttribute(rootRef, "camliRoot", rootName)
+	idxd.SetAttribute(rootRef, "camliPath:singlepic", picNode.String())
+	idxd.SetAttribute(picNode, "title", "picnode without a pic?")
+	idxd.SetAttribute(rootRef, "camliPath:camping", galRef.String())
+	idxd.AddAttribute(galRef, "camliMember", camp0.String())
+	idxd.AddAttribute(galRef, "camliMember", camp1.String())
+	idxd.SetAttribute(camp0, "camliContent", camp0f.String())
+	idxd.SetAttribute(camp1, "camliContent", camp1f.String())
 
-	// URL to a gallery -> picture permanode -> its file
-	// (following two hops, returning the file download)
-	{
-		path:    "/pics/camping/-/h9876543210/hf00ff00f00a/=f/marshmallow.jpg",
-		subject: "picfile-f00ff00f00a5",
-		subres:  "/=f/marshmallow.jpg",
-	},
+	publishURLTests = []publishURLTest{
+		// URL to a single picture permanode (returning its HTML wrapper page)
+		{
+			path:    "/pics/singlepic",
+			subject: picNode.String(),
+		},
 
-	// URL to a gallery -> picture permanode -> its file
-	// (following two hops, returning the file, scaled as an image)
-	{
-		path:    "/pics/camping/-/h9876543210/hf00ff00f00a/=i/marshmallow.jpg?mw=200&mh=200",
-		subject: "picfile-f00ff00f00a5",
-		subres:  "/=i/marshmallow.jpg",
-	},
+		// URL to a gallery permanode (returning its HTML wrapper page)
+		{
+			path:    "/pics/camping",
+			subject: galRef.String(),
+		},
 
-	// Path to a static file in the root.
-	// TODO: ditch these and use content-addressable javascript + css, having
-	// the server digest them on start, or rather part of fileembed. This is
-	// a short-term hack to unblock Lindsey.
-	{
-		path:    "/pics/=s/pics.js",
-		subject: "",
-		subres:  "/=s/pics.js",
-	},
-}
+		// URL to a picture permanode within a gallery (following one hop, returning HTML)
+		{
+			path:    "/pics/camping/-/h2d473e07ca",
+			subject: camp0.String(),
+		},
 
-func setupContent(owner blob.Ref, rootName string) *test.FakeIndex {
+		// URL to a gallery -> picture permanode -> its file
+		// (following two hops, returning HTML)
+		{
+			path:    "/pics/camping/-/h2d473e07ca/h01dbcb193f",
+			subject: camp0f.String(),
+		},
 
-	picNode := blob.MustParse("picpn-1234")
-	galRef := blob.MustParse("gal-1234")
-	rootRef := blob.MustParse("root-abcd")
-	camp0 := blob.MustParse("picpn-9876543210")
-	camp1 := blob.MustParse("picpn-9876543211")
-	camp0f := blob.MustParse("picfile-f00ff00f00a5")
-	camp1f := blob.MustParse("picfile-f00ff00f00b6")
+		// URL to a gallery -> picture permanode -> its file
+		// (following two hops, returning the file download)
+		{
+			path:    "/pics/camping/-/h2d473e07ca/h01dbcb193f/=f/marshmallow.jpg",
+			subject: camp0f.String(),
+			subres:  "/=f/marshmallow.jpg",
+		},
 
-	idx := test.NewFakeIndex()
-	idx.AddSignerAttrValue(owner, "camliRoot", rootName, rootRef)
+		// URL to a gallery -> picture permanode -> its file
+		// (following two hops, returning the file, scaled as an image)
+		{
+			path:    "/pics/camping/-/h961b700536/h1213ec17a4/=i/marshmallow.jpg?mw=200&mh=200",
+			subject: camp1f.String(),
+			subres:  "/=i/marshmallow.jpg",
+		},
 
-	idx.AddMeta(owner, "", 100)
-	for _, br := range []blob.Ref{picNode, galRef, rootRef, camp0, camp1} {
-		idx.AddMeta(br, "permanode", 100)
-	}
-	for _, br := range []blob.Ref{camp0f, camp1f} {
-		idx.AddMeta(br, "file", 100)
+		// Path to a static file in the root.
+		// TODO: ditch these and use content-addressable javascript + css, having
+		// the server digest them on start, or rather part of fileembed. This is
+		// a short-term hack to unblock Lindsey.
+		{
+			path:    "/pics/=s/pics.js",
+			subject: "",
+			subres:  "/=s/pics.js",
+		},
 	}
 
-	idx.AddClaim(owner, rootRef, "set-attribute", "camliPath:singlepic", picNode.String())
-	idx.AddClaim(owner, rootRef, "set-attribute", "camliPath:camping", galRef.String())
-	idx.AddClaim(owner, galRef, "add-attribute", "camliMember", camp0.String())
-	idx.AddClaim(owner, galRef, "add-attribute", "camliMember", camp1.String())
-	idx.AddClaim(owner, camp0, "set-attribute", "camliContent", camp0f.String())
-	idx.AddClaim(owner, camp1, "set-attribute", "camliContent", camp1f.String())
-
-	return idx
+	return idxd
 }
 
 func TestPublishURLs(t *testing.T) {
-
-	owner := blob.MustParse("owner-1234")
 	rootName := "foo"
+	idxd := setupContent(rootName)
+	sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef)
+	corpus, err := idxd.Index.KeepInMemory()
+	if err != nil {
+		t.Fatalf("error slurping index to memory: %v", err)
+	}
+	sh.SetCorpus(corpus)
+	ph := &PublishHandler{
+		RootName: rootName,
+		Search:   sh,
+	}
 
 	for ti, tt := range publishURLTests {
-		idx := setupContent(owner, rootName)
-
-		sh := search.NewHandler(idx, owner)
-		ph := &PublishHandler{
-			RootName: rootName,
-			Search:   sh,
-		}
-
 		rw := httptest.NewRecorder()
 		if !strings.HasPrefix(tt.path, "/pics/") {
 			panic("expected /pics/ prefix on " + tt.path)
@@ -162,12 +159,15 @@ func TestPublishURLs(t *testing.T) {
 }
 
 func TestPublishMembers(t *testing.T) {
-	owner := blob.MustParse("owner-1234")
 	rootName := "foo"
+	idxd := setupContent(rootName)
 
-	idx := setupContent(owner, rootName)
-
-	sh := search.NewHandler(idx, owner)
+	sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef)
+	corpus, err := idxd.Index.KeepInMemory()
+	if err != nil {
+		t.Fatalf("error slurping index to memory: %v", err)
+	}
+	sh.SetCorpus(corpus)
 	ph := &PublishHandler{
 		RootName: rootName,
 		Search:   sh,