From 9cadbcd5bc24552ba6898d8ac167d79a3bb323e9 Mon Sep 17 00:00:00 2001 From: mpl Date: Tue, 18 Mar 2014 00:27:37 +0100 Subject: [PATCH] publish: use generic queries Use generic queries instead of specialized index queries. This is a step towards the publisher app, because its client will have to use generic queries. Context: http://camlistore.org/issue/365 Change-Id: I2781a345e024174e3bea8511b6cdc6f342d5a7c1 --- pkg/index/corpus.go | 2 + pkg/search/handler.go | 3 + pkg/search/query.go | 4 +- pkg/server/publish.go | 54 +++++++++--- pkg/server/publish_test.go | 172 ++++++++++++++++++------------------- 5 files changed, 137 insertions(+), 98 deletions(-) diff --git a/pkg/index/corpus.go b/pkg/index/corpus.go index a28f8915e..c06b54ca9 100644 --- a/pkg/index/corpus.go +++ b/pkg/index/corpus.go @@ -854,6 +854,8 @@ func (c *Corpus) PermanodeModtimeLocked(pn blob.Ref) (t time.Time, ok bool) { return t, !t.IsZero() } +// AppendPermanodeAttrValues appends to dst all the values for the attribute +// attr set on permaNode. // signerFilter is optional. // dst must start with length 0 (laziness, mostly) func (c *Corpus) AppendPermanodeAttrValues(dst []string, diff --git a/pkg/search/handler.go b/pkg/search/handler.go index ed01ec491..1f59b9e8e 100644 --- a/pkg/search/handler.go +++ b/pkg/search/handler.go @@ -366,6 +366,9 @@ func (m MetaMap) Get(br blob.Ref) *DescribedBlob { return m[br.String()] } +// TODO(mpl): it looks like we never populate RecentResponse.Error*, shouldn't we remove them? +// Same for WithAttrResponse. I suppose it doesn't matter much if we end up removing GetRecentPermanodes anyway... + // RecentResponse is the JSON response from $searchRoot/camli/search/recent. type RecentResponse struct { Recent []*RecentItem `json:"recent"` diff --git a/pkg/search/query.go b/pkg/search/query.go index 6cb8d1187..2e1df0fea 100644 --- a/pkg/search/query.go +++ b/pkg/search/query.go @@ -77,7 +77,7 @@ func (t *SortType) UnmarshalJSON(v []byte) error { type SearchQuery struct { // Exactly one of Expression or Contraint must be set. - // If an Expression is set, it's compiled to an Constraint. + // If an Expression is set, it's compiled to a Constraint. // Expression is a textual search query in minimal form, // e.g. "hawaii before:2008" or "tag:foo" or "foo" or "location:portland" @@ -1235,6 +1235,8 @@ func (c *PermanodeConstraint) blobMatches(s *search, br blob.Ref, bm camtypes.Bl return true, nil } +// permanodeMatchesAttrVals checks that the values in vals - all of them, if c.ValueAll is set - +// match the values for c.Attr. // vals are the current permanode values of c.Attr. func (c *PermanodeConstraint) permanodeMatchesAttrVals(s *search, vals []string) (bool, error) { if c.NumValue != nil && !c.NumValue.intMatches(int64(len(vals))) { diff --git a/pkg/server/publish.go b/pkg/server/publish.go index 8600887d0..778689f11 100644 --- a/pkg/server/publish.go +++ b/pkg/server/publish.go @@ -33,7 +33,6 @@ import ( "regexp" "strconv" "strings" - "time" "camlistore.org/pkg/auth" "camlistore.org/pkg/blob" @@ -239,31 +238,62 @@ func (ph *PublishHandler) makeClosureHandler(root string) (http.Handler, error) return makeClosureHandler(root, "publish") } +func (ph *PublishHandler) camliRootQuery() (*search.SearchResult, error) { + // TODO(mpl): I've voluntarily omitted the owner because it's not clear to + // that we actually care about that. Same for signer in lookupPathTarget. + return ph.Search.Query(&search.SearchQuery{ + Limit: 1, + Constraint: &search.Constraint{ + Permanode: &search.PermanodeConstraint{ + Attr: "camliRoot", + Value: ph.RootName, + }, + }, + }) +} + func (ph *PublishHandler) rootPermanode() (blob.Ref, error) { // TODO: caching, but this can change over time (though // probably rare). might be worth a 5 second cache or // something in-memory? better invalidation story first would // be nice. - br, err := ph.Search.Index().PermanodeOfSignerAttrValue(ph.Search.Owner(), "camliRoot", ph.RootName) + result, err := ph.camliRootQuery() if err != nil { - log.Printf("Error: publish handler at serving root name %q has no configured permanode: %v", - ph.RootName, err) + return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.RootName, err) } - return br, err + if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() { + return blob.Ref{}, fmt.Errorf("could not find permanode for root %q of publish handler: %v", ph.RootName, os.ErrNotExist) + } + return result.Blobs[0].Blob, nil } func (ph *PublishHandler) lookupPathTarget(root blob.Ref, suffix string) (blob.Ref, error) { if suffix == "" { return root, nil } - path, err := ph.Search.Index().PathLookup(ph.Search.Owner(), root, suffix, time.Time{}) + // TODO: verify it's optimized: http://camlistore.org/issue/405 + result, err := ph.Search.Query(&search.SearchQuery{ + Limit: 1, + Constraint: &search.Constraint{ + Permanode: &search.PermanodeConstraint{ + SkipHidden: true, + Relation: &search.RelationConstraint{ + Relation: "parent", + EdgeType: "camliPath:" + suffix, + Any: &search.Constraint{ + BlobRefPrefix: root.String(), + }, + }, + }, + }, + }) if err != nil { return blob.Ref{}, err } - if !path.Target.Valid() { + if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() { return blob.Ref{}, os.ErrNotExist } - return path.Target, nil + return result.Blobs[0].Blob, nil } func (ph *PublishHandler) serveDiscovery(rw http.ResponseWriter, req *http.Request) { @@ -612,6 +642,7 @@ func (pr *publishRequest) parent() (parentPath string, parentBlobRef blob.Ref, e if pr.subjectBasePath == "" { return "", blob.Ref{}, errors.New("subjectBasePath not set") } + // TODO(mpl): this fails when the parent is the root. fix it. hops := publishedPath(pr.subjectBasePath).splitHops() if len(hops) == 0 { return "", blob.Ref{}, errors.New("No subresource digest in subjectBasePath") @@ -1026,12 +1057,13 @@ func (ph *PublishHandler) setRootNode(jsonSign *signhandler.Handler, pn blob.Ref } func (ph *PublishHandler) bootstrapPermanode(jsonSign *signhandler.Handler) (err error) { - if pn, err := ph.Search.Index().PermanodeOfSignerAttrValue(ph.Search.Owner(), "camliRoot", ph.RootName); err == nil { - log.Printf("Publish root %q using existing permanode %s", ph.RootName, pn) + result, err := ph.camliRootQuery() + if err == nil && len(result.Blobs) > 0 && result.Blobs[0].Blob.Valid() { + log.Printf("Publish root %q using existing permanode %s", ph.RootName, result.Blobs[0].Blob) return nil } - log.Printf("Publish root %q needs a permanode + claim", ph.RootName) + log.Printf("Publish root %q needs a permanode + claim", ph.RootName) pn, err := ph.signUpload(jsonSign, "permanode", schema.NewUnsignedPermanode()) if err != nil { return err diff --git a/pkg/server/publish_test.go b/pkg/server/publish_test.go index 4fd623c22..683639dd1 100644 --- a/pkg/server/publish_test.go +++ b/pkg/server/publish_test.go @@ -21,11 +21,12 @@ import ( "net/http/httptest" "strings" "testing" + "time" - "camlistore.org/pkg/blob" "camlistore.org/pkg/httputil" + "camlistore.org/pkg/index" + "camlistore.org/pkg/index/indextest" "camlistore.org/pkg/search" - "camlistore.org/pkg/test" ) type publishURLTest struct { @@ -33,104 +34,100 @@ type publishURLTest struct { subject, subres string // expected } -var publishURLTests = []publishURLTest{ - // URL to a single picture permanoe (returning its HTML wrapper page) - { - path: "/pics/singlepic", - subject: "picpn-1234", - }, +var publishURLTests []publishURLTest - // URL to a gallery permanode (returning its HTML wrapper page) - { - path: "/pics/camping", - subject: "gal-1234", - }, +func setupContent(rootName string) *indextest.IndexDeps { + idx := index.NewMemoryIndex() + idxd := indextest.NewIndexDeps(idx) - // URL to a picture permanode within a gallery (following one hop, returning HTML) - { - path: "/pics/camping/-/h9876543210", - subject: "picpn-9876543210", - }, + picNode := idxd.NewPlannedPermanode("picpn-1234") // sha1-f5e90fcc50a79caa8b22a4aa63ba92e436cab9ec + galRef := idxd.NewPlannedPermanode("gal-1234") // sha1-2bdf2053922c3dfa70b01a4827168fce1c1df691 + rootRef := idxd.NewPlannedPermanode("root-abcd") // sha1-dbb3e5f28c7e01536d43ce194f3dd7b921b8460d + camp0 := idxd.NewPlannedPermanode("picpn-9876543210") // sha1-2d473e07ca760231dd82edeef4019d5b7d0ccb42 + camp1 := idxd.NewPlannedPermanode("picpn-9876543211") // sha1-961b700536d5151fc1f3920955cc92767572a064 + camp0f, _ := idxd.UploadFile("picfile-f00ff00f00a5.jpg", "picfile-f00ff00f00a5", time.Time{}) // sha1-01dbcb193fc789033fb2d08ed22abe7105b48640 + camp1f, _ := idxd.UploadFile("picfile-f00ff00f00b6.jpg", "picfile-f00ff00f00b6", time.Time{}) // sha1-1213ec17a42cc51bdeb95ff91ac1b5fc5157740f - // URL to a gallery -> picture permanode -> its file - // (following two hops, returning HTML) - { - path: "/pics/camping/-/h9876543210/hf00ff00f00a", - subject: "picfile-f00ff00f00a5", - }, + idxd.SetAttribute(rootRef, "camliRoot", rootName) + idxd.SetAttribute(rootRef, "camliPath:singlepic", picNode.String()) + idxd.SetAttribute(picNode, "title", "picnode without a pic?") + idxd.SetAttribute(rootRef, "camliPath:camping", galRef.String()) + idxd.AddAttribute(galRef, "camliMember", camp0.String()) + idxd.AddAttribute(galRef, "camliMember", camp1.String()) + idxd.SetAttribute(camp0, "camliContent", camp0f.String()) + idxd.SetAttribute(camp1, "camliContent", camp1f.String()) - // URL to a gallery -> picture permanode -> its file - // (following two hops, returning the file download) - { - path: "/pics/camping/-/h9876543210/hf00ff00f00a/=f/marshmallow.jpg", - subject: "picfile-f00ff00f00a5", - subres: "/=f/marshmallow.jpg", - }, + publishURLTests = []publishURLTest{ + // URL to a single picture permanode (returning its HTML wrapper page) + { + path: "/pics/singlepic", + subject: picNode.String(), + }, - // URL to a gallery -> picture permanode -> its file - // (following two hops, returning the file, scaled as an image) - { - path: "/pics/camping/-/h9876543210/hf00ff00f00a/=i/marshmallow.jpg?mw=200&mh=200", - subject: "picfile-f00ff00f00a5", - subres: "/=i/marshmallow.jpg", - }, + // URL to a gallery permanode (returning its HTML wrapper page) + { + path: "/pics/camping", + subject: galRef.String(), + }, - // Path to a static file in the root. - // TODO: ditch these and use content-addressable javascript + css, having - // the server digest them on start, or rather part of fileembed. This is - // a short-term hack to unblock Lindsey. - { - path: "/pics/=s/pics.js", - subject: "", - subres: "/=s/pics.js", - }, -} + // URL to a picture permanode within a gallery (following one hop, returning HTML) + { + path: "/pics/camping/-/h2d473e07ca", + subject: camp0.String(), + }, -func setupContent(owner blob.Ref, rootName string) *test.FakeIndex { + // URL to a gallery -> picture permanode -> its file + // (following two hops, returning HTML) + { + path: "/pics/camping/-/h2d473e07ca/h01dbcb193f", + subject: camp0f.String(), + }, - picNode := blob.MustParse("picpn-1234") - galRef := blob.MustParse("gal-1234") - rootRef := blob.MustParse("root-abcd") - camp0 := blob.MustParse("picpn-9876543210") - camp1 := blob.MustParse("picpn-9876543211") - camp0f := blob.MustParse("picfile-f00ff00f00a5") - camp1f := blob.MustParse("picfile-f00ff00f00b6") + // URL to a gallery -> picture permanode -> its file + // (following two hops, returning the file download) + { + path: "/pics/camping/-/h2d473e07ca/h01dbcb193f/=f/marshmallow.jpg", + subject: camp0f.String(), + subres: "/=f/marshmallow.jpg", + }, - idx := test.NewFakeIndex() - idx.AddSignerAttrValue(owner, "camliRoot", rootName, rootRef) + // URL to a gallery -> picture permanode -> its file + // (following two hops, returning the file, scaled as an image) + { + path: "/pics/camping/-/h961b700536/h1213ec17a4/=i/marshmallow.jpg?mw=200&mh=200", + subject: camp1f.String(), + subres: "/=i/marshmallow.jpg", + }, - idx.AddMeta(owner, "", 100) - for _, br := range []blob.Ref{picNode, galRef, rootRef, camp0, camp1} { - idx.AddMeta(br, "permanode", 100) - } - for _, br := range []blob.Ref{camp0f, camp1f} { - idx.AddMeta(br, "file", 100) + // Path to a static file in the root. + // TODO: ditch these and use content-addressable javascript + css, having + // the server digest them on start, or rather part of fileembed. This is + // a short-term hack to unblock Lindsey. + { + path: "/pics/=s/pics.js", + subject: "", + subres: "/=s/pics.js", + }, } - idx.AddClaim(owner, rootRef, "set-attribute", "camliPath:singlepic", picNode.String()) - idx.AddClaim(owner, rootRef, "set-attribute", "camliPath:camping", galRef.String()) - idx.AddClaim(owner, galRef, "add-attribute", "camliMember", camp0.String()) - idx.AddClaim(owner, galRef, "add-attribute", "camliMember", camp1.String()) - idx.AddClaim(owner, camp0, "set-attribute", "camliContent", camp0f.String()) - idx.AddClaim(owner, camp1, "set-attribute", "camliContent", camp1f.String()) - - return idx + return idxd } func TestPublishURLs(t *testing.T) { - - owner := blob.MustParse("owner-1234") rootName := "foo" + idxd := setupContent(rootName) + sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef) + corpus, err := idxd.Index.KeepInMemory() + if err != nil { + t.Fatalf("error slurping index to memory: %v", err) + } + sh.SetCorpus(corpus) + ph := &PublishHandler{ + RootName: rootName, + Search: sh, + } for ti, tt := range publishURLTests { - idx := setupContent(owner, rootName) - - sh := search.NewHandler(idx, owner) - ph := &PublishHandler{ - RootName: rootName, - Search: sh, - } - rw := httptest.NewRecorder() if !strings.HasPrefix(tt.path, "/pics/") { panic("expected /pics/ prefix on " + tt.path) @@ -162,12 +159,15 @@ func TestPublishURLs(t *testing.T) { } func TestPublishMembers(t *testing.T) { - owner := blob.MustParse("owner-1234") rootName := "foo" + idxd := setupContent(rootName) - idx := setupContent(owner, rootName) - - sh := search.NewHandler(idx, owner) + sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef) + corpus, err := idxd.Index.KeepInMemory() + if err != nil { + t.Fatalf("error slurping index to memory: %v", err) + } + sh.SetCorpus(corpus) ph := &PublishHandler{ RootName: rootName, Search: sh,