From 231ba4233f26c7a1664afa0fe7fdc496589e3a3c Mon Sep 17 00:00:00 2001 From: aviau Date: Sat, 16 Jan 2021 22:05:35 -0500 Subject: [PATCH] pkg/schema: create CamliType type Create a CamliType type in pkg/schema and use it in a couple of packages. It can be implemented in other packages as we go. --- app/publisher/main.go | 3 +- app/publisher/zip.go | 4 +- cmd/pk-get/get.go | 14 ++--- cmd/pk-get/graph.go | 2 +- cmd/pk-put/files.go | 4 +- cmd/pk-put/put.go | 3 +- cmd/pk/list.go | 7 +-- cmd/pk/makestatic.go | 2 +- pkg/blobserver/blobpacked/blobpacked.go | 2 +- pkg/client/upload.go | 2 +- pkg/fs/fs.go | 12 ++--- pkg/index/corpus.go | 12 ++--- pkg/index/index.go | 12 ++--- pkg/index/indextest/tests.go | 2 +- pkg/index/receive.go | 12 ++--- pkg/index/sniff.go | 6 +-- pkg/schema/blob.go | 40 +++++++-------- pkg/schema/fileread_test.go | 2 +- pkg/schema/filewriter.go | 2 +- pkg/schema/schema.go | 68 ++++++++++++++++--------- pkg/schema/schema_test.go | 8 +-- pkg/search/describe.go | 11 ++-- pkg/search/handler.go | 9 ++-- pkg/search/query.go | 15 +++--- pkg/search/websocket.go | 5 +- pkg/server/download.go | 17 ++++--- pkg/server/filetree.go | 2 +- pkg/server/import_share.go | 2 +- pkg/server/share.go | 6 +-- pkg/types/camtypes/search.go | 7 +-- 30 files changed, 161 insertions(+), 132 deletions(-) diff --git a/app/publisher/main.go b/app/publisher/main.go index 59aaf4011..4796ab211 100644 --- a/app/publisher/main.go +++ b/app/publisher/main.go @@ -55,6 +55,7 @@ import ( "perkeep.org/pkg/constants" "perkeep.org/pkg/fileembed" "perkeep.org/pkg/publish" + "perkeep.org/pkg/schema" "perkeep.org/pkg/search" "perkeep.org/pkg/server" "perkeep.org/pkg/sorted" @@ -869,7 +870,7 @@ func (pr *publishRequest) serveSubjectTemplate() { pr.ph.cacheDescribed(res.Meta) subdes := res.Meta[pr.subject.String()] - if subdes.CamliType == "file" { + if subdes.CamliType == schema.TypeFile { pr.serveFileDownload(subdes) return } diff --git a/app/publisher/zip.go b/app/publisher/zip.go index 8811f7af3..01dd43348 100644 --- a/app/publisher/zip.go +++ b/app/publisher/zip.go @@ -169,9 +169,9 @@ func (zh *zipHandler) blobsFromDir(dirPath string, dirBlob blob.Ref) ([]*blobFil for _, v := range ent { fullpath := path.Join(dirPath, v.FileName()) switch v.CamliType() { - case "file": + case schema.TypeFile: list = append(list, &blobFile{v.BlobRef(), fullpath}) - case "directory": + case schema.TypeDirectory: children, err := zh.blobsFromDir(fullpath, v.BlobRef()) if err != nil { return nil, fmt.Errorf("Could not get list of blobs from %v: %v", v.BlobRef(), err) diff --git a/cmd/pk-get/get.go b/cmd/pk-get/get.go index b7f325e9a..bb280b133 100644 --- a/cmd/pk-get/get.go +++ b/cmd/pk-get/get.go @@ -240,7 +240,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) rcc.Close() switch b.Type() { - case "directory": + case schema.TypeDirectory: dir := filepath.Join(targ, b.FileName()) if *flagVerbose { log.Printf("Fetching directory %v into %s", br, dir) @@ -256,7 +256,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) return fmt.Errorf("bad entries blobref in dir %v", b.BlobRef()) } return smartFetch(ctx, src, dir, entries) - case "static-set": + case schema.TypeStaticSet: if *flagVerbose { log.Printf("Fetching directory entries %v into %s", br, targ) } @@ -289,7 +289,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) } } return nil - case "file": + case schema.TypeFile: fr, err := schema.NewFileReader(ctx, src, br) if err != nil { return fmt.Errorf("NewFileReader: %v", err) @@ -322,7 +322,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) log.Print(err) } return nil - case "symlink": + case schema.TypeSymlink: if *flagSkipIrregular { return nil } @@ -354,7 +354,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) // os.Chtimes always dereferences (does not act on the // symlink but its target). return err - case "fifo": + case schema.TypeFIFO: if *flagSkipIrregular { return nil } @@ -389,7 +389,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) return nil - case "socket": + case schema.TypeSocket: if *flagSkipIrregular { return nil } @@ -425,7 +425,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) return nil default: - return errors.New("unknown blob type: " + b.Type()) + return errors.New("unknown blob type: " + string(b.Type())) } panic("unreachable") } diff --git a/cmd/pk-get/graph.go b/cmd/pk-get/graph.go index 9385dfd73..93f4e8b7e 100644 --- a/cmd/pk-get/graph.go +++ b/cmd/pk-get/graph.go @@ -53,7 +53,7 @@ func (n *node) dotLabel() string { if n.blob == nil { return fmt.Sprintf("%s\n%d bytes", name, n.size) } - return name + "\n" + n.blob.Type() + return name + "\n" + string(n.blob.Type()) } func (n *node) color() string { diff --git a/cmd/pk-put/files.go b/cmd/pk-put/files.go index f398b67e6..358592fb2 100644 --- a/cmd/pk-put/files.go +++ b/cmd/pk-put/files.go @@ -391,9 +391,9 @@ func (up *Uploader) uploadNode(ctx context.Context, n *node) (*client.PutResult, // including mode & os.ModeCharDevice fallthrough case mode&os.ModeSocket != 0: - bb.SetType("socket") + bb.SetType(schema.TypeSocket) case mode&os.ModeNamedPipe != 0: // fifo - bb.SetType("fifo") + bb.SetType(schema.TypeFIFO) default: return nil, fmt.Errorf("pk-put.files: unsupported file type %v for file %v", mode, n.fullPath) case fi.IsDir(): diff --git a/cmd/pk-put/put.go b/cmd/pk-put/put.go index 62c3835c1..c79e83ac0 100644 --- a/cmd/pk-put/put.go +++ b/cmd/pk-put/put.go @@ -33,6 +33,7 @@ import ( "perkeep.org/pkg/blobserver/dir" "perkeep.org/pkg/client" "perkeep.org/pkg/cmdmain" + "perkeep.org/pkg/schema" "go4.org/syncutil" ) @@ -119,7 +120,7 @@ func initUploader() { uploader = up } -func handleResult(what string, pr *client.PutResult, err error) error { +func handleResult(what schema.CamliType, pr *client.PutResult, err error) error { if err != nil { cmdmain.Errorf("Error putting %s: %s\n", what, err) cmdmain.ExitWithFailure = true diff --git a/cmd/pk/list.go b/cmd/pk/list.go index 200d8f301..2453f1100 100644 --- a/cmd/pk/list.go +++ b/cmd/pk/list.go @@ -28,6 +28,7 @@ import ( "perkeep.org/pkg/blob" "perkeep.org/pkg/client" "perkeep.org/pkg/cmdmain" + "perkeep.org/pkg/schema" "perkeep.org/pkg/search" ) @@ -119,7 +120,7 @@ func (c *listCmd) RunCommand(args []string) error { continue } - if c.camliType == "" || blob.CamliType == c.camliType { + if c.camliType == "" || string(blob.CamliType) == c.camliType { detailed := detail(blob) if detailed != "" { detailed = fmt.Sprintf("\t%v", detailed) @@ -167,8 +168,8 @@ func (c *listCmd) setClient() error { func detail(blob *search.DescribedBlob) string { // TODO(mpl): attrType, value for claim. but I don't think they're accessible just with a describe req. - if blob.CamliType == "file" { + if blob.CamliType == schema.TypeFile { return fmt.Sprintf("%v (%v size=%v)", blob.CamliType, blob.File.FileName, blob.File.Size) } - return blob.CamliType + return string(blob.CamliType) } diff --git a/cmd/pk/makestatic.go b/cmd/pk/makestatic.go index 4ba33972a..6251143fd 100644 --- a/cmd/pk/makestatic.go +++ b/cmd/pk/makestatic.go @@ -79,7 +79,7 @@ func (c *makeStaticCmd) RunCommand(args []string) error { return err } - camliType := func(ref string) string { + camliType := func(ref string) schema.CamliType { m := res.Meta[ref] if m == nil { return "" diff --git a/pkg/blobserver/blobpacked/blobpacked.go b/pkg/blobserver/blobpacked/blobpacked.go index 0d2e65ac7..7fc364fc7 100644 --- a/pkg/blobserver/blobpacked/blobpacked.go +++ b/pkg/blobserver/blobpacked/blobpacked.go @@ -956,7 +956,7 @@ func (s *storage) ReceiveBlob(ctx context.Context, br blob.Ref, source io.Reader size := uint32(buf.Len()) isFile := false fileBlob, err := schema.BlobFromReader(br, bytes.NewReader(buf.Bytes())) - if err == nil && fileBlob.Type() == "file" { + if err == nil && fileBlob.Type() == schema.TypeFile { isFile = true } meta, err := s.getMetaRow(br) diff --git a/pkg/client/upload.go b/pkg/client/upload.go index 517d59266..976a2c1af 100644 --- a/pkg/client/upload.go +++ b/pkg/client/upload.go @@ -464,7 +464,7 @@ func (c *Client) UploadFile(ctx context.Context, filename string, contents io.Re fileMap.SetModTime(modTime) } } - fileMap.SetType("file") + fileMap.SetType(schema.TypeFile) var wholeRef []blob.Ref if opts != nil && opts.WholeRef.Valid() { diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 47a0a0c71..ccc0f3437 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -194,7 +194,7 @@ func (n *node) Open(ctx context.Context, req *fuse.OpenRequest, res *fuse.OpenRe Logger.Printf("open of %v: %v", n.blobref, err) return nil, fuse.EIO } - if ss.Type() == "directory" { + if ss.Type() == schema.TypeDirectory { return n, nil } fr, err := ss.NewFileReader(n.fs.fetcher) @@ -304,13 +304,13 @@ func (n *node) populateAttr() error { } switch meta.Type() { - case "file": + case schema.TypeFile: n.attr.Size = uint64(meta.PartsSize()) n.attr.Blocks = 0 // TODO: set? n.attr.Mode |= 0400 - case "directory": + case schema.TypeDirectory: n.attr.Mode |= 0500 - case "symlink": + case schema.TypeSymlink: n.attr.Mode |= 0400 default: Logger.Printf("unknown attr ss.Type %q in populateAttr", meta.Type()) @@ -371,12 +371,12 @@ func (fs *CamliFileSystem) newNodeFromBlobRef(root blob.Ref) (fusefs.Node, error } switch blob.Type() { - case "directory": + case schema.TypeDirectory: n := &node{fs: fs, blobref: root, meta: blob} n.populateAttr() return n, nil - case "permanode": + case schema.TypePermanode: // other mutDirs listed in the default fileystem have names and are displayed return &mutDir{fs: fs, permanode: root, name: "-"}, nil } diff --git a/pkg/index/corpus.go b/pkg/index/corpus.go index bb0867e61..8bb8316a4 100644 --- a/pkg/index/corpus.go +++ b/pkg/index/corpus.go @@ -69,7 +69,7 @@ type Corpus struct { // camBlobs maps from camliType ("file") to blobref to the meta. // The value is the same one in blobs. - camBlobs map[string]map[blob.Ref]*camtypes.BlobMeta + camBlobs map[schema.CamliType]map[blob.Ref]*camtypes.BlobMeta // TODO: add GoLLRB to vendor; keep sorted BlobMeta keyId signerFromBlobrefMap @@ -326,7 +326,7 @@ func (pm *PermanodeMeta) valuesAtSigner(at time.Time, func newCorpus() *Corpus { c := &Corpus{ blobs: make(map[blob.Ref]*camtypes.BlobMeta), - camBlobs: make(map[string]map[blob.Ref]*camtypes.BlobMeta), + camBlobs: make(map[schema.CamliType]map[blob.Ref]*camtypes.BlobMeta), files: make(map[blob.Ref]camtypes.FileInfo), permanodes: make(map[blob.Ref]*PermanodeMeta), imageInfo: make(map[blob.Ref]camtypes.ImageInfo), @@ -463,8 +463,8 @@ func (c *Corpus) scanFromStorage(s sorted.KeyValue) error { return err } - c.files = make(map[blob.Ref]camtypes.FileInfo, len(c.camBlobs["file"])) - c.permanodes = make(map[blob.Ref]*PermanodeMeta, len(c.camBlobs["permanode"])) + c.files = make(map[blob.Ref]camtypes.FileInfo, len(c.camBlobs[schema.TypeFile])) + c.permanodes = make(map[blob.Ref]*PermanodeMeta, len(c.camBlobs[schema.TypePermanode])) cpu0 := osutil.CPUUsage() var grp syncutil.Group @@ -675,7 +675,7 @@ func (c *Corpus) mergeBlobMeta(bm camtypes.BlobMeta) error { if _, dup := c.blobs[bm.Ref]; dup { panic("dup blob seen") } - bm.CamliType = c.str(bm.CamliType) + bm.CamliType = schema.CamliType((c.str(string(bm.CamliType)))) c.blobs[bm.Ref] = &bm c.sumBlobBytes += int64(bm.Size) @@ -961,7 +961,7 @@ func (c *Corpus) br(br blob.Ref) blob.Ref { // types to call fn for. If empty, all are emitted. // // If fn returns false, iteration ends. -func (c *Corpus) EnumerateCamliBlobs(camType string, fn func(camtypes.BlobMeta) bool) { +func (c *Corpus) EnumerateCamliBlobs(camType schema.CamliType, fn func(camtypes.BlobMeta) bool) { if camType != "" { for _, bm := range c.camBlobs[camType] { if !fn(*bm) { diff --git a/pkg/index/index.go b/pkg/index/index.go index e088c0cd4..aa64b7596 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -1533,7 +1533,7 @@ func (x *Index) EdgesTo(ref blob.Ref, opts *camtypes.EdgesToOpts) (edges []*camt continue } edge.To = ref - if edge.FromType == "permanode" { + if edge.FromType == schema.TypePermanode { permanodeParents[edge.From.String()] = edge } else { edges = append(edges, edge) @@ -1569,7 +1569,7 @@ func kvEdgeBackward(k, v string) (edge *camtypes.Edge, ok bool) { } return &camtypes.Edge{ From: parentRef, - FromType: valPart[0], + FromType: schema.CamliType(valPart[0]), FromTitle: valPart[1], BlobRef: blobRef, }, true @@ -1771,16 +1771,16 @@ var camliTypeMIMEPrefixBytes = []byte(camliTypeMIMEPrefix) // "application/json; camliType=file" => "file" // "image/gif" => "" -func camliTypeFromMIME(mime string) string { +func camliTypeFromMIME(mime string) schema.CamliType { if v := strings.TrimPrefix(mime, camliTypeMIMEPrefix); v != mime { - return v + return schema.CamliType(v) } return "" } -func camliTypeFromMIME_bytes(mime []byte) string { +func camliTypeFromMIME_bytes(mime []byte) schema.CamliType { if v := bytes.TrimPrefix(mime, camliTypeMIMEPrefixBytes); len(v) != len(mime) { - return strutil.StringFromBytes(v) + return schema.CamliType(strutil.StringFromBytes(v)) } return "" } diff --git a/pkg/index/indextest/tests.go b/pkg/index/indextest/tests.go index bad03a478..07f95f3fd 100644 --- a/pkg/index/indextest/tests.go +++ b/pkg/index/indextest/tests.go @@ -686,7 +686,7 @@ func Index(t *testing.T, initIdx func() *index.Index) { if err != nil { t.Errorf("GetBlobMeta(%q) = %v", pn, err) } else { - if e := "permanode"; meta.CamliType != e { + if e := schema.TypePermanode; meta.CamliType != e { t.Errorf("GetBlobMeta(%q) mime = %q, want %q", pn, meta.CamliType, e) } if meta.Size == 0 { diff --git a/pkg/index/receive.go b/pkg/index/receive.go index b592f0f51..94b577a82 100644 --- a/pkg/index/receive.go +++ b/pkg/index/receive.go @@ -336,11 +336,11 @@ func (ix *Index) populateMutationMap(ctx context.Context, fetcher *missTrackFetc var err error if blob, ok := sniffer.SchemaBlob(); ok { switch blob.Type() { - case "claim": + case schema.TypeClaim: err = ix.populateClaim(ctx, fetcher, blob, mm) - case "file": + case schema.TypeFile: err = ix.populateFile(ctx, fetcher, blob, mm) - case "directory": + case schema.TypeDirectory: err = ix.populateDir(ctx, fetcher, blob, mm) } } @@ -827,14 +827,12 @@ func (ix *Index) populateDeleteClaim(ctx context.Context, cl schema.Claim, vr *j return nil } - // TODO(mpl): create consts somewhere for "claim" and "permanode" as camliTypes, and use them, - // instead of hardcoding. Unless they already exist ? (didn't find them). - if meta.CamliType != "permanode" && meta.CamliType != "claim" { + if meta.CamliType != schema.TypePermanode && meta.CamliType != schema.TypeClaim { log.Print(fmt.Errorf("delete claim target in %v is neither a permanode nor a claim: %v", br, meta.CamliType)) return nil } mm.Set(keyDeleted.Key(target, cl.ClaimDateString(), br), "") - if meta.CamliType == "claim" { + if meta.CamliType == schema.TypeClaim { return nil } recentKey := keyRecentPermanode.Key(vr.SignerKeyId, cl.ClaimDateString(), br) diff --git a/pkg/index/sniff.go b/pkg/index/sniff.go index d8a97cc84..98f80bc94 100644 --- a/pkg/index/sniff.go +++ b/pkg/index/sniff.go @@ -32,7 +32,7 @@ type BlobSniffer struct { written int64 meta *schema.Blob // or nil mimeType string - camliType string + camliType schema.CamliType } func NewBlobSniffer(ref blob.Ref) *BlobSniffer { @@ -87,12 +87,12 @@ func (sn *BlobSniffer) Body() ([]byte, error) { // the form "application/json; camliType=foo". func (sn *BlobSniffer) MIMEType() string { return sn.mimeType } -func (sn *BlobSniffer) CamliType() string { return sn.camliType } +func (sn *BlobSniffer) CamliType() schema.CamliType { return sn.camliType } func (sn *BlobSniffer) Parse() { if sn.bufferIsCamliJSON() { sn.camliType = sn.meta.Type() - sn.mimeType = "application/json; camliType=" + sn.camliType + sn.mimeType = "application/json; camliType=" + string(sn.camliType) } else { sn.mimeType = magic.MIMEType(sn.contents) } diff --git a/pkg/schema/blob.go b/pkg/schema/blob.go index 6a64bc3d7..e81970d71 100644 --- a/pkg/schema/blob.go +++ b/pkg/schema/blob.go @@ -60,7 +60,7 @@ type Blob struct { } // Type returns the blob's "camliType" field. -func (b *Blob) Type() string { return b.ss.Type } +func (b *Blob) Type() CamliType { return b.ss.Type } // BlobRef returns the schema blob's blobref. func (b *Blob) BlobRef() blob.Ref { return b.br } @@ -144,7 +144,7 @@ func (b *Blob) AsShare() (s Share, ok bool) { // DirectoryEntries the "entries" field if valid and b's type is "directory". func (b *Blob) DirectoryEntries() (br blob.Ref, ok bool) { - if b.Type() != "directory" { + if b.Type() != TypeDirectory { return } return b.ss.Entries, true @@ -154,7 +154,7 @@ func (b *Blob) DirectoryEntries() (br blob.Ref, ok bool) { // "static-set" schema. Note that if it is a large static-set, the members are // actually spread as subsets in "mergeSets". See StaticSetMergeSets. func (b *Blob) StaticSetMembers() []blob.Ref { - if b.Type() != "static-set" { + if b.Type() != TypeStaticSet { return nil } @@ -170,7 +170,7 @@ func (b *Blob) StaticSetMembers() []blob.Ref { // StaticSetMergeSets returns the refs of the static-sets in "mergeSets". These // are the subsets of all the static-set members in the case of a large directory. func (b *Blob) StaticSetMergeSets() []blob.Ref { - if b.Type() != "static-set" { + if b.Type() != TypeStaticSet { return nil } @@ -287,7 +287,7 @@ func (b *Blob) AsStaticFile() (sf StaticFile, ok bool) { // Perkeep and change the implementation of StaticFile to // reflect that. t := b.ss.Type - if t == "file" || t == "symlink" || t == "fifo" || t == "socket" { + if t == TypeFile || t == TypeSymlink || t == TypeFIFO || t == TypeSocket { return StaticFile{b}, true } @@ -322,7 +322,7 @@ func (sl StaticSymlink) SymlinkTargetString() string { // StaticFile represents a symlink. Othwerwise, it returns the zero // value of StaticSymlink and false. func (sf StaticFile) AsStaticSymlink() (s StaticSymlink, ok bool) { - if sf.b.ss.Type == "symlink" { + if sf.b.ss.Type == TypeSymlink { return StaticSymlink{sf}, true } @@ -333,7 +333,7 @@ func (sf StaticFile) AsStaticSymlink() (s StaticSymlink, ok bool) { // StaticFile represents a fifo. Otherwise, it returns the zero value // of StaticFIFO and false. func (sf StaticFile) AsStaticFIFO() (fifo StaticFIFO, ok bool) { - if sf.b.ss.Type == "fifo" { + if sf.b.ss.Type == TypeFIFO { return StaticFIFO{sf}, true } @@ -344,7 +344,7 @@ func (sf StaticFile) AsStaticFIFO() (fifo StaticFIFO, ok bool) { // StaticFile represents a socket. Otherwise, it returns the zero // value of StaticSocket and false. func (sf StaticFile) AsStaticSocket() (ss StaticSocket, ok bool) { - if sf.b.ss.Type == "socket" { + if sf.b.ss.Type == TypeSocket { return StaticSocket{sf}, true } @@ -369,7 +369,7 @@ func NewBuilder() *Builder { // SetShareTarget sets the target of share claim. // It panics if bb isn't a "share" claim type. func (bb *Builder) SetShareTarget(t blob.Ref) *Builder { - if bb.Type() != "claim" || bb.ClaimType() != ShareClaim { + if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim { panic("called SetShareTarget on non-share") } bb.m["target"] = t.String() @@ -380,7 +380,7 @@ func (bb *Builder) SetShareTarget(t blob.Ref) *Builder { // q is assumed to be of type *search.SearchQuery. // It panics if bb isn't a "share" claim type. func (bb *Builder) SetShareSearch(q SearchQuery) *Builder { - if bb.Type() != "claim" || bb.ClaimType() != ShareClaim { + if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim { panic("called SetShareSearch on non-share") } bb.m["search"] = q @@ -391,7 +391,7 @@ func (bb *Builder) SetShareSearch(q SearchQuery) *Builder { // It panics if bb isn't a "share" claim type. // If t is zero, the expiration is removed. func (bb *Builder) SetShareExpiration(t time.Time) *Builder { - if bb.Type() != "claim" || bb.ClaimType() != ShareClaim { + if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim { panic("called SetShareExpiration on non-share") } if t.IsZero() { @@ -403,7 +403,7 @@ func (bb *Builder) SetShareExpiration(t time.Time) *Builder { } func (bb *Builder) SetShareIsTransitive(b bool) *Builder { - if bb.Type() != "claim" || bb.ClaimType() != ShareClaim { + if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim { panic("called SetShareIsTransitive on non-share") } if !b { @@ -468,7 +468,7 @@ func (bb *Builder) Sign(ctx context.Context, signer *Signer) (string, error) { // for planned permanodes. If the zero value, the current time is used. func (bb *Builder) SignAt(ctx context.Context, signer *Signer, sigTime time.Time) (string, error) { switch bb.Type() { - case "permanode", "claim": + case TypePermanode, TypeClaim: default: return "", fmt.Errorf("can't sign camliType %q", bb.Type()) } @@ -480,15 +480,15 @@ func (bb *Builder) SignAt(ctx context.Context, signer *Signer, sigTime time.Time } // SetType sets the camliType field. -func (bb *Builder) SetType(t string) *Builder { - bb.m["camliType"] = t +func (bb *Builder) SetType(t CamliType) *Builder { + bb.m["camliType"] = string(t) return bb } // Type returns the camliType value. -func (bb *Builder) Type() string { +func (bb *Builder) Type() CamliType { if s, ok := bb.m["camliType"].(string); ok { - return s + return CamliType(s) } return "" } @@ -515,7 +515,7 @@ func (bb *Builder) SetFileName(name string) *Builder { // SetSymlinkTarget sets bb to be of type "symlink" and sets the symlink's target. func (bb *Builder) SetSymlinkTarget(target string) *Builder { - bb.SetType("symlink") + bb.SetType(TypeSymlink) if utf8.ValidString(target) { bb.m["symlinkTarget"] = target } else { @@ -528,7 +528,7 @@ func (bb *Builder) SetSymlinkTarget(target string) *Builder { // which should be signed. (a "claim" or "permanode") func (bb *Builder) IsClaimType() bool { switch bb.Type() { - case "claim", "permanode": + case TypeClaim, TypePermanode: return true } return false @@ -583,7 +583,7 @@ func (bb *Builder) ModTime() (t time.Time, ok bool) { // PopulateDirectoryMap sets the type of *Builder to "directory" and sets // the "entries" field to the provided staticSet blobref. func (bb *Builder) PopulateDirectoryMap(staticSetRef blob.Ref) *Builder { - bb.m["camliType"] = "directory" + bb.m["camliType"] = string(TypeDirectory) bb.m["entries"] = staticSetRef.String() return bb } diff --git a/pkg/schema/fileread_test.go b/pkg/schema/fileread_test.go index c01a0a8b4..4c436ade1 100644 --- a/pkg/schema/fileread_test.go +++ b/pkg/schema/fileread_test.go @@ -144,7 +144,7 @@ func skipBytes(fr *FileReader, skipBytes uint64) uint64 { func TestReader(t *testing.T) { for idx, rt := range readTests { ss := new(superset) - ss.Type = "file" + ss.Type = TypeFile ss.Version = 1 ss.Parts = rt.parts fr, err := ss.NewFileReader(testFetcher) diff --git a/pkg/schema/filewriter.go b/pkg/schema/filewriter.go index 8f9abafce..8960a2ff2 100644 --- a/pkg/schema/filewriter.go +++ b/pkg/schema/filewriter.go @@ -173,7 +173,7 @@ func uploadBytes(ctx context.Context, bs blobserver.StatReceiver, bb *Builder, s // the "file" schema before any of its parts arrive, then the indexer // can get confused. So wait on the parts before, and then upload // the "file" blob afterwards. - if bb.Type() == "file" { + if bb.Type() == TypeFile { future.errc <- nil _, err := future.Get() // may not be nil, if children parts failed future = newUploadBytesFuture() diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 1f498b53f..98a3e2b9a 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -126,7 +126,7 @@ type DirectoryEntry interface { // CamliType returns the schema blob's "camliType" field. // This may be "file", "directory", "symlink", or other more // obscure types added in the future. - CamliType() string + CamliType() CamliType FileName() string BlobRef() blob.Ref @@ -150,7 +150,7 @@ type dirEntry struct { // This type breaks an otherwise-circular dependency. type SearchQuery interface{} -func (de *dirEntry) CamliType() string { +func (de *dirEntry) CamliType() CamliType { return de.ss.Type } @@ -164,8 +164,8 @@ func (de *dirEntry) BlobRef() blob.Ref { func (de *dirEntry) File(ctx context.Context) (File, error) { if de.fr == nil { - if de.ss.Type != "file" { - return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, "file") + if de.ss.Type != TypeFile { + return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, TypeFile) } fr, err := NewFileReader(ctx, de.fetcher, de.ss.BlobRef) if err != nil { @@ -178,8 +178,8 @@ func (de *dirEntry) File(ctx context.Context) (File, error) { func (de *dirEntry) Directory(ctx context.Context) (Directory, error) { if de.dr == nil { - if de.ss.Type != "directory" { - return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, "directory") + if de.ss.Type != TypeDirectory { + return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, TypeDirectory) } dr, err := NewDirReader(ctx, de.fetcher, de.ss.BlobRef) if err != nil { @@ -214,7 +214,7 @@ func newDirectoryEntry(fetcher blob.Fetcher, ss *superset) (DirectoryEntry, erro return nil, errors.New("ss.BlobRef was invalid") } switch ss.Type { - case "file", "directory", "symlink", "fifo", "socket": + case TypeFile, TypeDirectory, TypeSymlink, TypeFIFO, TypeSocket: // Okay default: return nil, fmt.Errorf("invalid DirectoryEntry camliType of %q", ss.Type) @@ -244,8 +244,8 @@ type superset struct { // for convenience. BlobRef blob.Ref - Version int `json:"camliVersion"` - Type string `json:"camliType"` + Version int `json:"camliVersion"` + Type CamliType `json:"camliType"` Signer blob.Ref `json:"camliSigner"` Sig string `json:"camliSig"` @@ -484,20 +484,20 @@ func (ss *superset) FileMode() os.FileMode { // TODO: add other types (block, char, etc) switch ss.Type { - case "directory": + case TypeDirectory: mode = mode | os.ModeDir - case "file": + case TypeFile: // No extra bit. - case "symlink": + case TypeSymlink: mode = mode | os.ModeSymlink - case "fifo": + case TypeFIFO: mode = mode | os.ModeNamedPipe - case "socket": + case TypeSocket: mode = mode | os.ModeSocket } if !hasPerm { switch ss.Type { - case "directory": + case TypeDirectory: mode |= 0755 default: mode |= 0644 @@ -579,7 +579,7 @@ var maxStaticSetMembers = 10000 // NewStaticSet returns the "static-set" schema for a directory. Its members // should be populated with SetStaticSetMembers. func NewStaticSet() *Builder { - return base(1, "static-set") + return base(1, TypeStaticSet) } // SetStaticSetMembers sets the given members as the static-set members of this @@ -590,7 +590,7 @@ func NewStaticSet() *Builder { // static-set created from this builder. // SetStaticSetMembers panics if bb isn't a "static-set" claim type. func (bb *Builder) SetStaticSetMembers(members []blob.Ref) []*Blob { - if bb.Type() != "static-set" { + if bb.Type() != TypeStaticSet { panic("called SetStaticSetMembers on non static-set") } @@ -649,16 +649,16 @@ func (bb *Builder) SetStaticSetMembers(members []blob.Ref) []*Blob { return allSubsets } -func base(version int, ctype string) *Builder { +func base(version int, ctype CamliType) *Builder { return &Builder{map[string]interface{}{ "camliVersion": version, - "camliType": ctype, + "camliType": string(ctype), }} } // NewUnsignedPermanode returns a new random permanode, not yet signed. func NewUnsignedPermanode() *Builder { - bb := base(1, "permanode") + bb := base(1, TypePermanode) chars := make([]byte, 20) _, err := io.ReadFull(rand.Reader, chars) if err != nil { @@ -674,7 +674,7 @@ func NewUnsignedPermanode() *Builder { // GPG date to create consistent JSON encodings of the Map (its // blobref), between runs. func NewPlannedPermanode(key string) *Builder { - bb := base(1, "permanode") + bb := base(1, TypePermanode) bb.m["key"] = key return bb } @@ -711,12 +711,12 @@ func mapJSON(m map[string]interface{}) (string, error) { // NewFileMap returns a new builder of a type "file" schema for the provided fileName. // The chunk parts of the file are not populated. func NewFileMap(fileName string) *Builder { - return newCommonFilenameMap(fileName).SetType("file") + return newCommonFilenameMap(fileName).SetType(TypeFile) } // NewDirMap returns a new builder of a type "directory" schema for the provided fileName. func NewDirMap(fileName string) *Builder { - return newCommonFilenameMap(fileName).SetType("directory") + return newCommonFilenameMap(fileName).SetType(TypeDirectory) } func newCommonFilenameMap(fileName string) *Builder { @@ -785,9 +785,27 @@ func populateParts(m map[string]interface{}, size int64, parts []BytesPart) erro } func newBytes() *Builder { - return base(1, "bytes") + return base(1, TypeBytes) } +// CamliType is one of the valid "camliType" fields in a schema blob. See doc/schema. +type CamliType string + +const ( + TypeBytes CamliType = "bytes" + TypeClaim CamliType = "claim" + TypeDirectory CamliType = "directory" + TypeFIFO CamliType = "fifo" + TypeFile CamliType = "file" + TypeInode CamliType = "inode" + TypeKeep CamliType = "keep" + TypePermanode CamliType = "permanode" + TypeShare CamliType = "share" + TypeSocket CamliType = "socket" + TypeStaticSet CamliType = "static-set" + TypeSymlink CamliType = "symlink" +) + // ClaimType is one of the valid "claimType" fields in a "claim" schema blob. See doc/schema/claims/. type ClaimType string @@ -819,7 +837,7 @@ type claimParam struct { } func newClaim(claims ...*claimParam) *Builder { - bb := base(1, "claim") + bb := base(1, TypeClaim) bb.SetClaimDate(clockNow()) if len(claims) == 1 { cp := claims[0] diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index adbdd4b08..8b5afc6ae 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -553,7 +553,7 @@ func TestStaticFileAndStaticSymlink(t *testing.T) { } bb := NewCommonFileMap(fd.Name(), fi) - bb.SetType("file") + bb.SetType(TypeFile) bb.SetFileName(fd.Name()) blob := bb.Blob() @@ -592,7 +592,7 @@ func TestStaticFileAndStaticSymlink(t *testing.T) { } bb = NewCommonFileMap(src, fi) - bb.SetType("symlink") + bb.SetType(TypeSymlink) bb.SetFileName(src) bb.SetSymlinkTarget(target) blob = bb.Blob() @@ -638,7 +638,7 @@ func TestStaticFIFO(t *testing.T) { } bb := NewCommonFileMap(fifoPath, fi) - bb.SetType("fifo") + bb.SetType(TypeFIFO) bb.SetFileName(fifoPath) blob := bb.Blob() t.Logf("Got JSON for fifo: %s\n", blob.JSON()) @@ -675,7 +675,7 @@ func TestStaticSocket(t *testing.T) { } bb := NewCommonFileMap(sockPath, fi) - bb.SetType("socket") + bb.SetType(TypeSocket) bb.SetFileName(sockPath) blob := bb.Blob() t.Logf("Got JSON for socket: %s\n", blob.JSON()) diff --git a/pkg/search/describe.go b/pkg/search/describe.go index 271fa0c4b..162762790 100644 --- a/pkg/search/describe.go +++ b/pkg/search/describe.go @@ -34,6 +34,7 @@ import ( "go4.org/types" "perkeep.org/internal/httputil" "perkeep.org/pkg/blob" + "perkeep.org/pkg/schema" "perkeep.org/pkg/types/camtypes" ) @@ -223,9 +224,9 @@ type MetaMap map[string]*DescribedBlob type DescribedBlob struct { Request *DescribeRequest `json:"-"` - BlobRef blob.Ref `json:"blobRef"` - CamliType string `json:"camliType,omitempty"` - Size int64 `json:"size,"` + BlobRef blob.Ref `json:"blobRef"` + CamliType schema.CamliType `json:"camliType,omitempty"` + Size int64 `json:"size,"` // if camliType "permanode" Permanode *DescribedPermanode `json:"permanode,omitempty"` @@ -739,7 +740,7 @@ func (dr *DescribeRequest) doDescribe(ctx context.Context, br blob.Ref, depth in // maps. Then add JSON marhsallers to those types. Add tests. des := dr.describedBlob(br) if meta.CamliType != "" { - des.setMIMEType("application/json; camliType=" + meta.CamliType) + des.setMIMEType("application/json; camliType=" + string(meta.CamliType)) } des.Size = int64(meta.Size) @@ -907,6 +908,6 @@ func (dr *DescribeRequest) describeRefs(ctx context.Context, str string, depth i func (b *DescribedBlob) setMIMEType(mime string) { if strings.HasPrefix(mime, camliTypePrefix) { - b.CamliType = strings.TrimPrefix(mime, camliTypePrefix) + b.CamliType = schema.CamliType(strings.TrimPrefix(mime, camliTypePrefix)) } } diff --git a/pkg/search/handler.go b/pkg/search/handler.go index b6f601d3b..a58444755 100644 --- a/pkg/search/handler.go +++ b/pkg/search/handler.go @@ -39,6 +39,7 @@ import ( "perkeep.org/pkg/blobserver" "perkeep.org/pkg/index" "perkeep.org/pkg/jsonsign" + "perkeep.org/pkg/schema" "perkeep.org/pkg/types/camtypes" "perkeep.org/pkg/types/serverconfig" ) @@ -504,8 +505,8 @@ type EdgesResponse struct { // An EdgeItem is an item returned from $searchRoot/camli/search/edgesto. type EdgeItem struct { - From blob.Ref `json:"from"` - FromType string `json:"fromType"` + From blob.Ref `json:"from"` + FromType schema.CamliType `json:"fromType"` } var testHookBug121 = func() {} @@ -808,14 +809,14 @@ func (h *Handler) EdgesTo(req *EdgesRequest) (*EdgesResponse, error) { if found { ei = &EdgeItem{ From: edge.From, - FromType: "permanode", + FromType: schema.TypePermanode, } } resc <- edgeOrError{edge: ei} } verifying := 0 for _, edge := range edges { - if edge.FromType == "permanode" { + if edge.FromType == schema.TypePermanode { verifying++ go verify(edge) continue diff --git a/pkg/search/query.go b/pkg/search/query.go index 1f74ca2e5..e06eca8d5 100644 --- a/pkg/search/query.go +++ b/pkg/search/query.go @@ -35,6 +35,7 @@ import ( "perkeep.org/pkg/blob" "perkeep.org/pkg/index" + "perkeep.org/pkg/schema" "perkeep.org/pkg/types/camtypes" "context" @@ -294,9 +295,9 @@ type Constraint struct { // Anything, if true, matches all blobs. Anything bool `json:"anything,omitempty"` - CamliType string `json:"camliType,omitempty"` // camliType of the JSON blob - AnyCamliType bool `json:"anyCamliType,omitempty"` // if true, any camli JSON blob matches - BlobRefPrefix string `json:"blobRefPrefix,omitempty"` + CamliType schema.CamliType `json:"camliType,omitempty"` // camliType of the JSON blob + AnyCamliType bool `json:"anyCamliType,omitempty"` // if true, any camli JSON blob matches + BlobRefPrefix string `json:"blobRefPrefix,omitempty"` File *FileConstraint `json:"file,omitempty"` Dir *DirConstraint `json:"dir,omitempty"` @@ -383,7 +384,7 @@ func (c *Constraint) matchesAtMostOneBlob() blob.Ref { } func (c *Constraint) onlyMatchesPermanode() bool { - if c.Permanode != nil || c.CamliType == "permanode" { + if c.Permanode != nil || c.CamliType == schema.TypePermanode { return true } @@ -1455,7 +1456,7 @@ func (q *SearchQuery) pickCandidateSource(s *search) (src candidateSource) { if c.matchesFileByWholeRef() { src.name = "corpus_file_meta" src.send = func(ctx context.Context, s *search, fn func(camtypes.BlobMeta) bool) error { - corpus.EnumerateCamliBlobs("file", fn) + corpus.EnumerateCamliBlobs(schema.TypeFile, fn) return nil } return @@ -1679,7 +1680,7 @@ func (c *PermanodeConstraint) hasValueConstraint() bool { } func (c *PermanodeConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (ok bool, err error) { - if bm.CamliType != "permanode" { + if bm.CamliType != schema.TypePermanode { return false, nil } corpus := s.h.corpus @@ -2085,7 +2086,7 @@ func (c *Constraint) fileOrDirOrLogicalMatches(ctx context.Context, s *search, b } func (c *DirConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) { - if bm.CamliType != "directory" { + if bm.CamliType != schema.TypeDirectory { return false, nil } // TODO(mpl): I've added c.BlobRefPrefix, so that c.ParentDir can be directly diff --git a/pkg/search/websocket.go b/pkg/search/websocket.go index 6dc1ba7a4..3fd3ea7e0 100644 --- a/pkg/search/websocket.go +++ b/pkg/search/websocket.go @@ -28,6 +28,7 @@ import ( "time" "github.com/gorilla/websocket" + "perkeep.org/pkg/schema" ) const ( @@ -51,7 +52,7 @@ type wsHub struct { register chan *wsConn unregister chan *wsConn watchReq chan watchReq - newBlobRecv chan string // new blob received. string is camliType. + newBlobRecv chan schema.CamliType // new blob received. updatedResults chan *watchedQuery statusUpdate chan json.RawMessage @@ -66,7 +67,7 @@ func newWebsocketHub(sh *Handler) *wsHub { unregister: make(chan *wsConn), // unbuffered; issue 563 conns: make(map[*wsConn]bool), watchReq: make(chan watchReq, buffered), - newBlobRecv: make(chan string, buffered), + newBlobRecv: make(chan schema.CamliType, buffered), updatedResults: make(chan *watchedQuery, buffered), statusUpdate: make(chan json.RawMessage, buffered), } diff --git a/pkg/server/download.go b/pkg/server/download.go index efd75474f..e349a9bae 100644 --- a/pkg/server/download.go +++ b/pkg/server/download.go @@ -141,10 +141,10 @@ func (dh *DownloadHandler) fileInfo(ctx context.Context, file blob.Ref) (fi file return fi, false, fmt.Errorf("could not read %v as blob: %v", file, err) } tp := b.Type() - if tp != "file" { + if tp != schema.TypeFile { // for non-regular files var contents string - if tp == "symlink" { + if tp == schema.TypeSymlink { sf, _ := b.AsStaticFile() sl, _ := sf.AsStaticSymlink() contents = sl.SymlinkTargetString() @@ -416,7 +416,12 @@ func (dh *DownloadHandler) statFiles(refs []blob.Ref) error { return nil } -var allowedFileTypes = map[string]bool{"file": true, "symlink": true, "fifo": true, "socket": true} +var allowedFileTypes = map[schema.CamliType]bool{ + schema.TypeFile: true, + schema.TypeSymlink: true, + schema.TypeFIFO: true, + schema.TypeSocket: true, +} // checkFiles reads, and discards, the file contents for each of the given file refs. // It is used to check that all files requested for download are readable before @@ -435,10 +440,10 @@ func (dh *DownloadHandler) checkFiles(ctx context.Context, parentPath string, fi return fmt.Errorf("could not read %v as blob: %v", br, err) } tp := b.Type() - if _, ok := allowedFileTypes[tp]; !ok && tp != "directory" { + if _, ok := allowedFileTypes[tp]; !ok && tp != schema.TypeDirectory { return fmt.Errorf("%v not a supported file or directory type: %q", br, tp) } - if tp == "directory" { + if tp == schema.TypeDirectory { dr, err := b.NewDirReader(ctx, dh.Fetcher) if err != nil { return fmt.Errorf("could not open %v as directory: %v", br, err) @@ -452,7 +457,7 @@ func (dh *DownloadHandler) checkFiles(ctx context.Context, parentPath string, fi } continue } - if tp != "file" { + if tp != schema.TypeFile { // We only bother checking regular files. symlinks, fifos, and sockets are // assumed ok. dh.pathByRef[br] = filepath.Join(parentPath, b.FileName()) diff --git a/pkg/server/filetree.go b/pkg/server/filetree.go index 77fe4f925..48d1ed4ef 100644 --- a/pkg/server/filetree.go +++ b/pkg/server/filetree.go @@ -37,7 +37,7 @@ type FileTreeNode struct { Name string `json:"name"` // Type is the camliType of the node. This may be "file", "directory", "symlink" // or other in the future. - Type string `json:"type"` + Type schema.CamliType `json:"type"` // BlobRef is the blob.Ref of the node. BlobRef blob.Ref `json:"blobRef"` } diff --git a/pkg/server/import_share.go b/pkg/server/import_share.go index f088d257e..1529ecb2f 100644 --- a/pkg/server/import_share.go +++ b/pkg/server/import_share.go @@ -167,7 +167,7 @@ func (si *shareImporter) imprt(ctx context.Context, br blob.Ref) error { return nil // TODO(mpl): other camliTypes, at least symlink. default: - return errors.New("unknown blob type: " + b.Type()) + return errors.New("unknown blob type: " + string(b.Type())) } } diff --git a/pkg/server/share.go b/pkg/server/share.go index f8792e5b6..e9e4a12d5 100644 --- a/pkg/server/share.go +++ b/pkg/server/share.go @@ -324,7 +324,7 @@ func bytesHaveSchemaLink(br blob.Ref, bb []byte, target blob.Ref) bool { } typ := b.Type() switch typ { - case "file", "bytes": + case schema.TypeFile, schema.TypeBytes: for _, bp := range b.ByteParts() { if bp.BlobRef.Valid() { if bp.BlobRef == target { @@ -337,11 +337,11 @@ func bytesHaveSchemaLink(br blob.Ref, bb []byte, target blob.Ref) bool { } } } - case "directory": + case schema.TypeDirectory: if d, ok := b.DirectoryEntries(); ok { return d == target } - case "static-set": + case schema.TypeStaticSet: for _, m := range b.StaticSetMembers() { if m == target { return true diff --git a/pkg/types/camtypes/search.go b/pkg/types/camtypes/search.go index 778de9640..0b3d888c8 100644 --- a/pkg/types/camtypes/search.go +++ b/pkg/types/camtypes/search.go @@ -27,6 +27,7 @@ import ( "perkeep.org/internal/magic" "perkeep.org/pkg/blob" + "perkeep.org/pkg/schema" "go4.org/types" ) @@ -196,8 +197,8 @@ type EdgesToOpts struct { type Edge struct { From blob.Ref - FromType string // "permanode", "directory", etc - FromTitle string // name of source permanode or directory + FromType schema.CamliType // "permanode", "directory", etc + FromTitle string // name of source permanode or directory To blob.Ref BlobRef blob.Ref // the blob responsible for the edge relationship } @@ -214,7 +215,7 @@ type BlobMeta struct { // CamliType is non-empty if this blob is a Perkeep JSON // schema blob. If so, this is its "camliType" attribute. - CamliType string + CamliType schema.CamliType // TODO(bradfitz): change CamliTypethis *string to save 8 bytes }