Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pkg/schema: create CamliType type 

Create a CamliType type in pkg/schema and use it in a couple of
packages.

It can be implemented in other packages as we go.
This commit is contained in:
aviau 2021-01-16 22:05:35 -05:00 committed by Alexandre Viau
parent 9534e2b534
commit 231ba4233f
30 changed files with 161 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/publisher/main.go
View File

@ -55,6 +55,7 @@ import (
"perkeep.org/pkg/constants"
"perkeep.org/pkg/fileembed"
"perkeep.org/pkg/publish"
"perkeep.org/pkg/schema"
"perkeep.org/pkg/search"
"perkeep.org/pkg/server"
"perkeep.org/pkg/sorted"
@ -869,7 +870,7 @@ func (pr *publishRequest) serveSubjectTemplate() {
pr.ph.cacheDescribed(res.Meta)
subdes := res.Meta[pr.subject.String()]
if subdes.CamliType == "file" {
if subdes.CamliType == schema.TypeFile {
pr.serveFileDownload(subdes)
return
}

4
app/publisher/zip.go
View File

@ -169,9 +169,9 @@ func (zh *zipHandler) blobsFromDir(dirPath string, dirBlob blob.Ref) ([]*blobFil
for _, v := range ent {
fullpath := path.Join(dirPath, v.FileName())
switch v.CamliType() {
case "file":
case schema.TypeFile:
list = append(list, &blobFile{v.BlobRef(), fullpath})
case "directory":
case schema.TypeDirectory:
children, err := zh.blobsFromDir(fullpath, v.BlobRef())
if err != nil {
return nil, fmt.Errorf("Could not get list of blobs from %v: %v", v.BlobRef(), err)

14
cmd/pk-get/get.go
View File

@ -240,7 +240,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref)
rcc.Close()
switch b.Type() {
case "directory":
case schema.TypeDirectory:
dir := filepath.Join(targ, b.FileName())
if *flagVerbose {
log.Printf("Fetching directory %v into %s", br, dir)
@ -256,7 +256,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref)
return fmt.Errorf("bad entries blobref in dir %v", b.BlobRef())
}
return smartFetch(ctx, src, dir, entries)
case "static-set":
case schema.TypeStaticSet:
if *flagVerbose {
log.Printf("Fetching directory entries %v into %s", br, targ)
}
@ -289,7 +289,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref)
}
}
return nil
case "file":
case schema.TypeFile:
fr, err := schema.NewFileReader(ctx, src, br)
if err != nil {
return fmt.Errorf("NewFileReader: %v", err)
@ -322,7 +322,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref)
log.Print(err)
}
return nil
case "symlink":
case schema.TypeSymlink:
if *flagSkipIrregular {
return nil
}
@ -354,7 +354,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref)
// os.Chtimes always dereferences (does not act on the
// symlink but its target).
return err
case "fifo":
case schema.TypeFIFO:
if *flagSkipIrregular {
return nil
}
@ -389,7 +389,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref)
return nil
case "socket":
case schema.TypeSocket:
if *flagSkipIrregular {
return nil
}
@ -425,7 +425,7 @@ func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref)
return nil
default:
return errors.New("unknown blob type: " + b.Type())
return errors.New("unknown blob type: " + string(b.Type()))
}
panic("unreachable")
}

2
cmd/pk-get/graph.go
View File

@ -53,7 +53,7 @@ func (n *node) dotLabel() string {
if n.blob == nil {
return fmt.Sprintf("%s\n%d bytes", name, n.size)
}
return name + "\n" + n.blob.Type()
return name + "\n" + string(n.blob.Type())
}
func (n *node) color() string {

4
cmd/pk-put/files.go
View File

@ -391,9 +391,9 @@ func (up *Uploader) uploadNode(ctx context.Context, n *node) (*client.PutResult,
// including mode & os.ModeCharDevice
fallthrough
case mode&os.ModeSocket != 0:
bb.SetType("socket")
bb.SetType(schema.TypeSocket)
case mode&os.ModeNamedPipe != 0: // fifo
bb.SetType("fifo")
bb.SetType(schema.TypeFIFO)
default:
return nil, fmt.Errorf("pk-put.files: unsupported file type %v for file %v", mode, n.fullPath)
case fi.IsDir():

3
cmd/pk-put/put.go
View File

@ -33,6 +33,7 @@ import (
"perkeep.org/pkg/blobserver/dir"
"perkeep.org/pkg/client"
"perkeep.org/pkg/cmdmain"
"perkeep.org/pkg/schema"
"go4.org/syncutil"
)
@ -119,7 +120,7 @@ func initUploader() {
uploader = up
}
func handleResult(what string, pr *client.PutResult, err error) error {
func handleResult(what schema.CamliType, pr *client.PutResult, err error) error {
if err != nil {
cmdmain.Errorf("Error putting %s: %s\n", what, err)
cmdmain.ExitWithFailure = true

7
cmd/pk/list.go
View File

@ -28,6 +28,7 @@ import (
"perkeep.org/pkg/blob"
"perkeep.org/pkg/client"
"perkeep.org/pkg/cmdmain"
"perkeep.org/pkg/schema"
"perkeep.org/pkg/search"
)
@ -119,7 +120,7 @@ func (c *listCmd) RunCommand(args []string) error {
continue
}
if c.camliType == "" || blob.CamliType == c.camliType {
if c.camliType == "" || string(blob.CamliType) == c.camliType {
detailed := detail(blob)
if detailed != "" {
detailed = fmt.Sprintf("\t%v", detailed)
@ -167,8 +168,8 @@ func (c *listCmd) setClient() error {
func detail(blob *search.DescribedBlob) string {
// TODO(mpl): attrType, value for claim. but I don't think they're accessible just with a describe req.
if blob.CamliType == "file" {
if blob.CamliType == schema.TypeFile {
return fmt.Sprintf("%v (%v size=%v)", blob.CamliType, blob.File.FileName, blob.File.Size)
}
return blob.CamliType
return string(blob.CamliType)
}

2
cmd/pk/makestatic.go
View File

@ -79,7 +79,7 @@ func (c *makeStaticCmd) RunCommand(args []string) error {
return err
}
camliType := func(ref string) string {
camliType := func(ref string) schema.CamliType {
m := res.Meta[ref]
if m == nil {
return ""

2
pkg/blobserver/blobpacked/blobpacked.go
View File

@ -956,7 +956,7 @@ func (s *storage) ReceiveBlob(ctx context.Context, br blob.Ref, source io.Reader
size := uint32(buf.Len())
isFile := false
fileBlob, err := schema.BlobFromReader(br, bytes.NewReader(buf.Bytes()))
if err == nil && fileBlob.Type() == "file" {
if err == nil && fileBlob.Type() == schema.TypeFile {
isFile = true
}
meta, err := s.getMetaRow(br)

2
pkg/client/upload.go
View File

@ -464,7 +464,7 @@ func (c *Client) UploadFile(ctx context.Context, filename string, contents io.Re
fileMap.SetModTime(modTime)
}
}
fileMap.SetType("file")
fileMap.SetType(schema.TypeFile)
var wholeRef []blob.Ref
if opts != nil && opts.WholeRef.Valid() {

12
pkg/fs/fs.go
View File

@ -194,7 +194,7 @@ func (n *node) Open(ctx context.Context, req *fuse.OpenRequest, res *fuse.OpenRe
Logger.Printf("open of %v: %v", n.blobref, err)
return nil, fuse.EIO
}
if ss.Type() == "directory" {
if ss.Type() == schema.TypeDirectory {
return n, nil
}
fr, err := ss.NewFileReader(n.fs.fetcher)
@ -304,13 +304,13 @@ func (n *node) populateAttr() error {
}
switch meta.Type() {
case "file":
case schema.TypeFile:
n.attr.Size = uint64(meta.PartsSize())
n.attr.Blocks = 0 // TODO: set?
n.attr.Mode |= 0400
case "directory":
case schema.TypeDirectory:
n.attr.Mode |= 0500
case "symlink":
case schema.TypeSymlink:
n.attr.Mode |= 0400
default:
Logger.Printf("unknown attr ss.Type %q in populateAttr", meta.Type())
@ -371,12 +371,12 @@ func (fs *CamliFileSystem) newNodeFromBlobRef(root blob.Ref) (fusefs.Node, error
}
switch blob.Type() {
case "directory":
case schema.TypeDirectory:
n := &node{fs: fs, blobref: root, meta: blob}
n.populateAttr()
return n, nil
case "permanode":
case schema.TypePermanode:
// other mutDirs listed in the default fileystem have names and are displayed
return &mutDir{fs: fs, permanode: root, name: "-"}, nil
}

12
pkg/index/corpus.go
View File

@ -69,7 +69,7 @@ type Corpus struct {
// camBlobs maps from camliType ("file") to blobref to the meta.
// The value is the same one in blobs.
camBlobs map[string]map[blob.Ref]*camtypes.BlobMeta
camBlobs map[schema.CamliType]map[blob.Ref]*camtypes.BlobMeta
// TODO: add GoLLRB to vendor; keep sorted BlobMeta
keyId signerFromBlobrefMap
@ -326,7 +326,7 @@ func (pm *PermanodeMeta) valuesAtSigner(at time.Time,
func newCorpus() *Corpus {
c := &Corpus{
blobs: make(map[blob.Ref]*camtypes.BlobMeta),
camBlobs: make(map[string]map[blob.Ref]*camtypes.BlobMeta),
camBlobs: make(map[schema.CamliType]map[blob.Ref]*camtypes.BlobMeta),
files: make(map[blob.Ref]camtypes.FileInfo),
permanodes: make(map[blob.Ref]*PermanodeMeta),
imageInfo: make(map[blob.Ref]camtypes.ImageInfo),
@ -463,8 +463,8 @@ func (c *Corpus) scanFromStorage(s sorted.KeyValue) error {
return err
}
c.files = make(map[blob.Ref]camtypes.FileInfo, len(c.camBlobs["file"]))
c.permanodes = make(map[blob.Ref]*PermanodeMeta, len(c.camBlobs["permanode"]))
c.files = make(map[blob.Ref]camtypes.FileInfo, len(c.camBlobs[schema.TypeFile]))
c.permanodes = make(map[blob.Ref]*PermanodeMeta, len(c.camBlobs[schema.TypePermanode]))
cpu0 := osutil.CPUUsage()
var grp syncutil.Group
@ -675,7 +675,7 @@ func (c *Corpus) mergeBlobMeta(bm camtypes.BlobMeta) error {
if _, dup := c.blobs[bm.Ref]; dup {
panic("dup blob seen")
}
bm.CamliType = c.str(bm.CamliType)
bm.CamliType = schema.CamliType((c.str(string(bm.CamliType))))
c.blobs[bm.Ref] = &bm
c.sumBlobBytes += int64(bm.Size)
@ -961,7 +961,7 @@ func (c *Corpus) br(br blob.Ref) blob.Ref {
// types to call fn for. If empty, all are emitted.
//
// If fn returns false, iteration ends.
func (c *Corpus) EnumerateCamliBlobs(camType string, fn func(camtypes.BlobMeta) bool) {
func (c *Corpus) EnumerateCamliBlobs(camType schema.CamliType, fn func(camtypes.BlobMeta) bool) {
if camType != "" {
for _, bm := range c.camBlobs[camType] {
if !fn(*bm) {

12
pkg/index/index.go
View File

@ -1533,7 +1533,7 @@ func (x *Index) EdgesTo(ref blob.Ref, opts *camtypes.EdgesToOpts) (edges []*camt
continue
}
edge.To = ref
if edge.FromType == "permanode" {
if edge.FromType == schema.TypePermanode {
permanodeParents[edge.From.String()] = edge
} else {
edges = append(edges, edge)
@ -1569,7 +1569,7 @@ func kvEdgeBackward(k, v string) (edge *camtypes.Edge, ok bool) {
}
return &camtypes.Edge{
From: parentRef,
FromType: valPart[0],
FromType: schema.CamliType(valPart[0]),
FromTitle: valPart[1],
BlobRef: blobRef,
}, true
@ -1771,16 +1771,16 @@ var camliTypeMIMEPrefixBytes = []byte(camliTypeMIMEPrefix)
// "application/json; camliType=file" => "file"
// "image/gif" => ""
func camliTypeFromMIME(mime string) string {
func camliTypeFromMIME(mime string) schema.CamliType {
if v := strings.TrimPrefix(mime, camliTypeMIMEPrefix); v != mime {
return v
return schema.CamliType(v)
}
return ""
}
func camliTypeFromMIME_bytes(mime []byte) string {
func camliTypeFromMIME_bytes(mime []byte) schema.CamliType {
if v := bytes.TrimPrefix(mime, camliTypeMIMEPrefixBytes); len(v) != len(mime) {
return strutil.StringFromBytes(v)
return schema.CamliType(strutil.StringFromBytes(v))
}
return ""
}

2
pkg/index/indextest/tests.go
View File

@ -686,7 +686,7 @@ func Index(t *testing.T, initIdx func() *index.Index) {
if err != nil {
t.Errorf("GetBlobMeta(%q) = %v", pn, err)
} else {
if e := "permanode"; meta.CamliType != e {
if e := schema.TypePermanode; meta.CamliType != e {
t.Errorf("GetBlobMeta(%q) mime = %q, want %q", pn, meta.CamliType, e)
}
if meta.Size == 0 {

12
pkg/index/receive.go
View File

@ -336,11 +336,11 @@ func (ix *Index) populateMutationMap(ctx context.Context, fetcher *missTrackFetc
var err error
if blob, ok := sniffer.SchemaBlob(); ok {
switch blob.Type() {
case "claim":
case schema.TypeClaim:
err = ix.populateClaim(ctx, fetcher, blob, mm)
case "file":
case schema.TypeFile:
err = ix.populateFile(ctx, fetcher, blob, mm)
case "directory":
case schema.TypeDirectory:
err = ix.populateDir(ctx, fetcher, blob, mm)
}
}
@ -827,14 +827,12 @@ func (ix *Index) populateDeleteClaim(ctx context.Context, cl schema.Claim, vr *j
return nil
}
// TODO(mpl): create consts somewhere for "claim" and "permanode" as camliTypes, and use them,
// instead of hardcoding. Unless they already exist ? (didn't find them).
if meta.CamliType != "permanode" && meta.CamliType != "claim" {
if meta.CamliType != schema.TypePermanode && meta.CamliType != schema.TypeClaim {
log.Print(fmt.Errorf("delete claim target in %v is neither a permanode nor a claim: %v", br, meta.CamliType))
return nil
}
mm.Set(keyDeleted.Key(target, cl.ClaimDateString(), br), "")
if meta.CamliType == "claim" {
if meta.CamliType == schema.TypeClaim {
return nil
}
recentKey := keyRecentPermanode.Key(vr.SignerKeyId, cl.ClaimDateString(), br)

6
pkg/index/sniff.go
View File

@ -32,7 +32,7 @@ type BlobSniffer struct {
written int64
meta *schema.Blob // or nil
mimeType string
camliType string
camliType schema.CamliType
}
func NewBlobSniffer(ref blob.Ref) *BlobSniffer {
@ -87,12 +87,12 @@ func (sn *BlobSniffer) Body() ([]byte, error) {
// the form "application/json; camliType=foo".
func (sn *BlobSniffer) MIMEType() string { return sn.mimeType }
func (sn *BlobSniffer) CamliType() string { return sn.camliType }
func (sn *BlobSniffer) CamliType() schema.CamliType { return sn.camliType }
func (sn *BlobSniffer) Parse() {
if sn.bufferIsCamliJSON() {
sn.camliType = sn.meta.Type()
sn.mimeType = "application/json; camliType=" + sn.camliType
sn.mimeType = "application/json; camliType=" + string(sn.camliType)
} else {
sn.mimeType = magic.MIMEType(sn.contents)
}

40
pkg/schema/blob.go
View File

@ -60,7 +60,7 @@ type Blob struct {
}
// Type returns the blob's "camliType" field.
func (b *Blob) Type() string { return b.ss.Type }
func (b *Blob) Type() CamliType { return b.ss.Type }
// BlobRef returns the schema blob's blobref.
func (b *Blob) BlobRef() blob.Ref { return b.br }
@ -144,7 +144,7 @@ func (b *Blob) AsShare() (s Share, ok bool) {
// DirectoryEntries the "entries" field if valid and b's type is "directory".
func (b *Blob) DirectoryEntries() (br blob.Ref, ok bool) {
if b.Type() != "directory" {
if b.Type() != TypeDirectory {
return
}
return b.ss.Entries, true
@ -154,7 +154,7 @@ func (b *Blob) DirectoryEntries() (br blob.Ref, ok bool) {
// "static-set" schema. Note that if it is a large static-set, the members are
// actually spread as subsets in "mergeSets". See StaticSetMergeSets.
func (b *Blob) StaticSetMembers() []blob.Ref {
if b.Type() != "static-set" {
if b.Type() != TypeStaticSet {
return nil
}
@ -170,7 +170,7 @@ func (b *Blob) StaticSetMembers() []blob.Ref {
// StaticSetMergeSets returns the refs of the static-sets in "mergeSets". These
// are the subsets of all the static-set members in the case of a large directory.
func (b *Blob) StaticSetMergeSets() []blob.Ref {
if b.Type() != "static-set" {
if b.Type() != TypeStaticSet {
return nil
}
@ -287,7 +287,7 @@ func (b *Blob) AsStaticFile() (sf StaticFile, ok bool) {
// Perkeep and change the implementation of StaticFile to
// reflect that.
t := b.ss.Type
if t == "file" || t == "symlink" || t == "fifo" || t == "socket" {
if t == TypeFile || t == TypeSymlink || t == TypeFIFO || t == TypeSocket {
return StaticFile{b}, true
}
@ -322,7 +322,7 @@ func (sl StaticSymlink) SymlinkTargetString() string {
// StaticFile represents a symlink. Othwerwise, it returns the zero
// value of StaticSymlink and false.
func (sf StaticFile) AsStaticSymlink() (s StaticSymlink, ok bool) {
if sf.b.ss.Type == "symlink" {
if sf.b.ss.Type == TypeSymlink {
return StaticSymlink{sf}, true
}
@ -333,7 +333,7 @@ func (sf StaticFile) AsStaticSymlink() (s StaticSymlink, ok bool) {
// StaticFile represents a fifo. Otherwise, it returns the zero value
// of StaticFIFO and false.
func (sf StaticFile) AsStaticFIFO() (fifo StaticFIFO, ok bool) {
if sf.b.ss.Type == "fifo" {
if sf.b.ss.Type == TypeFIFO {
return StaticFIFO{sf}, true
}
@ -344,7 +344,7 @@ func (sf StaticFile) AsStaticFIFO() (fifo StaticFIFO, ok bool) {
// StaticFile represents a socket. Otherwise, it returns the zero
// value of StaticSocket and false.
func (sf StaticFile) AsStaticSocket() (ss StaticSocket, ok bool) {
if sf.b.ss.Type == "socket" {
if sf.b.ss.Type == TypeSocket {
return StaticSocket{sf}, true
}
@ -369,7 +369,7 @@ func NewBuilder() *Builder {
// SetShareTarget sets the target of share claim.
// It panics if bb isn't a "share" claim type.
func (bb *Builder) SetShareTarget(t blob.Ref) *Builder {
if bb.Type() != "claim" || bb.ClaimType() != ShareClaim {
if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim {
panic("called SetShareTarget on non-share")
}
bb.m["target"] = t.String()
@ -380,7 +380,7 @@ func (bb *Builder) SetShareTarget(t blob.Ref) *Builder {
// q is assumed to be of type *search.SearchQuery.
// It panics if bb isn't a "share" claim type.
func (bb *Builder) SetShareSearch(q SearchQuery) *Builder {
if bb.Type() != "claim" || bb.ClaimType() != ShareClaim {
if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim {
panic("called SetShareSearch on non-share")
}
bb.m["search"] = q
@ -391,7 +391,7 @@ func (bb *Builder) SetShareSearch(q SearchQuery) *Builder {
// It panics if bb isn't a "share" claim type.
// If t is zero, the expiration is removed.
func (bb *Builder) SetShareExpiration(t time.Time) *Builder {
if bb.Type() != "claim" || bb.ClaimType() != ShareClaim {
if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim {
panic("called SetShareExpiration on non-share")
}
if t.IsZero() {
@ -403,7 +403,7 @@ func (bb *Builder) SetShareExpiration(t time.Time) *Builder {
}
func (bb *Builder) SetShareIsTransitive(b bool) *Builder {
if bb.Type() != "claim" || bb.ClaimType() != ShareClaim {
if bb.Type() != TypeClaim || bb.ClaimType() != ShareClaim {
panic("called SetShareIsTransitive on non-share")
}
if !b {
@ -468,7 +468,7 @@ func (bb *Builder) Sign(ctx context.Context, signer *Signer) (string, error) {
// for planned permanodes. If the zero value, the current time is used.
func (bb *Builder) SignAt(ctx context.Context, signer *Signer, sigTime time.Time) (string, error) {
switch bb.Type() {
case "permanode", "claim":
case TypePermanode, TypeClaim:
default:
return "", fmt.Errorf("can't sign camliType %q", bb.Type())
}
@ -480,15 +480,15 @@ func (bb *Builder) SignAt(ctx context.Context, signer *Signer, sigTime time.Time
}
// SetType sets the camliType field.
func (bb *Builder) SetType(t string) *Builder {
bb.m["camliType"] = t
func (bb *Builder) SetType(t CamliType) *Builder {
bb.m["camliType"] = string(t)
return bb
}
// Type returns the camliType value.
func (bb *Builder) Type() string {
func (bb *Builder) Type() CamliType {
if s, ok := bb.m["camliType"].(string); ok {
return s
return CamliType(s)
}
return ""
}
@ -515,7 +515,7 @@ func (bb *Builder) SetFileName(name string) *Builder {
// SetSymlinkTarget sets bb to be of type "symlink" and sets the symlink's target.
func (bb *Builder) SetSymlinkTarget(target string) *Builder {
bb.SetType("symlink")
bb.SetType(TypeSymlink)
if utf8.ValidString(target) {
bb.m["symlinkTarget"] = target
} else {
@ -528,7 +528,7 @@ func (bb *Builder) SetSymlinkTarget(target string) *Builder {
// which should be signed. (a "claim" or "permanode")
func (bb *Builder) IsClaimType() bool {
switch bb.Type() {
case "claim", "permanode":
case TypeClaim, TypePermanode:
return true
}
return false
@ -583,7 +583,7 @@ func (bb *Builder) ModTime() (t time.Time, ok bool) {
// PopulateDirectoryMap sets the type of *Builder to "directory" and sets
// the "entries" field to the provided staticSet blobref.
func (bb *Builder) PopulateDirectoryMap(staticSetRef blob.Ref) *Builder {
bb.m["camliType"] = "directory"
bb.m["camliType"] = string(TypeDirectory)
bb.m["entries"] = staticSetRef.String()
return bb
}

2
pkg/schema/fileread_test.go
View File

@ -144,7 +144,7 @@ func skipBytes(fr *FileReader, skipBytes uint64) uint64 {
func TestReader(t *testing.T) {
for idx, rt := range readTests {
ss := new(superset)
ss.Type = "file"
ss.Type = TypeFile
ss.Version = 1
ss.Parts = rt.parts
fr, err := ss.NewFileReader(testFetcher)

2
pkg/schema/filewriter.go
View File

@ -173,7 +173,7 @@ func uploadBytes(ctx context.Context, bs blobserver.StatReceiver, bb *Builder, s
// the "file" schema before any of its parts arrive, then the indexer
// can get confused. So wait on the parts before, and then upload
// the "file" blob afterwards.
if bb.Type() == "file" {
if bb.Type() == TypeFile {
future.errc <- nil
_, err := future.Get() // may not be nil, if children parts failed
future = newUploadBytesFuture()

68
pkg/schema/schema.go
View File

@ -126,7 +126,7 @@ type DirectoryEntry interface {
// CamliType returns the schema blob's "camliType" field.
// This may be "file", "directory", "symlink", or other more
// obscure types added in the future.
CamliType() string
CamliType() CamliType
FileName() string
BlobRef() blob.Ref
@ -150,7 +150,7 @@ type dirEntry struct {
// This type breaks an otherwise-circular dependency.
type SearchQuery interface{}
func (de *dirEntry) CamliType() string {
func (de *dirEntry) CamliType() CamliType {
return de.ss.Type
}
@ -164,8 +164,8 @@ func (de *dirEntry) BlobRef() blob.Ref {
func (de *dirEntry) File(ctx context.Context) (File, error) {
if de.fr == nil {
if de.ss.Type != "file" {
return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, "file")
if de.ss.Type != TypeFile {
return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, TypeFile)
}
fr, err := NewFileReader(ctx, de.fetcher, de.ss.BlobRef)
if err != nil {
@ -178,8 +178,8 @@ func (de *dirEntry) File(ctx context.Context) (File, error) {
func (de *dirEntry) Directory(ctx context.Context) (Directory, error) {
if de.dr == nil {
if de.ss.Type != "directory" {
return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, "directory")
if de.ss.Type != TypeDirectory {
return nil, fmt.Errorf("DirectoryEntry is camliType %q, not %q", de.ss.Type, TypeDirectory)
}
dr, err := NewDirReader(ctx, de.fetcher, de.ss.BlobRef)
if err != nil {
@ -214,7 +214,7 @@ func newDirectoryEntry(fetcher blob.Fetcher, ss *superset) (DirectoryEntry, erro
return nil, errors.New("ss.BlobRef was invalid")
}
switch ss.Type {
case "file", "directory", "symlink", "fifo", "socket":
case TypeFile, TypeDirectory, TypeSymlink, TypeFIFO, TypeSocket:
// Okay
default:
return nil, fmt.Errorf("invalid DirectoryEntry camliType of %q", ss.Type)
@ -244,8 +244,8 @@ type superset struct {
// for convenience.
BlobRef blob.Ref
Version int `json:"camliVersion"`
Type string `json:"camliType"`
Version int `json:"camliVersion"`
Type CamliType `json:"camliType"`
Signer blob.Ref `json:"camliSigner"`
Sig string `json:"camliSig"`
@ -484,20 +484,20 @@ func (ss *superset) FileMode() os.FileMode {
// TODO: add other types (block, char, etc)
switch ss.Type {
case "directory":
case TypeDirectory:
mode = mode | os.ModeDir
case "file":
case TypeFile:
// No extra bit.
case "symlink":
case TypeSymlink:
mode = mode | os.ModeSymlink
case "fifo":
case TypeFIFO:
mode = mode | os.ModeNamedPipe
case "socket":
case TypeSocket:
mode = mode | os.ModeSocket
}
if !hasPerm {
switch ss.Type {
case "directory":
case TypeDirectory:
mode |= 0755
default:
mode |= 0644
@ -579,7 +579,7 @@ var maxStaticSetMembers = 10000
// NewStaticSet returns the "static-set" schema for a directory. Its members
// should be populated with SetStaticSetMembers.
func NewStaticSet() *Builder {
return base(1, "static-set")
return base(1, TypeStaticSet)
}
// SetStaticSetMembers sets the given members as the static-set members of this
@ -590,7 +590,7 @@ func NewStaticSet() *Builder {
// static-set created from this builder.
// SetStaticSetMembers panics if bb isn't a "static-set" claim type.
func (bb *Builder) SetStaticSetMembers(members []blob.Ref) []*Blob {
if bb.Type() != "static-set" {
if bb.Type() != TypeStaticSet {
panic("called SetStaticSetMembers on non static-set")
}
@ -649,16 +649,16 @@ func (bb *Builder) SetStaticSetMembers(members []blob.Ref) []*Blob {
return allSubsets
}
func base(version int, ctype string) *Builder {
func base(version int, ctype CamliType) *Builder {
return &Builder{map[string]interface{}{
"camliVersion": version,
"camliType": ctype,
"camliType": string(ctype),
}}
}
// NewUnsignedPermanode returns a new random permanode, not yet signed.
func NewUnsignedPermanode() *Builder {
bb := base(1, "permanode")
bb := base(1, TypePermanode)
chars := make([]byte, 20)
_, err := io.ReadFull(rand.Reader, chars)
if err != nil {
@ -674,7 +674,7 @@ func NewUnsignedPermanode() *Builder {
// GPG date to create consistent JSON encodings of the Map (its
// blobref), between runs.
func NewPlannedPermanode(key string) *Builder {
bb := base(1, "permanode")
bb := base(1, TypePermanode)
bb.m["key"] = key
return bb
}
@ -711,12 +711,12 @@ func mapJSON(m map[string]interface{}) (string, error) {
// NewFileMap returns a new builder of a type "file" schema for the provided fileName.
// The chunk parts of the file are not populated.
func NewFileMap(fileName string) *Builder {
return newCommonFilenameMap(fileName).SetType("file")
return newCommonFilenameMap(fileName).SetType(TypeFile)
}
// NewDirMap returns a new builder of a type "directory" schema for the provided fileName.
func NewDirMap(fileName string) *Builder {
return newCommonFilenameMap(fileName).SetType("directory")
return newCommonFilenameMap(fileName).SetType(TypeDirectory)
}
func newCommonFilenameMap(fileName string) *Builder {
@ -785,9 +785,27 @@ func populateParts(m map[string]interface{}, size int64, parts []BytesPart) erro
}
func newBytes() *Builder {
return base(1, "bytes")
return base(1, TypeBytes)
}
// CamliType is one of the valid "camliType" fields in a schema blob. See doc/schema.
type CamliType string
const (
TypeBytes CamliType = "bytes"
TypeClaim CamliType = "claim"
TypeDirectory CamliType = "directory"
TypeFIFO CamliType = "fifo"
TypeFile CamliType = "file"
TypeInode CamliType = "inode"
TypeKeep CamliType = "keep"
TypePermanode CamliType = "permanode"
TypeShare CamliType = "share"
TypeSocket CamliType = "socket"
TypeStaticSet CamliType = "static-set"
TypeSymlink CamliType = "symlink"
)
// ClaimType is one of the valid "claimType" fields in a "claim" schema blob. See doc/schema/claims/.
type ClaimType string
@ -819,7 +837,7 @@ type claimParam struct {
}
func newClaim(claims ...*claimParam) *Builder {
bb := base(1, "claim")
bb := base(1, TypeClaim)
bb.SetClaimDate(clockNow())
if len(claims) == 1 {
cp := claims[0]

8
pkg/schema/schema_test.go
View File

@ -553,7 +553,7 @@ func TestStaticFileAndStaticSymlink(t *testing.T) {
}
bb := NewCommonFileMap(fd.Name(), fi)
bb.SetType("file")
bb.SetType(TypeFile)
bb.SetFileName(fd.Name())
blob := bb.Blob()
@ -592,7 +592,7 @@ func TestStaticFileAndStaticSymlink(t *testing.T) {
}
bb = NewCommonFileMap(src, fi)
bb.SetType("symlink")
bb.SetType(TypeSymlink)
bb.SetFileName(src)
bb.SetSymlinkTarget(target)
blob = bb.Blob()
@ -638,7 +638,7 @@ func TestStaticFIFO(t *testing.T) {
}
bb := NewCommonFileMap(fifoPath, fi)
bb.SetType("fifo")
bb.SetType(TypeFIFO)
bb.SetFileName(fifoPath)
blob := bb.Blob()
t.Logf("Got JSON for fifo: %s\n", blob.JSON())
@ -675,7 +675,7 @@ func TestStaticSocket(t *testing.T) {
}
bb := NewCommonFileMap(sockPath, fi)
bb.SetType("socket")
bb.SetType(TypeSocket)
bb.SetFileName(sockPath)
blob := bb.Blob()
t.Logf("Got JSON for socket: %s\n", blob.JSON())

11
pkg/search/describe.go
View File

@ -34,6 +34,7 @@ import (
"go4.org/types"
"perkeep.org/internal/httputil"
"perkeep.org/pkg/blob"
"perkeep.org/pkg/schema"
"perkeep.org/pkg/types/camtypes"
)
@ -223,9 +224,9 @@ type MetaMap map[string]*DescribedBlob
type DescribedBlob struct {
Request *DescribeRequest `json:"-"`
BlobRef blob.Ref `json:"blobRef"`
CamliType string `json:"camliType,omitempty"`
Size int64 `json:"size,"`
BlobRef blob.Ref `json:"blobRef"`
CamliType schema.CamliType `json:"camliType,omitempty"`
Size int64 `json:"size,"`
// if camliType "permanode"
Permanode *DescribedPermanode `json:"permanode,omitempty"`
@ -739,7 +740,7 @@ func (dr *DescribeRequest) doDescribe(ctx context.Context, br blob.Ref, depth in
// maps. Then add JSON marhsallers to those types. Add tests.
des := dr.describedBlob(br)
if meta.CamliType != "" {
des.setMIMEType("application/json; camliType=" + meta.CamliType)
des.setMIMEType("application/json; camliType=" + string(meta.CamliType))
}
des.Size = int64(meta.Size)
@ -907,6 +908,6 @@ func (dr *DescribeRequest) describeRefs(ctx context.Context, str string, depth i
func (b *DescribedBlob) setMIMEType(mime string) {
if strings.HasPrefix(mime, camliTypePrefix) {
b.CamliType = strings.TrimPrefix(mime, camliTypePrefix)
b.CamliType = schema.CamliType(strings.TrimPrefix(mime, camliTypePrefix))
}
}

9
pkg/search/handler.go
View File

@ -39,6 +39,7 @@ import (
"perkeep.org/pkg/blobserver"
"perkeep.org/pkg/index"
"perkeep.org/pkg/jsonsign"
"perkeep.org/pkg/schema"
"perkeep.org/pkg/types/camtypes"
"perkeep.org/pkg/types/serverconfig"
)
@ -504,8 +505,8 @@ type EdgesResponse struct {
// An EdgeItem is an item returned from $searchRoot/camli/search/edgesto.
type EdgeItem struct {
From blob.Ref `json:"from"`
FromType string `json:"fromType"`
From blob.Ref `json:"from"`
FromType schema.CamliType `json:"fromType"`
}
var testHookBug121 = func() {}
@ -808,14 +809,14 @@ func (h *Handler) EdgesTo(req *EdgesRequest) (*EdgesResponse, error) {
if found {
ei = &EdgeItem{
From: edge.From,
FromType: "permanode",
FromType: schema.TypePermanode,
}
}
resc <- edgeOrError{edge: ei}
}
verifying := 0
for _, edge := range edges {
if edge.FromType == "permanode" {
if edge.FromType == schema.TypePermanode {
verifying++
go verify(edge)
continue

15
pkg/search/query.go
View File

@ -35,6 +35,7 @@ import (
"perkeep.org/pkg/blob"
"perkeep.org/pkg/index"
"perkeep.org/pkg/schema"
"perkeep.org/pkg/types/camtypes"
"context"
@ -294,9 +295,9 @@ type Constraint struct {
// Anything, if true, matches all blobs.
Anything bool `json:"anything,omitempty"`
CamliType string `json:"camliType,omitempty"` // camliType of the JSON blob
AnyCamliType bool `json:"anyCamliType,omitempty"` // if true, any camli JSON blob matches
BlobRefPrefix string `json:"blobRefPrefix,omitempty"`
CamliType schema.CamliType `json:"camliType,omitempty"` // camliType of the JSON blob
AnyCamliType bool `json:"anyCamliType,omitempty"` // if true, any camli JSON blob matches
BlobRefPrefix string `json:"blobRefPrefix,omitempty"`
File *FileConstraint `json:"file,omitempty"`
Dir *DirConstraint `json:"dir,omitempty"`
@ -383,7 +384,7 @@ func (c *Constraint) matchesAtMostOneBlob() blob.Ref {
}
func (c *Constraint) onlyMatchesPermanode() bool {
if c.Permanode != nil || c.CamliType == "permanode" {
if c.Permanode != nil || c.CamliType == schema.TypePermanode {
return true
}
@ -1455,7 +1456,7 @@ func (q *SearchQuery) pickCandidateSource(s *search) (src candidateSource) {
if c.matchesFileByWholeRef() {
src.name = "corpus_file_meta"
src.send = func(ctx context.Context, s *search, fn func(camtypes.BlobMeta) bool) error {
corpus.EnumerateCamliBlobs("file", fn)
corpus.EnumerateCamliBlobs(schema.TypeFile, fn)
return nil
}
return
@ -1679,7 +1680,7 @@ func (c *PermanodeConstraint) hasValueConstraint() bool {
}
func (c *PermanodeConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (ok bool, err error) {
if bm.CamliType != "permanode" {
if bm.CamliType != schema.TypePermanode {
return false, nil
}
corpus := s.h.corpus
@ -2085,7 +2086,7 @@ func (c *Constraint) fileOrDirOrLogicalMatches(ctx context.Context, s *search, b
}
func (c *DirConstraint) blobMatches(ctx context.Context, s *search, br blob.Ref, bm camtypes.BlobMeta) (bool, error) {
if bm.CamliType != "directory" {
if bm.CamliType != schema.TypeDirectory {
return false, nil
}
// TODO(mpl): I've added c.BlobRefPrefix, so that c.ParentDir can be directly

5
pkg/search/websocket.go
View File

@ -28,6 +28,7 @@ import (
"time"
"github.com/gorilla/websocket"
"perkeep.org/pkg/schema"
)
const (
@ -51,7 +52,7 @@ type wsHub struct {
register chan *wsConn
unregister chan *wsConn
watchReq chan watchReq
newBlobRecv chan string // new blob received. string is camliType.
newBlobRecv chan schema.CamliType // new blob received.
updatedResults chan *watchedQuery
statusUpdate chan json.RawMessage
@ -66,7 +67,7 @@ func newWebsocketHub(sh *Handler) *wsHub {
unregister: make(chan *wsConn), // unbuffered; issue 563
conns: make(map[*wsConn]bool),
watchReq: make(chan watchReq, buffered),
newBlobRecv: make(chan string, buffered),
newBlobRecv: make(chan schema.CamliType, buffered),
updatedResults: make(chan *watchedQuery, buffered),
statusUpdate: make(chan json.RawMessage, buffered),
}

17
pkg/server/download.go
View File

@ -141,10 +141,10 @@ func (dh *DownloadHandler) fileInfo(ctx context.Context, file blob.Ref) (fi file
return fi, false, fmt.Errorf("could not read %v as blob: %v", file, err)
}
tp := b.Type()
if tp != "file" {
if tp != schema.TypeFile {
// for non-regular files
var contents string
if tp == "symlink" {
if tp == schema.TypeSymlink {
sf, _ := b.AsStaticFile()
sl, _ := sf.AsStaticSymlink()
contents = sl.SymlinkTargetString()
@ -416,7 +416,12 @@ func (dh *DownloadHandler) statFiles(refs []blob.Ref) error {
return nil
}
var allowedFileTypes = map[string]bool{"file": true, "symlink": true, "fifo": true, "socket": true}
var allowedFileTypes = map[schema.CamliType]bool{
schema.TypeFile: true,
schema.TypeSymlink: true,
schema.TypeFIFO: true,
schema.TypeSocket: true,
}
// checkFiles reads, and discards, the file contents for each of the given file refs.
// It is used to check that all files requested for download are readable before
@ -435,10 +440,10 @@ func (dh *DownloadHandler) checkFiles(ctx context.Context, parentPath string, fi
return fmt.Errorf("could not read %v as blob: %v", br, err)
}
tp := b.Type()
if _, ok := allowedFileTypes[tp]; !ok && tp != "directory" {
if _, ok := allowedFileTypes[tp]; !ok && tp != schema.TypeDirectory {
return fmt.Errorf("%v not a supported file or directory type: %q", br, tp)
}
if tp == "directory" {
if tp == schema.TypeDirectory {
dr, err := b.NewDirReader(ctx, dh.Fetcher)
if err != nil {
return fmt.Errorf("could not open %v as directory: %v", br, err)
@ -452,7 +457,7 @@ func (dh *DownloadHandler) checkFiles(ctx context.Context, parentPath string, fi
}
continue
}
if tp != "file" {
if tp != schema.TypeFile {
// We only bother checking regular files. symlinks, fifos, and sockets are
// assumed ok.
dh.pathByRef[br] = filepath.Join(parentPath, b.FileName())

2
pkg/server/filetree.go
View File

@ -37,7 +37,7 @@ type FileTreeNode struct {
Name string `json:"name"`
// Type is the camliType of the node. This may be "file", "directory", "symlink"
// or other in the future.
Type string `json:"type"`
Type schema.CamliType `json:"type"`
// BlobRef is the blob.Ref of the node.
BlobRef blob.Ref `json:"blobRef"`
}

2
pkg/server/import_share.go
View File

@ -167,7 +167,7 @@ func (si *shareImporter) imprt(ctx context.Context, br blob.Ref) error {
return nil
// TODO(mpl): other camliTypes, at least symlink.
default:
return errors.New("unknown blob type: " + b.Type())
return errors.New("unknown blob type: " + string(b.Type()))
}
}

6
pkg/server/share.go
View File

@ -324,7 +324,7 @@ func bytesHaveSchemaLink(br blob.Ref, bb []byte, target blob.Ref) bool {
}
typ := b.Type()
switch typ {
case "file", "bytes":
case schema.TypeFile, schema.TypeBytes:
for _, bp := range b.ByteParts() {
if bp.BlobRef.Valid() {
if bp.BlobRef == target {
@ -337,11 +337,11 @@ func bytesHaveSchemaLink(br blob.Ref, bb []byte, target blob.Ref) bool {
}
}
}
case "directory":
case schema.TypeDirectory:
if d, ok := b.DirectoryEntries(); ok {
return d == target
}
case "static-set":
case schema.TypeStaticSet:
for _, m := range b.StaticSetMembers() {
if m == target {
return true

7
pkg/types/camtypes/search.go
View File

@ -27,6 +27,7 @@ import (
"perkeep.org/internal/magic"
"perkeep.org/pkg/blob"
"perkeep.org/pkg/schema"
"go4.org/types"
)
@ -196,8 +197,8 @@ type EdgesToOpts struct {
type Edge struct {
From blob.Ref
FromType string // "permanode", "directory", etc
FromTitle string // name of source permanode or directory
FromType schema.CamliType // "permanode", "directory", etc
FromTitle string // name of source permanode or directory
To blob.Ref
BlobRef blob.Ref // the blob responsible for the edge relationship
}
@ -214,7 +215,7 @@ type BlobMeta struct {
// CamliType is non-empty if this blob is a Perkeep JSON
// schema blob. If so, this is its "camliType" attribute.
CamliType string
CamliType schema.CamliType
// TODO(bradfitz): change CamliTypethis *string to save 8 bytes
}