From 957da4337c8ce03de4954b63001aa254af9e9f16 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 22 Mar 2011 18:29:40 -0700 Subject: [PATCH 1/5] Ignore emacs spew when searching for .go files. --- build.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.pl b/build.pl index 74877b37f..ff2b369f1 100755 --- a/build.pl +++ b/build.pl @@ -280,7 +280,7 @@ sub find_go_camli_deps { my $t = $targets{$target} or die "Bogus or undeclared build target: $target\n"; opendir(my $dh, $target) or die; - my @go_files = grep { !/_testmain\.go$/ } grep { /\.go$/ } readdir($dh); + my @go_files = grep { !m!^\.\#! } grep { !/_testmain\.go$/ } grep { /\.go$/ } readdir($dh); closedir($dh); # TODO: just stat the files first and keep a cache file of the @@ -292,7 +292,7 @@ sub find_go_camli_deps { my @deps; my %seen; # $dep -> 1 for my $f (@go_files) { - open(my $fh, "$target/$f") or die; + open(my $fh, "$target/$f") or die "Failed to open $target/$f: $!"; my $src = do { local $/; <$fh>; }; unless ($src =~ m!\bimport\s*\((.+?)\)!s) { die "Failed to parse imports from $target/$f.\n". @@ -327,7 +327,7 @@ sub gen_target_makefile { my @deps = @{$t->{deps}}; opendir(my $dh, $target) or die; - my @go_files = grep { !/_testmain\.go$/ } grep { /\.go$/ } readdir($dh); + my @go_files = grep { !m!^\.\#! } grep { !/_testmain\.go$/ } grep { /\.go$/ } readdir($dh); closedir($dh); open(my $mf, ">$target/Makefile") or die; From 36e3c016c2ab22f47cfd0cd053bb9f669b4df1fc Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 22 Mar 2011 20:08:53 -0700 Subject: [PATCH 2/5] cammount fuse shell --- clients/go/cammount/fs.go | 127 ++++++++++++++++++++++++++++++++++++ clients/go/cammount/main.go | 24 ++++--- 2 files changed, 138 insertions(+), 13 deletions(-) create mode 100644 clients/go/cammount/fs.go diff --git a/clients/go/cammount/fs.go b/clients/go/cammount/fs.go new file mode 100644 index 000000000..7c8050358 --- /dev/null +++ b/clients/go/cammount/fs.go @@ -0,0 +1,127 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "log" + "os" + + "camli/blobref" + "camli/client" + "camli/third_party/github.com/hanwen/go-fuse/fuse" +) + +var _ = fmt.Println +var _ = log.Println + +type CamliFileSystem struct { + client *client.Client + root *blobref.BlobRef +} + +func NewCamliFileSystem(client *client.Client, root *blobref.BlobRef) *CamliFileSystem { + return &CamliFileSystem{client: client, root: root} +} + +func (fs *CamliFileSystem) Mount(connector *fuse.PathFileSystemConnector) fuse.Status { + log.Printf("cammount: Mount") + return fuse.OK +} + +func (fs *CamliFileSystem) Unmount() { + log.Printf("cammount: Unmount.") +} + +func (fs *CamliFileSystem) GetAttr(name string) (*fuse.Attr, fuse.Status) { + out := new(fuse.Attr) + var fi os.FileInfo + // TODO + fuse.CopyFileInfo(&fi, out) + return out, fuse.OK +} + +func (fs *CamliFileSystem) Access(name string, mode uint32) fuse.Status { + return fuse.OK +} + +func (fs *CamliFileSystem) Open(name string, flags uint32) (file fuse.RawFuseFile, code fuse.Status) { + // TODO + return nil, fuse.EACCES +} + +func (fs *CamliFileSystem) OpenDir(name string) (stream chan fuse.DirEntry, code fuse.Status) { + // TODO + return nil, fuse.EACCES +} + +func (fs *CamliFileSystem) Readlink(name string) (string, fuse.Status) { + // TODO + return "", fuse.EACCES +} + +// ************************************************************************* +// EACCESS stuff + +func (fs *CamliFileSystem) Chmod(name string, mode uint32) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Chown(name string, uid uint32, gid uint32) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Create(name string, flags uint32, mode uint32) (file fuse.RawFuseFile, code fuse.Status) { + code = fuse.EACCES + return +} + +func (fs *CamliFileSystem) Link(oldName string, newName string) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Mkdir(name string, mode uint32) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Mknod(name string, mode uint32, dev uint32) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Rename(oldName string, newName string) (code fuse.Status) { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Rmdir(name string) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Symlink(value string, linkName string) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Truncate(name string, offset uint64) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Unlink(name string) fuse.Status { + return fuse.EACCES +} + +func (fs *CamliFileSystem) Utimens(name string, AtimeNs uint64, CtimeNs uint64) fuse.Status { + return fuse.EACCES +} diff --git a/clients/go/cammount/main.go b/clients/go/cammount/main.go index 3ee135956..97f73f820 100644 --- a/clients/go/cammount/main.go +++ b/clients/go/cammount/main.go @@ -22,6 +22,8 @@ import ( "os" "sort" + "camli/blobref" + "camli/client" "camli/third_party/github.com/hanwen/go-fuse/fuse" ) @@ -45,23 +47,19 @@ func main() { threaded := flag.Bool("threaded", true, "switch off threading; print debugging messages.") flag.Parse() if flag.NArg() < 2 { - // TODO - where to get program name? - fmt.Println("usage: main ORIGINAL MOUNTPOINT") + fmt.Println("usage: cammount ") os.Exit(2) } - orig := flag.Arg(0) - fs := fuse.NewLoopbackFileSystem(orig) + root := blobref.Parse(flag.Arg(0)) + if root == nil { + fmt.Printf("Error parsing root blobref: %q\n", root) + os.Exit(2) + } + client := client.NewOrFail() // automatic from flags + fs := NewCamliFileSystem(client, root) timing := fuse.NewTimingPathFilesystem(fs) - var opts fuse.PathFileSystemConnectorOptions - - opts.AttrTimeout = 1.0 - opts.EntryTimeout = 1.0 - opts.NegativeTimeout = 1.0 - - fs.SetOptions(&opts) - conn := fuse.NewPathFileSystemConnector(timing) rawTiming := fuse.NewTimingRawFilesystem(conn) @@ -75,7 +73,7 @@ func main() { os.Exit(1) } - fmt.Printf("Mounted %s on %s (threaded=%v, debug=%v)\n", orig, mountPoint, *threaded, *debug) + fmt.Printf("Mounted %s on %s (threaded=%v, debug=%v)\n", root.String(), mountPoint, *threaded, *debug) state.Loop(*threaded) fmt.Println("Finished", state.Stats()) From bd40289c8821fdbfb68c3bb203b859213d2325f2 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 22 Mar 2011 20:35:04 -0700 Subject: [PATCH 3/5] cammount work commit before battery dies --- clients/go/cammount/fs.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/clients/go/cammount/fs.go b/clients/go/cammount/fs.go index 7c8050358..0bccade92 100644 --- a/clients/go/cammount/fs.go +++ b/clients/go/cammount/fs.go @@ -20,6 +20,7 @@ import ( "fmt" "log" "os" + "path/filepath" "camli/blobref" "camli/client" @@ -38,6 +39,17 @@ func NewCamliFileSystem(client *client.Client, root *blobref.BlobRef) *CamliFile return &CamliFileSystem{client: client, root: root} } +// Where name == "" for root, +// Returns nil on failure +func (fs *CamliFileSystem) blobRefFromName(name string) *blobref.BlobRef { + parts := filepath.SplitList(name) + if name == "" || len(parts) == 0 { + return fs.root + } + // TODO: walk + return nil +} + func (fs *CamliFileSystem) Mount(connector *fuse.PathFileSystemConnector) fuse.Status { log.Printf("cammount: Mount") return fuse.OK @@ -48,6 +60,11 @@ func (fs *CamliFileSystem) Unmount() { } func (fs *CamliFileSystem) GetAttr(name string) (*fuse.Attr, fuse.Status) { + blobref := fs.blobRefFromName(name) + if blobref == nil { + return nil, fuse.ENOENT + } + log.Printf("cammount: GetAttr(%q)", name) out := new(fuse.Attr) var fi os.FileInfo // TODO @@ -56,20 +73,24 @@ func (fs *CamliFileSystem) GetAttr(name string) (*fuse.Attr, fuse.Status) { } func (fs *CamliFileSystem) Access(name string, mode uint32) fuse.Status { + log.Printf("cammount: Access(%q, %d)", name, mode) return fuse.OK } func (fs *CamliFileSystem) Open(name string, flags uint32) (file fuse.RawFuseFile, code fuse.Status) { + log.Printf("cammount: Open(%q, %d)", name, flags) // TODO return nil, fuse.EACCES } func (fs *CamliFileSystem) OpenDir(name string) (stream chan fuse.DirEntry, code fuse.Status) { + log.Printf("cammount: OpenDir(%q)", name) // TODO return nil, fuse.EACCES } func (fs *CamliFileSystem) Readlink(name string) (string, fuse.Status) { + log.Printf("cammount: Readlink(%q)", name) // TODO return "", fuse.EACCES } From 0707a092e047b3ddcc28c12779664e8404937637 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 22 Mar 2011 22:11:27 -0700 Subject: [PATCH 4/5] cammount: more FUSE work, inode lookup --- clients/go/cammount/fs.go | 137 +++++++++++++++++++++++++++++++--- lib/go/camli/schema/schema.go | 20 +++++ 2 files changed, 145 insertions(+), 12 deletions(-) diff --git a/clients/go/cammount/fs.go b/clients/go/cammount/fs.go index 0bccade92..11158ee68 100644 --- a/clients/go/cammount/fs.go +++ b/clients/go/cammount/fs.go @@ -19,11 +19,14 @@ package main import ( "fmt" "log" + "json" "os" "path/filepath" + "sync" "camli/blobref" "camli/client" + "camli/schema" "camli/third_party/github.com/hanwen/go-fuse/fuse" ) @@ -31,23 +34,133 @@ var _ = fmt.Println var _ = log.Println type CamliFileSystem struct { - client *client.Client + fetcher blobref.Fetcher root *blobref.BlobRef + + lk sync.Mutex + nameToBlob map[string]*blobref.BlobRef } func NewCamliFileSystem(client *client.Client, root *blobref.BlobRef) *CamliFileSystem { - return &CamliFileSystem{client: client, root: root} + return &CamliFileSystem{ + fetcher: client, + root: root, + nameToBlob: make(map[string]*blobref.BlobRef), + } } // Where name == "" for root, // Returns nil on failure -func (fs *CamliFileSystem) blobRefFromName(name string) *blobref.BlobRef { - parts := filepath.SplitList(name) - if name == "" || len(parts) == 0 { - return fs.root +func (fs *CamliFileSystem) blobRefFromNameCached(name string) *blobref.BlobRef { + fs.lk.Lock() + defer fs.lk.Unlock() + return fs.nameToBlob[name] +} + +func (fs *CamliFileSystem) fetchSchemaSuperset(br *blobref.BlobRef) (*schema.Superset, os.Error) { + rsc, _, err := fs.fetcher.Fetch(br) + if err != nil { + return nil, err } - // TODO: walk - return nil + defer rsc.Close() + jd := json.NewDecoder(rsc) + ss := new(schema.Superset) + err = jd.Decode(ss) + if err != nil { + log.Printf("Error parsing %s as schema blob: %v", br, err) + return nil, os.EINVAL + } + return ss, nil +} + +// Where name == "" for root, +// Returns fuse.Status == fuse.OK on success or anything else on failure. +func (fs *CamliFileSystem) blobRefFromName(name string) (*blobref.BlobRef, fuse.Status) { + if name == "" { + return fs.root, fuse.OK + } + if br := fs.blobRefFromNameCached(name); br != nil { + return br, fuse.OK + } + dir, fileName := filepath.Split(name) + dirBlob, fuseStatus := fs.blobRefFromName(dir) + if fuseStatus != fuse.OK { + return nil, fuseStatus + } + + dirss, err := fs.fetchSchemaSuperset(dirBlob) + switch { + case err == os.ENOENT: + log.Printf("Failed to find directory %s", dirBlob) + return nil, fuse.ENOENT + case err == os.EINVAL: + log.Printf("Failed to parse directory %s", dirBlob) + return nil, fuse.ENOTDIR + case err != nil: + panic(fmt.Sprintf("Invalid fetcher error: %v", err)) + case dirss == nil: + panic("nil dirss") + case dirss.Type != "directory": + log.Printf("Expected %s to be a directory; actually a %s", + dirBlob, dirss.Type) + return nil, fuse.ENOTDIR + } + + if dirss.Entries == "" { + log.Printf("Expected %s to have 'entries'", dirBlob) + return nil, fuse.ENOTDIR + } + entriesBlob := blobref.Parse(dirss.Entries) + if entriesBlob == nil { + log.Printf("Blob %s had invalid blobref %q for its 'entries'", dirBlob, dirss.Entries) + return nil, fuse.ENOTDIR + } + + entss, err := fs.fetchSchemaSuperset(entriesBlob) + switch { + case err == os.ENOENT: + log.Printf("Failed to find entries %s via directory %s", entriesBlob, dirBlob) + return nil, fuse.ENOENT + case err == os.EINVAL: + log.Printf("Failed to parse entries %s via directory %s", entriesBlob, dirBlob) + return nil, fuse.ENOTDIR + case err != nil: + panic(fmt.Sprintf("Invalid fetcher error: %v", err)) + case entss == nil: + panic("nil entss") + case entss.Type != "static-set": + log.Printf("Expected %s to be a directory; actually a %s", + dirBlob, dirss.Type) + return nil, fuse.ENOTDIR + } + + wg := new(sync.WaitGroup) + foundCh := make(chan *blobref.BlobRef) + for _, m := range entss.Members { + wg.Add(1) + go func(memberBlobstr string) { + childss, err := fs.fetchSchemaSuperset(entriesBlob) + if err == nil && childss.HasFilename(fileName) { + foundCh <- entriesBlob + } + wg.Done() + }(m) + } + failCh := make(chan string) + go func() { + wg.Wait() + failCh <- "ENOENT" + }() + select { + case found := <-foundCh: + fs.lk.Lock() + defer fs.lk.Unlock() + fs.nameToBlob[name] = found + return found, fuse.OK + case <-failCh: + } + // TODO: negative cache + return nil, fuse.ENOENT } func (fs *CamliFileSystem) Mount(connector *fuse.PathFileSystemConnector) fuse.Status { @@ -60,11 +173,11 @@ func (fs *CamliFileSystem) Unmount() { } func (fs *CamliFileSystem) GetAttr(name string) (*fuse.Attr, fuse.Status) { - blobref := fs.blobRefFromName(name) - if blobref == nil { - return nil, fuse.ENOENT + blobref, errStatus := fs.blobRefFromName(name) + log.Printf("cammount: GetAttr(%q) (%s, %v)", name, blobref, errStatus) + if errStatus != fuse.OK { + return nil, errStatus } - log.Printf("cammount: GetAttr(%q)", name) out := new(fuse.Attr) var fi os.FileInfo // TODO diff --git a/lib/go/camli/schema/schema.go b/lib/go/camli/schema/schema.go index 1603bcf82..f4c785e47 100644 --- a/lib/go/camli/schema/schema.go +++ b/lib/go/camli/schema/schema.go @@ -55,6 +55,26 @@ type Superset struct { Permanode string "permaNode" Attribute string "attribute" Value string "value" + + FileName string "fileName" + FileNameBytes []byte "fileNameBytes" // TODO: needs custom UnmarshalJSON? + UnixPermission string "unixPermission" + UnixOwnerId int "unixOwnerId" + UnixOwner string "unixOwner" + UnixGroupId int "unixGroupId" + UnixGroup string "unixGroup" + UnixMtime string "unixMtime" + UnixCtime string "unixCtime" + UnixAtime string "unixAtime" + + Entries string "entries" // for directories, a blobref to a static-set + Members []string "members" // for static sets (for directory static-sets: + // blobrefs to child dirs/files) +} + +func (ss *Superset) HasFilename(name string) bool { + // TODO: use filename bytes too + return ss.FileName == name } var DefaultStatHasher = &defaultStatHasher{} From 9771c314e316b738570b4ee9fb4264b8c59b5fa7 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 22 Mar 2011 22:32:11 -0700 Subject: [PATCH 5/5] cammount: more FUSE work: basic GetAttr support --- clients/go/cammount/fs.go | 43 +++++++++++++++++++++++++++++++++-- lib/go/camli/schema/schema.go | 2 ++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/clients/go/cammount/fs.go b/clients/go/cammount/fs.go index 11158ee68..ed05d6f33 100644 --- a/clients/go/cammount/fs.go +++ b/clients/go/cammount/fs.go @@ -22,7 +22,9 @@ import ( "json" "os" "path/filepath" + "strconv" "sync" + "syscall" "camli/blobref" "camli/client" @@ -173,14 +175,51 @@ func (fs *CamliFileSystem) Unmount() { } func (fs *CamliFileSystem) GetAttr(name string) (*fuse.Attr, fuse.Status) { + log.Printf("cammount: GetAttr(%q)", name) blobref, errStatus := fs.blobRefFromName(name) - log.Printf("cammount: GetAttr(%q) (%s, %v)", name, blobref, errStatus) + log.Printf("cammount: GetAttr(%q), blobRefFromName err=%v", name, errStatus) if errStatus != fuse.OK { return nil, errStatus } + log.Printf("cammount: got blob %s", blobref) + + // TODO: this is redundant with what blobRefFromName already + // did. we should at least keep this in RAM (pre-de-JSON'd) + // so we don't have to fetch + unmarshal it again. + ss, err := fs.fetchSchemaSuperset(blobref) + if err != nil { + log.Printf("cammount: GetAttr(%q, %s): fetch schema error: %v", name, blobref, err) + return nil, fuse.EIO + } + out := new(fuse.Attr) var fi os.FileInfo - // TODO + + if ss.UnixPermission != "" { + // Convert from octal + mode, err := strconv.Btoui64(ss.UnixPermission, 8) + if err != nil { + fi.Mode = uint32(mode) + } + } + + // TODO: have a mode to set permissions equal to mounting user? + fi.Uid = ss.UnixOwnerId + fi.Gid = ss.UnixGroupId + + // TODO: other types + switch ss.Type { + case "directory": + fi.Mode = fi.Mode | syscall.S_IFDIR + case "file": + fi.Mode = fi.Mode | syscall.S_IFREG + fi.Size = ss.Size + case "symlink": + fi.Mode = fi.Mode | syscall.S_IFLNK + } + + // TODO: mtime and such + fuse.CopyFileInfo(&fi, out) return out, fuse.OK } diff --git a/lib/go/camli/schema/schema.go b/lib/go/camli/schema/schema.go index f4c785e47..1475b0de4 100644 --- a/lib/go/camli/schema/schema.go +++ b/lib/go/camli/schema/schema.go @@ -67,6 +67,8 @@ type Superset struct { UnixCtime string "unixCtime" UnixAtime string "unixAtime" + Size int64 "size" // for files + Entries string "entries" // for directories, a blobref to a static-set Members []string "members" // for static sets (for directory static-sets: // blobrefs to child dirs/files)