diff --git a/TESTS b/TESTS index 5b2ab49a5..4e09bdd8f 100644 --- a/TESTS +++ b/TESTS @@ -3,4 +3,6 @@ Tests needed -cmd/camput/ -verify that stat caching works. verify that -filenodes does create the permanode even if the file was already uploaded (and cached) in a previous run. --- blobserver/cond has no tests. +-- blobserver/{cond,replica,remote,shard} have no tests. should be easier now + that test.Fetcher is a full blobserver? see encrypt's nascent tests for + examples. diff --git a/TODO b/TODO index f7c44e53c..b9f7fefa8 100644 --- a/TODO +++ b/TODO @@ -4,6 +4,10 @@ There are two TODO lists. This file (good for airplanes) and the online bug trac Offline list: +-- why can't ./pkg/blobserver/gethandler be merged into + ./pkg/blobserver/handlers? I forget the reason, but it seems to + work now, so maybe the circular dependency is gone. + -- support for running cammount under camlistored. especially for OS X, where the lifetime of the background daemon will be the same as the user's login session. diff --git a/pkg/blobserver/google/storage.go b/pkg/blobserver/google/storage.go index ef3d1e8c4..f3bc3938b 100644 --- a/pkg/blobserver/google/storage.go +++ b/pkg/blobserver/google/storage.go @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package google registers the "google" blob storage type, storing blobs +// on Google Cloud Storage (not Google Drive). package google import ( diff --git a/pkg/blobserver/localdisk/localdisk.go b/pkg/blobserver/localdisk/localdisk.go index 21ff1f0b1..cfa97f8a3 100644 --- a/pkg/blobserver/localdisk/localdisk.go +++ b/pkg/blobserver/localdisk/localdisk.go @@ -14,6 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* +Package localdisk registers the "filesystem" blobserver storage type, +storing blobs in a forest of sharded directories at the specified root. + +Example low-level config: + + "/storage/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/var/camlistore/blobs" + } + }, + +*/ package localdisk import ( @@ -27,6 +41,8 @@ import ( "camlistore.org/pkg/jsonconfig" ) +// DiskStorage implements the blobserver.Storage interface using the +// local filesystem. type DiskStorage struct { *blobserver.SimpleBlobHubPartitionMap root string @@ -39,8 +55,8 @@ type DiskStorage struct { mirrorPartitions []*DiskStorage } -// New returns a new local disk storage implementation, rooted at the provided -// directory, which must already exist. +// New returns a new local disk storage implementation at the provided +// root directory, which must already exist. func New(root string) (*DiskStorage, error) { // Local disk. fi, err := os.Stat(root) diff --git a/pkg/blobserver/localdisk/path.go b/pkg/blobserver/localdisk/path.go index f2db6b5ce..f31148b6e 100644 --- a/pkg/blobserver/localdisk/path.go +++ b/pkg/blobserver/localdisk/path.go @@ -25,7 +25,7 @@ import ( "net/url" ) -func BlobFileBaseName(b *blobref.BlobRef) string { +func blobFileBaseName(b *blobref.BlobRef) string { return fmt.Sprintf("%s-%s.dat", b.HashName(), b.Digest()) } @@ -38,7 +38,7 @@ func (ds *DiskStorage) blobDirectory(partition string, b *blobref.BlobRef) strin } func (ds *DiskStorage) blobPath(partition string, b *blobref.BlobRef) string { - return filepath.Join(ds.blobDirectory(partition, b), BlobFileBaseName(b)) + return filepath.Join(ds.blobDirectory(partition, b), blobFileBaseName(b)) } func (ds *DiskStorage) PartitionRoot(partition string) string { diff --git a/pkg/blobserver/localdisk/receive.go b/pkg/blobserver/localdisk/receive.go index 1733faf63..9de26910f 100644 --- a/pkg/blobserver/localdisk/receive.go +++ b/pkg/blobserver/localdisk/receive.go @@ -41,7 +41,7 @@ func (ds *DiskStorage) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) ( return } - tempFile, err := ioutil.TempFile(hashedDirectory, BlobFileBaseName(blobRef)+".tmp") + tempFile, err := ioutil.TempFile(hashedDirectory, blobFileBaseName(blobRef)+".tmp") if err != nil { return } diff --git a/pkg/blobserver/remote/remote.go b/pkg/blobserver/remote/remote.go index 3b2e824ad..95954eec0 100644 --- a/pkg/blobserver/remote/remote.go +++ b/pkg/blobserver/remote/remote.go @@ -14,6 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* +Package remote registers the "remote" blobserver storage type, storing +and fetching blobs from a remote Camlistore server, speaking the HTTP +protocol. + +Example low-level config: + + "/peer/": { + "handler": "storage-remote", + "handlerArgs": { + "url": "http://10.0.0.17/base", + "skipStartupCheck": false + } + }, + +*/ package remote import ( @@ -35,6 +51,8 @@ type remoteStorage struct { var _ = blobserver.Storage((*remoteStorage)(nil)) +// NewFromClient returns a new Storage implementation using the +// provided Camlistore client. func NewFromClient(c *client.Client) blobserver.Storage { return &remoteStorage{client: c} } diff --git a/pkg/blobserver/replica/replica.go b/pkg/blobserver/replica/replica.go index 78c83ecaa..23d0e89a0 100644 --- a/pkg/blobserver/replica/replica.go +++ b/pkg/blobserver/replica/replica.go @@ -14,6 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* +Package replica registers the "replica" blobserver storage type, +providing synchronous replication to one more backends. + +Writes wait for minWritesForSuccess (default: all). Reads are +attempted in order and not load-balanced, randomized, or raced by +default. + +Example config: + + "/repl/": { + "handler": "storage-replica", + "handlerArgs": { + "backends": ["/b1/", "/b2/", "/b3/"], + "minWritesForSuccess": 2 + } + }, +*/ package replica import ( @@ -96,13 +114,8 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobser return sto, nil } -func (sto *replicaStorage) weightedRandomReplicas() []blobserver.Storage { - // TODO: implement something actually weighted or random. - return sto.replicas -} - func (sto *replicaStorage) FetchStreaming(b *blobref.BlobRef) (file io.ReadCloser, size int64, err error) { - for _, replica := range sto.weightedRandomReplicas() { + for _, replica := range sto.replicas { file, size, err = replica.FetchStreaming(b) if err == nil { return @@ -167,8 +180,7 @@ type sizedBlobAndError struct { err error } -// TODO-GO: s/xxgo/_/ once Go issue 1802 is fixd -func (sto *replicaStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (xxgo blobref.SizedBlobRef, err error) { +func (sto *replicaStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (_ blobref.SizedBlobRef, err error) { nReplicas := len(sto.replicas) rpipe, wpipe, writer := make([]*io.PipeReader, nReplicas), make([]*io.PipeWriter, nReplicas), make([]io.Writer, nReplicas) for idx := range sto.replicas { diff --git a/pkg/blobserver/s3/s3.go b/pkg/blobserver/s3/s3.go index 9767dc5ee..92a7c4a75 100644 --- a/pkg/blobserver/s3/s3.go +++ b/pkg/blobserver/s3/s3.go @@ -14,6 +14,23 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* +Package s3 registers the "s3" blobserver storage type, storing +blobs in an Amazon Web Services' S3 storage bucket. + +Example low-level config: + + "/r1/": { + "handler": "storage-s3", + "handlerArgs": { + "bucket": "foo", + "aws_access_key": "...", + "aws_secret_access_key": "...", + "skipStartupCheck": false + } + }, + +*/ package s3 import ( diff --git a/pkg/blobserver/shard/shard.go b/pkg/blobserver/shard/shard.go index 007a83dd0..3792bd9f7 100644 --- a/pkg/blobserver/shard/shard.go +++ b/pkg/blobserver/shard/shard.go @@ -14,6 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* +Package shard registers the "shard" blobserver storage type, +predictably spraying out blobs out over the provided backends +based on their blobref. Each blob maps to exactly one backend. + +Example low-level config: + + "/foo/": { + "handler": "storage-shard", + "handlerArgs": { + "backends": ["/s1/", "/s2/"] + } + }, + +*/ package shard import ( diff --git a/pkg/buildinfo/buildinfo.go b/pkg/buildinfo/buildinfo.go index 4b8a6c6a3..23e758708 100644 --- a/pkg/buildinfo/buildinfo.go +++ b/pkg/buildinfo/buildinfo.go @@ -1,3 +1,4 @@ +// Package buildinfo provides information about the current build. package buildinfo // GitInfo is either the empty string (the default) @@ -6,6 +7,8 @@ package buildinfo // $ go install --ldflags="-X camlistore.org/pkg/buildinfo.GitInfo "`./misc/gitversion` camlistore.org/server/camlistored var GitInfo string +// Version returns the git version of this binary. +// If the linker flags were not provided, the return value is "unknown". func Version() string { if GitInfo != "" { return GitInfo diff --git a/pkg/client/client.go b/pkg/client/client.go index c5e525698..92f144ecd 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package client implements a Camlistore client. package client import ( diff --git a/pkg/cmdmain/cmdmain.go b/pkg/cmdmain/cmdmain.go index 12d2cbaa0..3afaedaf3 100644 --- a/pkg/cmdmain/cmdmain.go +++ b/pkg/cmdmain/cmdmain.go @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package cmdmain contains the shared implementation for camget, +// camput, camtool, and other Camlistore command-line tools. package cmdmain import ( diff --git a/pkg/fileembed/fileembed.go b/pkg/fileembed/fileembed.go index b199723c7..69094d46b 100644 --- a/pkg/fileembed/fileembed.go +++ b/pkg/fileembed/fileembed.go @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package fileembed provides access to static data resources (images, +// HTML, css, etc) embedded into the binary with genfileembed. +// +// Most of the package contains internal details used by genfileembed. +// Normal applications will simply make a global Files variable. package fileembed import ( @@ -32,6 +37,7 @@ import ( "time" ) +// Files contains all the embedded resources. type Files struct { // Optional environment variable key to override OverrideEnv string diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 5f5cc22c0..d21751cc8 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -16,6 +16,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package fs implements a FUSE filesystem for Camlistore and is +// used by the cammount binary. package fs import ( diff --git a/pkg/googlestorage/googlestorage.go b/pkg/googlestorage/googlestorage.go index 952301c3a..dda323a54 100644 --- a/pkg/googlestorage/googlestorage.go +++ b/pkg/googlestorage/googlestorage.go @@ -14,9 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Implements the Google Storage API calls needed by camlistore. -// This is intended to be exclude camlistore-specific logic. - +// Package googlestorage implements a generic Google Storage API +// client. It does not include any Camlistore-specific logic. package googlestorage import ( diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go index 3a42c97bd..21d384a51 100644 --- a/pkg/httputil/httputil.go +++ b/pkg/httputil/httputil.go @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package httputil contains a bunch of HTTP utility code, some generic, +// and some Camlistore-specific. package httputil import (