More docs

Change-Id: I5c21f240c85bcf91fb67487cc172bf3faeb49fff

TESTS

@@ -3,4 +3,6 @@ Tests needed
 -cmd/camput/
 	-verify that stat caching works. verify that -filenodes does create the permanode even if the file was already uploaded (and cached) in a previous run.
 
--- blobserver/cond has no tests.
+-- blobserver/{cond,replica,remote,shard} have no tests. should be easier now
+   that test.Fetcher is a full blobserver? see encrypt's nascent tests for
+   examples.

TODO

@@ -4,6 +4,10 @@ There are two TODO lists. This file (good for airplanes) and the online bug trac
 
 Offline list:
 
+-- why can't ./pkg/blobserver/gethandler be merged into
+   ./pkg/blobserver/handlers? I forget the reason, but it seems to
+   work now, so maybe the circular dependency is gone.
+
 -- support for running cammount under camlistored. especially for OS X,
    where the lifetime of the background daemon will be the same as the
    user's login session.

pkg/blobserver/google/storage.go

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package google registers the "google" blob storage type, storing blobs
+// on Google Cloud Storage (not Google Drive).
 package google
 
 import (

pkg/blobserver/localdisk/localdisk.go

@@ -14,6 +14,20 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/*
+Package localdisk registers the "filesystem" blobserver storage type,
+storing blobs in a forest of sharded directories at the specified root.
+
+Example low-level config:
+
+     "/storage/": {
+         "handler": "storage-filesystem",
+         "handlerArgs": {
+            "path": "/var/camlistore/blobs"
+          }
+     },
+
+*/
 package localdisk
 
 import (
@@ -27,6 +41,8 @@ import (
 	"camlistore.org/pkg/jsonconfig"
 )
 
+// DiskStorage implements the blobserver.Storage interface using the
+// local filesystem.
 type DiskStorage struct {
 	*blobserver.SimpleBlobHubPartitionMap
 	root string
@@ -39,8 +55,8 @@ type DiskStorage struct {
 	mirrorPartitions []*DiskStorage
 }
 
-// New returns a new local disk storage implementation, rooted at the provided
-// directory, which must already exist.
+// New returns a new local disk storage implementation at the provided
+// root directory, which must already exist.
 func New(root string) (*DiskStorage, error) {
 	// Local disk.
 	fi, err := os.Stat(root)

pkg/blobserver/localdisk/path.go

@@ -25,7 +25,7 @@ import (
 	"net/url"
 )
 
-func BlobFileBaseName(b *blobref.BlobRef) string {
+func blobFileBaseName(b *blobref.BlobRef) string {
 	return fmt.Sprintf("%s-%s.dat", b.HashName(), b.Digest())
 }
 
@@ -38,7 +38,7 @@ func (ds *DiskStorage) blobDirectory(partition string, b *blobref.BlobRef) strin
 }
 
 func (ds *DiskStorage) blobPath(partition string, b *blobref.BlobRef) string {
-	return filepath.Join(ds.blobDirectory(partition, b), BlobFileBaseName(b))
+	return filepath.Join(ds.blobDirectory(partition, b), blobFileBaseName(b))
 }
 
 func (ds *DiskStorage) PartitionRoot(partition string) string {

pkg/blobserver/localdisk/receive.go

@@ -41,7 +41,7 @@ func (ds *DiskStorage) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (
 		return
 	}
 
-	tempFile, err := ioutil.TempFile(hashedDirectory, BlobFileBaseName(blobRef)+".tmp")
+	tempFile, err := ioutil.TempFile(hashedDirectory, blobFileBaseName(blobRef)+".tmp")
 	if err != nil {
 		return
 	}

pkg/blobserver/remote/remote.go

@@ -14,6 +14,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/*
+Package remote registers the "remote" blobserver storage type, storing
+and fetching blobs from a remote Camlistore server, speaking the HTTP
+protocol.
+
+Example low-level config:
+
+     "/peer/": {
+         "handler": "storage-remote",
+         "handlerArgs": {
+             "url": "http://10.0.0.17/base",
+             "skipStartupCheck": false
+          }
+     },
+
+*/
 package remote
 
 import (
@@ -35,6 +51,8 @@ type remoteStorage struct {
 
 var _ = blobserver.Storage((*remoteStorage)(nil))
 
+// NewFromClient returns a new Storage implementation using the
+// provided Camlistore client.
 func NewFromClient(c *client.Client) blobserver.Storage {
 	return &remoteStorage{client: c}
 }

pkg/blobserver/replica/replica.go

@@ -14,6 +14,24 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/*
+Package replica registers the "replica" blobserver storage type,
+providing synchronous replication to one more backends.
+
+Writes wait for minWritesForSuccess (default: all). Reads are
+attempted in order and not load-balanced, randomized, or raced by
+default.
+
+Example config:
+
+      "/repl/": {
+          "handler": "storage-replica",
+          "handlerArgs": {
+              "backends": ["/b1/", "/b2/", "/b3/"],
+              "minWritesForSuccess": 2
+          }
+      },
+*/
 package replica
 
 import (
@@ -96,13 +114,8 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobser
 	return sto, nil
 }
 
-func (sto *replicaStorage) weightedRandomReplicas() []blobserver.Storage {
-	// TODO: implement something actually weighted or random.
-	return sto.replicas
-}
-
 func (sto *replicaStorage) FetchStreaming(b *blobref.BlobRef) (file io.ReadCloser, size int64, err error) {
-	for _, replica := range sto.weightedRandomReplicas() {
+	for _, replica := range sto.replicas {
 		file, size, err = replica.FetchStreaming(b)
 		if err == nil {
 			return
@@ -167,8 +180,7 @@ type sizedBlobAndError struct {
 	err error
 }
 
-// TODO-GO: s/xxgo/_/ once Go issue 1802 is fixd
-func (sto *replicaStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (xxgo blobref.SizedBlobRef, err error) {
+func (sto *replicaStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (_ blobref.SizedBlobRef, err error) {
 	nReplicas := len(sto.replicas)
 	rpipe, wpipe, writer := make([]*io.PipeReader, nReplicas), make([]*io.PipeWriter, nReplicas), make([]io.Writer, nReplicas)
 	for idx := range sto.replicas {

pkg/blobserver/s3/s3.go

@@ -14,6 +14,23 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/*
+Package s3 registers the "s3" blobserver storage type, storing
+blobs in an Amazon Web Services' S3 storage bucket.
+
+Example low-level config:
+
+     "/r1/": {
+         "handler": "storage-s3",
+         "handlerArgs": {
+            "bucket": "foo",
+            "aws_access_key": "...",
+            "aws_secret_access_key": "...",
+            "skipStartupCheck": false
+          }
+     },
+
+*/
 package s3
 
 import (

pkg/blobserver/shard/shard.go

@@ -14,6 +14,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/*
+Package shard registers the "shard" blobserver storage type,
+predictably spraying out blobs out over the provided backends
+based on their blobref. Each blob maps to exactly one backend.
+
+Example low-level config:
+
+     "/foo/": {
+         "handler": "storage-shard",
+         "handlerArgs": {
+             "backends": ["/s1/", "/s2/"]
+          }
+     },
+
+*/
 package shard
 
 import (

pkg/buildinfo/buildinfo.go

@@ -1,3 +1,4 @@
+// Package buildinfo provides information about the current build.
 package buildinfo
 
 // GitInfo is either the empty string (the default)
@@ -6,6 +7,8 @@ package buildinfo
 // $ go install --ldflags="-X camlistore.org/pkg/buildinfo.GitInfo "`./misc/gitversion` camlistore.org/server/camlistored
 var GitInfo string
 
+// Version returns the git version of this binary.
+// If the linker flags were not provided, the return value is "unknown".
 func Version() string {
 	if GitInfo != "" {
 		return GitInfo

pkg/client/client.go

@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package client implements a Camlistore client.
 package client
 
 import (

pkg/cmdmain/cmdmain.go

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package cmdmain contains the shared implementation for camget,
+// camput, camtool, and other Camlistore command-line tools.
 package cmdmain
 
 import (

pkg/fileembed/fileembed.go

@@ -14,6 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package fileembed provides access to static data resources (images,
+// HTML, css, etc) embedded into the binary with genfileembed.
+//
+// Most of the package contains internal details used by genfileembed.
+// Normal applications will simply make a global Files variable.
 package fileembed
 
 import (
@@ -32,6 +37,7 @@ import (
 	"time"
 )
 
+// Files contains all the embedded resources.
 type Files struct {
 	// Optional environment variable key to override
 	OverrideEnv string

pkg/fs/fs.go

@@ -16,6 +16,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package fs implements a FUSE filesystem for Camlistore and is
+// used by the cammount binary.
 package fs
 
 import (

pkg/googlestorage/googlestorage.go

@@ -14,9 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Implements the Google Storage API calls needed by camlistore.
-// This is intended to be exclude camlistore-specific logic.
-
+// Package googlestorage implements a generic Google Storage API
+// client. It does not include any Camlistore-specific logic.
 package googlestorage
 
 import (

pkg/httputil/httputil.go

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package httputil contains a bunch of HTTP utility code, some generic,
+// and some Camlistore-specific.
 package httputil
 
 import (