camget, client, schema: start of camget --shared support. see flag docs.

Change-Id: I5dd43129cb0032821a5913a8f20da0ddb38c63da

cmd/camget/camget.go

@@ -44,7 +44,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"camlistore.org/pkg/blobref"
 	"camlistore.org/pkg/client"
@@ -56,43 +55,48 @@ var (
 	flagVerbose  = flag.Bool("verbose", false, "be verbose")
 	flagCheck    = flag.Bool("check", false, "just check for the existence of listed blobs; returning 0 if all are present")
 	flagOutput   = flag.String("o", "-", "Output file/directory to create.  Use -f to overwrite.")
-	flagVia      = flag.String("via", "", "Fetch the blob via the given comma-separated sharerefs (dev only).")
 	flagGraph    = flag.Bool("graph", false, "Output a graphviz directed graph .dot file of the provided root schema blob, to be rendered with 'dot -Tsvg -o graph.svg graph.dot'")
 	flagContents = flag.Bool("contents", false, "If true and the target blobref is a 'bytes' or 'file' schema blob, the contents of that file are output instead.")
+	flagShared   = flag.String("shared", "", "If non-empty, the URL of a \"share\" blob. The URL will be used as the root of future fetches. Only \"haveref\" shares are currently supported.")
 )
 
-var viaRefs []*blobref.BlobRef
-
 func main() {
 	client.AddFlags()
 	flag.Parse()
 
-	if len(*flagVia) > 0 {
-		vs := strings.Split(*flagVia, ",")
-		viaRefs = make([]*blobref.BlobRef, len(vs))
-		for i, sbr := range vs {
-			viaRefs[i] = blobref.Parse(sbr)
-			if viaRefs[i] == nil {
-				log.Fatalf("Invalid -via blobref: %q", sbr)
-			}
-			if *flagVerbose {
-				log.Printf("via: %s", sbr)
-			}
-		}
-	}
-
 	if *flagGraph && flag.NArg() != 1 {
 		log.Fatalf("The --graph option requires exactly one parameter.")
 	}
 
-	cl := client.NewOrFail()
+	var cl *client.Client
+	var items []*blobref.BlobRef
 
-	for n := 0; n < flag.NArg(); n++ {
-		arg := flag.Arg(n)
-		br := blobref.Parse(arg)
-		if br == nil {
-			log.Fatalf("Failed to parse argument %q as a blobref.", arg)
+	if *flagShared != "" {
+		if client.ExplicitServer() != "" {
+			log.Fatal("Can't use --shared with an explicit blobserver; blobserver is implicit from the --shared URL.")
 		}
+		if flag.NArg() != 0 {
+			log.Fatal("No arguments permitted when using --shared")
+		}
+		cl1, target, err := client.NewFromShareRoot(*flagShared)
+		if err != nil {
+			log.Fatal(err)
+		}
+		cl = cl1
+		items = append(items, target)
+	} else {
+		cl = client.NewOrFail()
+		for n := 0; n < flag.NArg(); n++ {
+			arg := flag.Arg(n)
+			br := blobref.Parse(arg)
+			if br == nil {
+				log.Fatalf("Failed to parse argument %q as a blobref.", arg)
+			}
+			items = append(items, br)
+		}
+	}
+
+	for _, br := range items {
 		if *flagGraph {
 			printGraph(cl, br)
 			return
@@ -130,13 +134,9 @@ func fetch(cl *client.Client, br *blobref.BlobRef) (r io.ReadCloser, err error)
 	if *flagVerbose {
 		log.Printf("Fetching %s", br.String())
 	}
-	if len(viaRefs) > 0 {
-		r, _, err = cl.FetchVia(br, viaRefs)
-	} else {
-		r, _, err = cl.FetchStreaming(br)
-	}
+	r, _, err = cl.FetchStreaming(br)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to fetch %q: %s", br, err)
+		return nil, fmt.Errorf("Failed to fetch %s: %s", br, err)
 	}
 	return r, err
 }

pkg/blobref/blobref.go

@@ -24,7 +24,11 @@ import (
 	"regexp"
 )
 
-var kBlobRefPattern *regexp.Regexp = regexp.MustCompile(`^([a-z0-9]+)-([a-f0-9]+)$`)
+// Pattern is the regular expression which matches a blobref.
+// It does not contain ^ or $.
+const Pattern = `\b([a-z0-9]+)-([a-f0-9]+)\b`
+
+var kBlobRefPattern = regexp.MustCompile("^" + Pattern + "$")
 
 var supportedDigests = map[string]func() hash.Hash{
 	"sha1": func() hash.Hash {

pkg/client/client.go

@@ -27,11 +27,13 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"regexp"
 	"strings"
 	"sync"
 
 	"camlistore.org/pkg/auth"
 	"camlistore.org/pkg/blobref"
+	"camlistore.org/pkg/schema"
 )
 
 // A Client provides access to a Camlistore server.
@@ -62,6 +64,11 @@ type Client struct {
 	statsMutex sync.Mutex
 	stats      Stats
 
+	// via maps the access path from a share root to a desired target.
+	// It is non-nil when in "sharing" mode, where the Client is fetching
+	// a share.
+	via map[string]string // target => via (target is referenced from via)
+
 	log     *log.Logger // not nil
 	reqGate chan bool
 }
@@ -91,6 +98,43 @@ func NewOrFail() *Client {
 	return c
 }
 
+var shareURLRx = regexp.MustCompile(`^(.+)/camli/(` + blobref.Pattern + ")")
+
+func NewFromShareRoot(shareBlobURL string) (c *Client, target *blobref.BlobRef, err error) {
+	var root string
+	if m := shareURLRx.FindStringSubmatch(shareBlobURL); m == nil {
+		return nil, nil, fmt.Errorf("Unkown URL base; doesn't contain /camli/")
+	} else {
+		c = New(m[1])
+		c.discoOnce.Do(func() { /* nothing */
+		})
+		c.prefixOnce.Do(func() { /* nothing */
+		})
+		c.prefixv = m[1]
+		c.authMode = auth.None{}
+		c.via = make(map[string]string)
+		root = m[2]
+	}
+	res, err := http.Get(shareBlobURL)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error fetching %s: %v", shareBlobURL, err)
+	}
+	defer res.Body.Close()
+	ss, err := schema.ParseSuperset(res.Body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error parsing JSON from %s: %v", shareBlobURL, err)
+	}
+	if ss.AuthType != "haveref" {
+		return nil, nil, fmt.Errorf("Unknown share authType of %q", ss.AuthType)
+	}
+	if ss.Target == nil {
+		return nil, nil, fmt.Errorf("No target.")
+	}
+	c.via[ss.Target.String()] = root
+	// TODO(bradfitz): send via in requests, populate via as we fetch more things
+	return c, ss.Target, nil
+}
+
 // SetHTTPClient sets the Camlistore client's HTTP client.
 // If nil, the default HTTP client is used.
 func (c *Client) SetHTTPClient(client *http.Client) {

pkg/client/config.go

@@ -42,6 +42,14 @@ func AddFlags() {
 	flagServer = flag.String("blobserver", "", "camlistore blob server")
 }
 
+// ExplicitServer returns the blobserver given in the flags, if any.
+func ExplicitServer() string {
+	if flagServer != nil {
+		return *flagServer
+	}
+	return ""
+}
+
 func ConfigFilePath() string {
 	return filepath.Join(osutil.CamliConfigDir(), "config")
 }

pkg/client/upload.go

@@ -283,7 +283,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, error) {
 		return nil, err
 	}
 	url_ := fmt.Sprintf("%s/camli/stat", pfx)
-	req := c.newRequest("POST", url_, strings.NewReader("camliversion=1&blob1=" + blobrefStr))
+	req := c.newRequest("POST", url_, strings.NewReader("camliversion=1&blob1="+blobrefStr))
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 
 	resp, err := c.doReq(req)

pkg/schema/schema.go

@@ -223,8 +223,9 @@ func NewDirectoryEntryFromBlobRef(fetcher blobref.SeekFetcher, blobRef *blobref.
 // Superset represents the superset of common Camlistore JSON schema
 // keys as a convenient json.Unmarshal target.
 type Superset struct {
-	BlobRef *blobref.BlobRef // Not in JSON, but included for
-	// those who want to set it.
+	// BlobRef isn't for a particular metadata blob field, but included
+	// for convenience.
+	BlobRef *blobref.BlobRef
 
 	Version int    `json:"camliVersion"`
 	Type    string `json:"camliType"`
@@ -261,8 +262,21 @@ type Superset struct {
 	Parts []*BytesPart `json:"parts"`
 
 	Entries string   `json:"entries"` // for directories, a blobref to a static-set
-	Members []string `json:"members"` // for static sets (for directory static-sets:
-	// blobrefs to child dirs/files)
+	Members []string `json:"members"` // for static sets (for directory static-sets: blobrefs to child dirs/files)
+
+	// Target is a "share" blob's target (the thing being shared)
+	Target *blobref.BlobRef
+	// Transitive is a property of a "share" blob.
+	Transitive bool
+	// AuthType is a "share" blob's authentication type that is required.
+	// Currently (2013-01-02) just "haveref" (if you know the share's blobref,
+	// you get access: the secret URL model)
+	AuthType string
+}
+
+func ParseSuperset(r io.Reader) (*Superset, error) {
+	var ss Superset
+	return &ss, json.NewDecoder(io.LimitReader(r, 1 << 20)).Decode(&ss)
 }
 
 // BytesPart is the type representing one of the "parts" in a "file"