client: parseStatResponse

lib/go/camli/client/stat_test.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+var response = `{
+   "stat": [
+      {"blobRef": "foo-abc",
+       "size": 123},
+      {"blobRef": "foo-def",
+       "size": 999}
+   ],
+   "maxUploadSize": 1048576,
+   "uploadUrl": "http://upload-server.example.com/some/server-chosen/url",
+   "uploadUrlExpirationSeconds": 7200,
+   "canLongPoll": true
+}
+`
+
+func TestParseStatResponse(t *testing.T) {
+	res, err := parseStatResponse(strings.NewReader(response))
+	if err != nil {
+		t.Fatal(err)
+	}
+	hm := res.HaveMap
+	res.HaveMap = nil
+	want := &statResponse{
+		HaveMap:                    nil,
+		maxUploadSize:              1048576,
+		uploadUrl:                  "http://upload-server.example.com/some/server-chosen/url",
+		uploadUrlExpirationSeconds: 7200,
+		canLongPoll:                true,
+	}
+	if !reflect.DeepEqual(want, res) {
+		t.Errorf(" Got: %#v\nWant: %#v", res, want)
+	}
+
+	if sb, ok := hm["foo-abc"]; !ok || sb.Size != 123 {
+		t.Errorf("Got unexpected map: %#v", hm)
+	}
+
+	if sb, ok := hm["foo-def"]; !ok || sb.Size != 999 {
+		t.Errorf("Got unexpected map: %#v", hm)
+	}
+}

lib/go/camli/client/upload.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"http"
 	"io"
+	"io/ioutil"
 	"json"
 	"log"
 	"os"
@@ -42,13 +43,80 @@ type PutResult struct {
 	Skipped bool // already present on blobserver
 }
 
-type nopCloser struct {
-	io.Reader
+type statResponse struct {
+	HaveMap                    map[string]blobref.SizedBlobRef
+	maxUploadSize              int64
+	uploadUrl                  string
+	uploadUrlExpirationSeconds int
+	canLongPoll                bool
 }
 
-func (nopCloser) Close() os.Error { return nil }
+type ResponseFormatError os.Error
+
+func newResFormatError(s string, arg ...interface{}) ResponseFormatError {
+	return ResponseFormatError(fmt.Errorf(s, arg...))
+}
+
+func parseStatResponse(r io.Reader) (*statResponse, os.Error) {
+	var (
+		ok   bool
+		err  os.Error
+		s    = &statResponse{HaveMap: make(map[string]blobref.SizedBlobRef)}
+		jmap = make(map[string]interface{})
+	)
+	if err = json.NewDecoder(io.LimitReader(r, 5<<20)).Decode(&jmap); err != nil {
+		return nil, ResponseFormatError(err)
+	}
+
+	s.uploadUrl, ok = jmap["uploadUrl"].(string)
+	if !ok {
+		return nil, newResFormatError("no 'uploadUrl' in stat response")
+	}
+
+	if n, ok := jmap["maxUploadSize"].(float64); ok {
+		s.maxUploadSize = int64(n)
+	} else {
+		return nil, newResFormatError("no 'maxUploadSize' in stat response")
+	}
+
+	if n, ok := jmap["uploadUrlExpirationSeconds"].(float64); ok {
+		s.uploadUrlExpirationSeconds = int(n)
+	} else {
+		return nil, newResFormatError("no 'uploadUrlExpirationSeconds' in stat response")
+	}
+
+	if v, ok := jmap["canLongPoll"].(bool); ok {
+		s.canLongPoll = v
+	}
+
+	alreadyHave, ok := jmap["stat"].([]interface{})
+	if !ok {
+		return nil, newResFormatError("no 'stat' key in stat response")
+	}
+
+	for _, li := range alreadyHave {
+		m, ok := li.(map[string]interface{})
+		if !ok {
+			return nil, newResFormatError("'stat' list value of unexpected type %T", li)
+		}
+		blobRefStr, ok := m["blobRef"].(string)
+		if !ok {
+			return nil, newResFormatError("'stat' list item has non-string 'blobRef' key")
+		}
+		size, ok := m["size"].(float64)
+		if !ok {
+			return nil, newResFormatError("'stat' list item has non-number 'size' key")
+		}
+		br := blobref.Parse(blobRefStr)
+		if br == nil {
+			return nil, newResFormatError("'stat' list item has invalid 'blobRef' key")
+		}
+		s.HaveMap[br.String()] = blobref.SizedBlobRef{br, int64(size)}
+	}
+
+	return s, nil
+}
 
-// Note: must not touch data after calling this.
 func NewUploadHandleFromString(data string) *UploadHandle {
 	s1 := sha1.New()
 	s1.Write([]byte(data))
@@ -115,8 +183,8 @@ func (c *Client) Stat(dest chan<- *blobref.SizedBlobRef, blobs []*blobref.BlobRe
 }
 
 func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
-	error := func(msg string, e os.Error) (*PutResult, os.Error) {
-		err := fmt.Errorf("Error uploading blob %s: %s; err=%v", h.BlobRef, msg, e)
+	error := func(msg string, arg ...interface{}) (*PutResult, os.Error) {
+		err := fmt.Errorf(msg, arg...)
 		c.log.Print(err.String())
 		return nil, err
 	}
@@ -134,39 +202,29 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
 	requestBody := "camliversion=1&blob1=" + blobRefString
 	req := c.newRequest("POST", url)
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	req.Body = &nopCloser{strings.NewReader(requestBody)}
+	req.Body = ioutil.NopCloser(strings.NewReader(requestBody))
 	req.ContentLength = int64(len(requestBody))
 	req.TransferEncoding = nil
 
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
-		return error("stat http error", err)
+		return error("stat http error: %v", err)
 	}
 
-	pur, err := c.jsonFromResponse("pre-upload stat", resp)
+	if resp.StatusCode != 200 {
+		return error("stat response had http status %d", resp.StatusCode)
+
+	}
+
+	stat, err := parseStatResponse(resp.Body)
 	if err != nil {
-		return error("json parse error", fmt.Errorf("response from %s wasn't valid JSON; wrong URL prefix?", url))
-	}
-
-	uploadUrl, ok := pur["uploadUrl"].(string)
-	if uploadUrl == "" {
-		return error("stat json validity error: no 'uploadUrl'", nil)
-	}
-	log.Printf("Got upload url: %q", uploadUrl)
-
-	alreadyHave, ok := pur["stat"].([]interface{})
-	if !ok {
-		return error("stat json validity error: no 'stat'", nil)
+		return nil, err
 	}
 
 	pr := &PutResult{BlobRef: h.BlobRef, Size: h.Size}
-
-	for _, haveObj := range alreadyHave {
-		haveObj := haveObj.(map[string]interface{})
-		if haveObj["blobRef"].(string) == h.BlobRef.String() {
-			pr.Skipped = true
-			return pr, nil
-		}
+	if _, ok := stat.HaveMap[h.BlobRef.String()]; ok {
+		pr.Skipped = true
+		return pr, nil
 	}
 
 	// TODO: use a proper random boundary
@@ -180,13 +238,13 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
 		h.BlobRef, h.BlobRef)
 	multiPartFooter := "\r\n--" + boundary + "--\r\n"
 
-	c.log.Printf("Uploading to URL: %s", uploadUrl)
-	req = c.newRequest("POST", uploadUrl)
+	c.log.Printf("Uploading to URL: %s", stat.uploadUrl)
+	req = c.newRequest("POST", stat.uploadUrl)
 	req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary)
-	req.Body = &nopCloser{io.MultiReader(
+	req.Body = ioutil.NopCloser(io.MultiReader(
 		strings.NewReader(multiPartHeader),
 		h.Contents,
-		strings.NewReader(multiPartFooter))}
+		strings.NewReader(multiPartFooter)))
 
 	req.ContentLength = int64(len(multiPartHeader)) + h.Size + int64(len(multiPartFooter))
 	req.TransferEncoding = nil
@@ -205,7 +263,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
 		if otherLocation == "" {
 			return error("303 without a Location", nil)
 		}
-		baseUrl, _ := http.ParseURL(uploadUrl)
+		baseUrl, _ := http.ParseURL(stat.uploadUrl)
 		absUrl, err := baseUrl.ParseURL(otherLocation)
 		if err != nil {
 			return error("303 Location URL relative resolve error", err)