vivify: support in camput and upload handler.

no special credentials/auth yet.
pkg/blobserver/handlers/get.go moved to
pkg/blobserver/gethandler/get.go to avoid
a dependency loop with the json sign helper.
pkg/server/sig.go was moved to pkg/jsonsign/signhandler
because it seemed inapproriate to import in
pkg/blobserver/handlers/upload.go something from
pkg/server

Change-Id: Ifeb14512e182e8a101d4fced6d6d4184e2b9cb99
This commit is contained in:
mpl 2012-12-29 15:51:42 +01:00
parent 8560c6d513
commit ed20da7592
10 changed files with 165 additions and 51 deletions

View File

@ -60,7 +60,11 @@ func init() {
cmd := new(fileCmd)
flags.BoolVar(&cmd.makePermanode, "permanode", false, "Create an associate a new permanode for the uploaded file or directory.")
flags.BoolVar(&cmd.filePermanodes, "filenodes", false, "Create (if necessary) content-based permanodes for each uploaded file.")
flags.BoolVar(&cmd.vivify, "vivify", false, "Ask the server to vivify that file for us.")
// TODO(mpl): check against possibly conflicting flags
flags.BoolVar(&cmd.vivify, "vivify", false,
"If true, ask the server to create and sign permanode(s) associated with each uploaded"+
" file. This permits the server to have your signing key. Used mostly with untrusted"+
" or at-risk clients, such as phones.")
flags.StringVar(&cmd.name, "name", "", "Optional name attribute to set on permanode when using -permanode.")
flags.StringVar(&cmd.tag, "tag", "", "Optional tag(s) to set on permanode when using -permanode or -filenodes. Single value or comma separated.")
@ -453,9 +457,9 @@ func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
if err != nil {
return nil, err
}
blobref := blobref.SHA1FromString(json)
bref := blobref.SHA1FromString(json)
h := &client.UploadHandle{
BlobRef: blobref,
BlobRef: bref,
Size: int64(len(json)),
Contents: strings.NewReader(json),
Vivify: true,

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package handlers
package gethandler
import (
"bufio"
@ -41,13 +41,13 @@ import (
var kGetPattern *regexp.Regexp = regexp.MustCompile(`/camli/([a-z0-9]+)-([a-f0-9]+)$`)
type GetHandler struct {
type Handler struct {
Fetcher blobref.StreamingFetcher
AllowGlobalAccess bool
}
func CreateGetHandler(fetcher blobref.StreamingFetcher) func(http.ResponseWriter, *http.Request) {
gh := &GetHandler{Fetcher: fetcher}
gh := &Handler{Fetcher: fetcher}
return func(conn http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/camli/sha1-deadbeef00000000000000000000000000000000" {
// Test handler.
@ -61,7 +61,7 @@ func CreateGetHandler(fetcher blobref.StreamingFetcher) func(http.ResponseWriter
const fetchFailureDelayNs = 200e6 // 200 ms
const maxJSONSize = 64 * 1024 // should be enough for everyone
func (h *GetHandler) ServeHTTP(conn http.ResponseWriter, req *http.Request) {
func (h *Handler) ServeHTTP(conn http.ResponseWriter, req *http.Request) {
blobRef := blobFromUrlPath(req.URL.Path)
if blobRef == nil {
http.Error(conn, "Malformed GET URL.", 400)

View File

@ -17,6 +17,7 @@ limitations under the License.
package handlers
import (
"crypto/sha1"
"errors"
"fmt"
"io"
@ -25,10 +26,13 @@ import (
"net/http"
"regexp"
"strings"
"time"
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/jsonsign/signhandler"
"camlistore.org/pkg/schema"
)
// We used to require that multipart sections had a content type and
@ -60,6 +64,81 @@ func wrapReceiveConfiger(cw blobserver.ContextWrapper,
return &mixAndMatch{newRC, oldRC}
}
// vivify verifies that all the chunks for the file described by fileblob are on the blobserver.
// It makes a planned permanode, signs it, and uploads it. It finally makes a camliContent claim
// on that permanode for fileblob, signs it, and uploads it to the blobserver.
func vivify(blobReceiver blobserver.BlobReceiveConfiger, fileblob blobref.SizedBlobRef) error {
sf, ok := blobReceiver.(blobref.StreamingFetcher)
if !ok {
return fmt.Errorf("BlobReceiver is not a StreamingFetcher")
}
fetcher := blobref.SeekerFromStreamingFetcher(sf)
fr, err := schema.NewFileReader(fetcher, fileblob.BlobRef)
if err != nil {
return fmt.Errorf("Filereader error for blobref %v: %v", fileblob.BlobRef.String(), err)
}
defer fr.Close()
h := sha1.New()
n, err := io.Copy(h, fr)
if err != nil {
return fmt.Errorf("Could not read all file of blobref %v: %v", fileblob.BlobRef.String(), err)
}
if n != fr.Size() {
return fmt.Errorf("Could not read all file of blobref %v. Wanted %v, got %v", fileblob.BlobRef.String(), fr.Size(), n)
}
config := blobReceiver.Config()
if config == nil {
return errors.New("blobReceiver has no config")
}
hf := config.HandlerFinder
if hf == nil {
return errors.New("blobReceiver config has no HandlerFinder")
}
JSONSignRoot, sh, err := hf.FindHandlerByType("jsonsign")
// TODO(mpl): second check should not be necessary, and yet it happens. Figure it out.
if err != nil || sh == nil {
return errors.New("jsonsign handler not found")
}
sigHelper, ok := sh.(*signhandler.Handler)
if !ok {
return errors.New("handler is not a JSON signhandler")
}
discoMap := sigHelper.DiscoveryMap(JSONSignRoot)
publicKeyBlobRef, ok := discoMap["publicKeyBlobRef"].(string)
if !ok {
return fmt.Errorf("Discovery: json decoding error: %v", err)
}
unsigned := schema.NewHashPlannedPermanode(h)
unsigned["camliSigner"] = publicKeyBlobRef
signed, err := sigHelper.SignMap(unsigned)
if err != nil {
return fmt.Errorf("Signing permanode %v: %v", signed, err)
}
signedPerm := blobref.SHA1FromString(signed)
_, err = blobReceiver.ReceiveBlob(signedPerm, strings.NewReader(signed))
if err != nil {
return fmt.Errorf("While uploading signed permanode %v: %v", signed, err)
}
contentAttr := schema.NewSetAttributeClaim(signedPerm, "camliContent", fileblob.BlobRef.String())
claimDate, err := time.Parse(time.RFC3339, fr.FileSchema().UnixMtime)
contentAttr.SetClaimDate(claimDate)
contentAttr["camliSigner"] = publicKeyBlobRef
signed, err = sigHelper.SignMap(contentAttr)
if err != nil {
return fmt.Errorf("Signing camliContent claim: %v", err)
}
signedClaim := blobref.SHA1FromString(signed)
_, err = blobReceiver.ReceiveBlob(signedClaim, strings.NewReader(signed))
if err != nil {
return fmt.Errorf("While uploading signed camliContent claim %v: %v", signed, err)
}
return nil
}
func handleMultiPartUpload(conn http.ResponseWriter, req *http.Request, blobReceiver blobserver.BlobReceiveConfiger) {
if w, ok := blobReceiver.(blobserver.ContextWrapper); ok {
blobReceiver = wrapReceiveConfiger(w, req, blobReceiver)
@ -148,10 +227,6 @@ func handleMultiPartUpload(conn http.ResponseWriter, req *http.Request, blobRece
receivedBlobs = append(receivedBlobs, blobGot)
}
if req.Header.Get("X-Camlistore-Vivify") == "1" {
// TODO(mpl)
}
ret, err := commonUploadResponse(blobReceiver, req)
if err != nil {
httputil.ServerError(conn, req, err)
@ -166,6 +241,17 @@ func handleMultiPartUpload(conn http.ResponseWriter, req *http.Request, blobRece
}
ret["received"] = received
if req.Header.Get("X-Camlistore-Vivify") == "1" {
for _, got := range receivedBlobs {
err := vivify(blobReceiver, got)
if err != nil {
addError(fmt.Sprintf("Error vivifying blob %v: %v\n", got.BlobRef.String(), err))
} else {
conn.Header().Add("X-Camlistore-Vivified", got.BlobRef.String())
}
}
}
if errText != "" {
ret["errorText"] = errText
}

View File

@ -114,7 +114,8 @@ type Config struct {
CanLongPoll bool
// the "http://host:port" and optional path (but without trailing slash) to have "/camli/*" appended
URLBase string
URLBase string
HandlerFinder FindHandlerByTyper
}
type Configer interface {
@ -214,4 +215,4 @@ func Unwrap(sto interface{}) interface{} {
return Unwrap(g.GetStorage())
}
return sto
}
}

View File

@ -27,7 +27,19 @@ import (
var ErrHandlerTypeNotFound = errors.New("requested handler type not loaded")
type FindHandlerByTyper interface {
// FindHandlerByType finds a handler by its handlerType and
// returns its prefix and handler if it's loaded. If it's not
// loaded, the error will be ErrHandlerTypeNotFound.
//
// This is used by handler constructors to find siblings (such as the "ui" type handler)
// which might have more knowledge about the configuration for discovery, etc.
FindHandlerByType(handlerType string) (prefix string, handler interface{}, err error)
}
type Loader interface {
FindHandlerByTyper
// MyPrefix returns the prefix of the handler currently being constructed.
MyPrefix() string
@ -37,14 +49,6 @@ type Loader interface {
// Returns either a Storage or an http.Handler
GetHandler(prefix string) (interface{}, error)
// FindHandlerByType finds a handler by its handlerType and
// returns its prefix and handler if it's loaded. If it's not
// loaded, the error will be ErrHandlerTypeNotFound.
//
// This is used by handler constructors to find siblings (such as the "ui" type handler)
// which might have more knowledge about the configuration for discovery, etc.
FindHandlerByType(handlerType string) (prefix string, handler interface{}, err error)
// If we're loading configuration in response to a web request
// (as we do with App Engine), then this returns a request and
// true.

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package server
package signhandler
import (
"crypto"
@ -27,7 +27,7 @@ import (
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/handlers"
"camlistore.org/pkg/blobserver/gethandler"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/jsonsign"
@ -40,7 +40,7 @@ var _ = log.Printf
const kMaxJSONLength = 1024 * 1024
type JSONSignHandler struct {
type Handler struct {
// Optional path to non-standard secret gpg keyring file
secretRing string
@ -57,7 +57,7 @@ type JSONSignHandler struct {
entity *openpgp.Entity
}
func (h *JSONSignHandler) secretRingPath() string {
func (h *Handler) secretRingPath() string {
if h.secretRing != "" {
return h.secretRing
}
@ -74,7 +74,7 @@ func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Hand
// either a short form ("26F5ABDA") or one the longer forms.
keyId := conf.RequiredString("keyId")
h := &JSONSignHandler{
h := &Handler{
secretRing: conf.OptionalString("secretRing", ""),
}
var err error
@ -115,7 +115,7 @@ func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Hand
}
}
h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
h.pubKeyHandler = &handlers.GetHandler{
h.pubKeyHandler = &gethandler.Handler{
Fetcher: ms,
AllowGlobalAccess: true, // just public keys
}
@ -123,7 +123,7 @@ func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Hand
return h, nil
}
func (h *JSONSignHandler) uploadPublicKey(sto blobserver.Storage, key string) error {
func (h *Handler) uploadPublicKey(sto blobserver.Storage, key string) error {
_, err := blobserver.StatBlob(sto, h.pubKeyBlobRef)
if err == nil {
return nil
@ -132,7 +132,7 @@ func (h *JSONSignHandler) uploadPublicKey(sto blobserver.Storage, key string) er
return err
}
func (h *JSONSignHandler) discoveryMap(base string) map[string]interface{} {
func (h *Handler) DiscoveryMap(base string) map[string]interface{} {
m := map[string]interface{}{
"publicKeyId": h.entity.PrimaryKey.KeyIdString(),
"signHandler": base + "camli/sig/sign",
@ -145,7 +145,7 @@ func (h *JSONSignHandler) discoveryMap(base string) map[string]interface{} {
return m
}
func (h *JSONSignHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
base := req.Header.Get("X-PrefixHandler-PathBase")
subPath := req.Header.Get("X-PrefixHandler-PathSuffix")
switch req.Method {
@ -163,7 +163,7 @@ func (h *JSONSignHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
http.Error(rw, "POST required", 400)
return
case "camli/sig/discovery":
httputil.ReturnJSON(rw, h.discoveryMap(base))
httputil.ReturnJSON(rw, h.DiscoveryMap(base))
return
}
case "POST":
@ -179,7 +179,7 @@ func (h *JSONSignHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
http.Error(rw, "Unsupported path or method.", http.StatusBadRequest)
}
func (h *JSONSignHandler) handleVerify(rw http.ResponseWriter, req *http.Request) {
func (h *Handler) handleVerify(rw http.ResponseWriter, req *http.Request) {
req.ParseForm()
sjson := req.FormValue("sjson")
if sjson == "" {
@ -208,7 +208,7 @@ func (h *JSONSignHandler) handleVerify(rw http.ResponseWriter, req *http.Request
httputil.ReturnJSON(rw, m)
}
func (h *JSONSignHandler) handleSign(rw http.ResponseWriter, req *http.Request) {
func (h *Handler) handleSign(rw http.ResponseWriter, req *http.Request) {
req.ParseForm()
badReq := func(s string) {
@ -243,7 +243,7 @@ func (h *JSONSignHandler) handleSign(rw http.ResponseWriter, req *http.Request)
rw.Write([]byte(signedJSON))
}
func (h *JSONSignHandler) SignMap(m schema.Map) (string, error) {
func (h *Handler) SignMap(m schema.Map) (string, error) {
m["camliSigner"] = h.pubKeyBlobRef.String()
unsigned, err := m.JSON()
if err != nil {

View File

@ -28,10 +28,12 @@ import (
"encoding/json"
"errors"
"fmt"
"hash"
"io"
"log"
"os"
"path/filepath"
"reflect"
"strconv"
"sync"
"time"
@ -40,6 +42,8 @@ import (
"camlistore.org/pkg/blobref"
)
var sha1Type = reflect.TypeOf(sha1.New())
// Map is an unencoded schema blob.
//
// A Map is typically used during construction of a new schema blob or
@ -441,6 +445,15 @@ func NewPlannedPermanode(key string) Map {
return m
}
// NewHashPlannedPermanode returns a planned permanode with the sum
// of the hash, prefixed with "sha1-", as the key.
func NewHashPlannedPermanode(h hash.Hash) Map {
if reflect.TypeOf(h) != sha1Type {
panic("Hash not supported. Only sha1 for now.")
}
return NewPlannedPermanode(fmt.Sprintf("sha1-%x", h.Sum(nil)))
}
// Map returns a Camli map of camliType "static-set"
func (ss *StaticSet) Map() Map {
m := newMap(1, "static-set")
@ -619,7 +632,7 @@ func NewDelAttributeClaim(permaNode *blobref.BlobRef, attr string) Map {
// MapFromReader parses a JSON schema map from the provided reader r.
func MapFromReader(r io.Reader) (Map, error) {
m := make(Map)
if err := json.NewDecoder(io.LimitReader(r, 1 << 20)).Decode(&m); err != nil {
if err := json.NewDecoder(io.LimitReader(r, 1<<20)).Decode(&m); err != nil {
return nil, err
}
return m, nil

View File

@ -34,6 +34,7 @@ import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/client" // just for NewUploadHandleFromString. move elsewhere?
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/jsonsign/signhandler"
"camlistore.org/pkg/schema"
"camlistore.org/pkg/search"
"net/url"
@ -105,7 +106,7 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han
return nil, fmt.Errorf("publish handler's rootPermanode first value not a jsonsign")
}
h, _ := ld.GetHandler(rootNode[0])
jsonSign := h.(*JSONSignHandler)
jsonSign := h.(*signhandler.Handler)
pn := blobref.Parse(rootNode[1])
if err := ph.setRootNode(jsonSign, pn); err != nil {
return nil, fmt.Errorf("error setting publish root permanode: %v", err)
@ -116,7 +117,7 @@ func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Han
return nil, fmt.Errorf("publish handler's devBootstrapPermanodeUsing must be of type jsonsign")
}
h, _ := ld.GetHandler(bootstrapSignRoot)
jsonSign := h.(*JSONSignHandler)
jsonSign := h.(*signhandler.Handler)
if err := ph.bootstrapPermanode(jsonSign); err != nil {
return nil, fmt.Errorf("error bootstrapping permanode: %v", err)
}
@ -600,7 +601,7 @@ func (pr *publishRequest) fileSchemaRefFromBlob(des *search.DescribedBlob) (file
return
}
func (ph *PublishHandler) signUpload(jsonSign *JSONSignHandler, name string, m map[string]interface{}) (*blobref.BlobRef, error) {
func (ph *PublishHandler) signUpload(jsonSign *signhandler.Handler, name string, m map[string]interface{}) (*blobref.BlobRef, error) {
signed, err := jsonSign.SignMap(m)
if err != nil {
return nil, fmt.Errorf("error signing %s: %v", name, err)
@ -613,7 +614,7 @@ func (ph *PublishHandler) signUpload(jsonSign *JSONSignHandler, name string, m m
return uh.BlobRef, nil
}
func (ph *PublishHandler) setRootNode(jsonSign *JSONSignHandler, pn *blobref.BlobRef) (err error) {
func (ph *PublishHandler) setRootNode(jsonSign *signhandler.Handler, pn *blobref.BlobRef) (err error) {
_, err = ph.signUpload(jsonSign, "set-attr camliRoot", schema.NewSetAttributeClaim(pn, "camliRoot", ph.RootName))
if err != nil {
return err
@ -622,7 +623,7 @@ func (ph *PublishHandler) setRootNode(jsonSign *JSONSignHandler, pn *blobref.Blo
return err
}
func (ph *PublishHandler) bootstrapPermanode(jsonSign *JSONSignHandler) (err error) {
func (ph *PublishHandler) bootstrapPermanode(jsonSign *signhandler.Handler) (err error) {
if pn, err := ph.Search.Index().PermanodeOfSignerAttrValue(ph.Search.Owner(), "camliRoot", ph.RootName); err == nil {
log.Printf("Publish root %q using existing permanode %s", ph.RootName, pn)
return nil

View File

@ -37,6 +37,7 @@ import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/jsonsign/signhandler"
"camlistore.org/pkg/osutil"
newuistatic "camlistore.org/server/camlistored/newui"
uistatic "camlistore.org/server/camlistored/ui"
@ -77,7 +78,7 @@ type UIHandler struct {
prefix string // of the UI handler itself
root *RootHandler
sigh *JSONSignHandler // or nil
sigh *signhandler.Handler // or nil
Cache blobserver.Storage // or nil
sc ScaledImage // cache for scaled images, optional
@ -104,7 +105,7 @@ func newUIFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler,
if ui.JSONSignRoot != "" {
h, _ := ld.GetHandler(ui.JSONSignRoot)
if sigh, ok := h.(*JSONSignHandler); ok {
if sigh, ok := h.(*signhandler.Handler); ok {
ui.sigh = sigh
}
}
@ -304,7 +305,7 @@ func (ui *UIHandler) populateDiscoveryMap(m map[string]interface{}) {
"publishRoots": pubRoots,
}
if ui.sigh != nil {
uiDisco["signing"] = ui.sigh.discoveryMap(ui.JSONSignRoot)
uiDisco["signing"] = ui.sigh.DiscoveryMap(ui.JSONSignRoot)
}
for k, v := range uiDisco {
if _, ok := m[k]; ok {

View File

@ -31,6 +31,7 @@ import (
"camlistore.org/pkg/auth"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/gethandler"
"camlistore.org/pkg/blobserver/handlers"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/jsonconfig"
@ -116,7 +117,7 @@ func handleCamliUsingStorage(conn http.ResponseWriter, req *http.Request, action
case "stat":
handler = auth.RequireAuth(handlers.CreateStatHandler(storage))
default:
handler = handlers.CreateGetHandler(storage)
handler = gethandler.CreateGetHandler(storage)
}
case "POST":
switch action {
@ -134,7 +135,7 @@ func handleCamliUsingStorage(conn http.ResponseWriter, req *http.Request, action
}
// where prefix is like "/" or "/s3/" for e.g. "/camli/" or "/s3/camli/*"
func makeCamliHandler(prefix, baseURL string, storage blobserver.Storage) http.Handler {
func makeCamliHandler(prefix, baseURL string, storage blobserver.Storage, hf blobserver.FindHandlerByTyper) http.Handler {
if !strings.HasSuffix(prefix, "/") {
panic("expected prefix to end in slash")
}
@ -146,11 +147,12 @@ func makeCamliHandler(prefix, baseURL string, storage blobserver.Storage) http.H
storageConfig := &storageAndConfig{
storage,
&blobserver.Config{
Writable: true,
Readable: true,
IsQueue: false,
URLBase: baseURL + prefix[:len(prefix)-1],
CanLongPoll: canLongPoll,
Writable: true,
Readable: true,
IsQueue: false,
URLBase: baseURL + prefix[:len(prefix)-1],
CanLongPoll: canLongPoll,
HandlerFinder: hf,
},
}
return http.HandlerFunc(func(conn http.ResponseWriter, req *http.Request) {
@ -169,6 +171,8 @@ func (hl *handlerLoader) GetRequestContext() (req *http.Request, ok bool) {
return hl.context, hl.context != nil
}
// TODO(mpl): investigate bug: when I used it to find /sighelper/ within
// makeCamliHandler, it returned "/sighelper", nil, nil.
func (hl *handlerLoader) FindHandlerByType(htype string) (prefix string, handler interface{}, err error) {
for prefix, config := range hl.config {
if config.htype == htype {
@ -269,7 +273,7 @@ func (hl *handlerLoader) setupHandler(prefix string) {
h.prefix, stype, err)
}
hl.handler[h.prefix] = pstorage
hl.installer.Handle(prefix+"camli/", makeCamliHandler(prefix, hl.baseURL, pstorage))
hl.installer.Handle(prefix+"camli/", makeCamliHandler(prefix, hl.baseURL, pstorage, hl))
return
}