/* Copyright 2014 The Perkeep Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package storage implements a generic Azure storage client, not specific // to Perkeep. package storage import ( "bytes" "context" "encoding/base64" "encoding/xml" "errors" "fmt" "hash" "io" "io/ioutil" "log" "net/http" "net/url" "os" "strconv" ) const maxList = 5000 // Client is an Azure storage client. type Client struct { *Auth Transport http.RoundTripper // or nil for the default // Hostname is the hostname to use in the requests. // By default its .blob.core.windows.net. Hostname string } const defaultHost = "blob.core.windows.net" func (c *Client) hostname() string { if c.Hostname != "" { return c.Hostname } if c.Account == "" { panic("Account not set") } return c.Account + "." + defaultHost } // Container is the result of an enumeration of containers // TODO(gv): There are come more properties being exposed by Azure that we don't need right now type Container struct { Name string } func (c *Client) transport() http.RoundTripper { if c.Transport != nil { return c.Transport } return http.DefaultTransport } // containerURL returns the URL prefix of the container, with trailing slash func (c *Client) containerURL(container string) string { return fmt.Sprintf("https://%s/%s/", c.hostname(), container) } func (c *Client) keyURL(container, key string) string { return c.containerURL(container) + key } func newReq(ctx context.Context, url_ string) *http.Request { req, err := http.NewRequest("GET", url_, nil) if err != nil { panic(fmt.Sprintf("azure client; invalid URL: %v", err)) } req.Header.Set("User-Agent", "go-camlistore-azure") req.Header.Set("x-ms-version", "2014-02-14") return req.WithContext(ctx) } // Containers list the containers active under the current account. func (c *Client) Containers(ctx context.Context) ([]*Container, error) { req := newReq(ctx, "https://"+c.hostname()+"/") c.Auth.SignRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return nil, err } defer res.Body.Close() if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("azure: Unexpected status code %d fetching container list", res.StatusCode) } return parseListAllMyContainers(res.Body) } func parseListAllMyContainers(r io.Reader) ([]*Container, error) { type allMyContainers struct { Containers struct { Container []*Container } } var res allMyContainers if err := xml.NewDecoder(r).Decode(&res); err != nil { return nil, err } return res.Containers.Container, nil } // Stat Stats a blob in Azure. // It returns 0, os.ErrNotExist if not found on Azure, otherwise reterr is real. func (c *Client) Stat(ctx context.Context, key, container string) (size int64, reterr error) { req := newReq(ctx, c.keyURL(container, key)) req.Method = "HEAD" c.Auth.SignRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return 0, err } if res.Body != nil { defer res.Body.Close() } switch res.StatusCode { case http.StatusNotFound: return 0, os.ErrNotExist case http.StatusOK: return strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64) } return 0, fmt.Errorf("azure: Unexpected status code %d statting object %v", res.StatusCode, key) } // PutObject puts a blob to the specified container on Azure func (c *Client) PutObject(ctx context.Context, key, container string, md5 hash.Hash, size int64, body io.Reader) error { req := newReq(ctx, c.keyURL(container, key)) req.Method = "PUT" req.ContentLength = size if md5 != nil { b64 := new(bytes.Buffer) encoder := base64.NewEncoder(base64.StdEncoding, b64) encoder.Write(md5.Sum(nil)) encoder.Close() req.Header.Set("Content-MD5", b64.String()) } req.Header.Set("Content-Length", strconv.Itoa(int(size))) req.Header.Set("x-ms-blob-type", "BlockBlob") c.Auth.SignRequest(req) req.Body = ioutil.NopCloser(body) res, err := c.transport().RoundTrip(req) if res != nil && res.Body != nil { defer res.Body.Close() } if err != nil { return err } if res.StatusCode != http.StatusCreated { if res.StatusCode < 500 { aerr := getAzureError("PutObject", res) return aerr } return fmt.Errorf("got response code %d from Azure", res.StatusCode) } return nil } // BlobProperties holds some information about the blobs. // There are many more fields than just the one below, see: // http://msdn.microsoft.com/en-us/library/azure/dd135734.aspx type BlobProperties struct { ContentLength int `xml:"Content-Length"` } // Blob holds the name and properties of a blob when returned with a list-operation type Blob struct { Name string Properties BlobProperties } type listBlobsResults struct { Blobs struct { Blob []*Blob } MaxResults int ContainerName string `xml:",attr"` Marker string NextMarker string } // ListBlobs returns 0 to maxKeys (inclusive) items from the provided // container. If the length of the returned items is equal to maxKeys, // there is no indication whether or not the returned list is truncated. func (c *Client) ListBlobs(ctx context.Context, container string, maxResults int) (blobs []*Blob, err error) { if maxResults < 0 { return nil, errors.New("invalid negative maxKeys") } marker := "" for len(blobs) < maxResults { fetchN := maxResults - len(blobs) if fetchN > maxList { fetchN = maxList } var bres listBlobsResults listURL := fmt.Sprintf("%s?restype=container&comp=list&marker=%s&maxresults=%d", c.containerURL(container), url.QueryEscape(marker), fetchN) req := newReq(ctx, listURL) c.Auth.SignRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return nil, err } if res.StatusCode != http.StatusOK { if res.StatusCode < 500 { aerr := getAzureError("ListBlobs", res) return nil, aerr } } else { bres = listBlobsResults{} var logbuf bytes.Buffer err = xml.NewDecoder(io.TeeReader(res.Body, &logbuf)).Decode(&bres) if err != nil { log.Printf("Error parsing Azure XML response: %v for %q", err, logbuf.Bytes()) } else if bres.MaxResults != fetchN || bres.ContainerName != container || bres.Marker != marker { err = fmt.Errorf("unexpected parse from server: %#v from: %s", bres, logbuf.Bytes()) log.Print(err) } } res.Body.Close() if err != nil { log.Print(err) return nil, err } for _, it := range bres.Blobs.Blob { blobs = append(blobs, it) } if bres.NextMarker == "" { // No more blobs to list break } marker = bres.NextMarker } return blobs, nil } func getAzureError(operation string, res *http.Response) *Error { body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20)) aerr := &Error{ Op: operation, Code: res.StatusCode, Body: body, Header: res.Header, } aerr.parseXML() res.Body.Close() return aerr } // Get retrieves a blob from Azure or returns os.ErrNotExist if not found func (c *Client) Get(ctx context.Context, container, key string) (body io.ReadCloser, size int64, err error) { req := newReq(ctx, c.keyURL(container, key)) c.Auth.SignRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return } switch res.StatusCode { case http.StatusOK: return res.Body, res.ContentLength, nil case http.StatusNotFound: res.Body.Close() return nil, 0, os.ErrNotExist default: res.Body.Close() return nil, 0, fmt.Errorf("azure HTTP error on GET: %d", res.StatusCode) } } // GetPartial fetches part of the blob in container. // If length is negative, the rest of the object is returned. // The caller must close rc. func (c *Client) GetPartial(ctx context.Context, container, key string, offset, length int64) (rc io.ReadCloser, err error) { if offset < 0 { return nil, errors.New("invalid negative length") } req := newReq(ctx, c.keyURL(container, key)) if length >= 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) } else { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } c.Auth.SignRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return } switch res.StatusCode { case http.StatusOK, http.StatusPartialContent: return res.Body, nil case http.StatusNotFound: res.Body.Close() return nil, os.ErrNotExist default: res.Body.Close() return nil, fmt.Errorf("azure HTTP error on GET: %d", res.StatusCode) } } // Delete deletes a blob from the specified container. // It may take a few moments before the blob is actually deleted by Azure. func (c *Client) Delete(ctx context.Context, container, key string) error { req := newReq(ctx, c.keyURL(container, key)) req.Method = "DELETE" c.Auth.SignRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return err } if res != nil && res.Body != nil { defer res.Body.Close() } if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusNoContent || res.StatusCode == http.StatusAccepted { return nil } return fmt.Errorf("azure HTTP error on DELETE: %d", res.StatusCode) } // IsValidContainer reports whether container is a valid container name, per Microsoft's naming restrictions. // // See http://msdn.microsoft.com/en-us/library/azure/dd135715.aspx func IsValidContainer(container string) bool { l := len(container) if l < 3 || l > 63 { return false } valid := false prev := byte('-') for i := 0; i < len(container); i++ { c := container[i] switch { default: return false case 'a' <= c && c <= 'z': valid = true case '0' <= c && c <= '9': // Is allowed, but containername can't be just numbers. // Therefore, don't set valid to true case c == '-': if prev == '-' { return false } } prev = c } if prev == '-' { return false } return valid } // Error is the type returned by some API operations. type Error struct { Op string Code int // HTTP status code Body []byte // response body Header http.Header // response headers AzureError XMLError } // Error returns a formatted error message func (e *Error) Error() string { if e.AzureError.Code != "" { return fmt.Sprintf("azure.%s: status %d, code: %s", e.Op, e.Code, e.AzureError.Code) } return fmt.Sprintf("azure.%s: status %d", e.Op, e.Code) } func (e *Error) parseXML() { _ = xml.NewDecoder(bytes.NewReader(e.Body)).Decode(&e.AzureError) if e.AzureError.Code == "AuthenticationFailed" { log.Printf("Azure AuthenticationFailed. Details: %s", e.AzureError.AuthenticationErrorDetail) } } // XMLError is the Error response from Azure. type XMLError struct { Code string Message string AuthenticationErrorDetail string }