/* Copyright 2011 Google Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "bufio" "bytes" "errors" "fmt" "hash/crc32" "io" "log" "net/url" "os" "path/filepath" "strconv" "strings" "sync" "camlistore.org/pkg/blobref" "camlistore.org/pkg/client" "camlistore.org/pkg/osutil" ) type statFingerprint string var cleanSysStat func(v interface{}) interface{} func fileInfoToFingerprint(fi os.FileInfo) statFingerprint { // We calculate the CRC32 of the underlying system stat structure to get // ctime, owner, group, etc. This is overkill (e.g. we don't care about // the inode or device number probably), but works. sysHash := uint32(0) if sys := fi.Sys(); sys != nil { if clean := cleanSysStat; clean != nil { // TODO: don't clean bad fields, but provide a // portable way to extract all good fields. // This is a Linux+Mac-specific hack for now. sys = clean(sys) } var buf bytes.Buffer fmt.Fprintf(&buf, "%#v", sys) sysHash = crc32.ChecksumIEEE(buf.Bytes()) } return statFingerprint(fmt.Sprintf("%dB/%dMOD/sys-%d", fi.Size(), fi.ModTime().UnixNano(), sysHash)) } type fileInfoPutRes struct { Fingerprint statFingerprint Result client.PutResult } // FlatStatCache is an ugly hack, until leveldb-go is ready // (http://code.google.com/p/leveldb-go/) type FlatStatCache struct { mu sync.RWMutex filename string m map[string]fileInfoPutRes af *os.File // for appending } func escapeGen(gen string) string { // Good enough: return url.QueryEscape(gen) } func NewFlatStatCache(gen string) *FlatStatCache { filename := filepath.Join(osutil.CacheDir(), "camput.statcache."+escapeGen(gen)) fc := &FlatStatCache{ filename: filename, m: make(map[string]fileInfoPutRes), } f, err := os.Open(filename) if os.IsNotExist(err) { return fc } if err != nil { log.Fatalf("opening camput stat cache: %v", filename, err) } defer f.Close() br := bufio.NewReader(f) for { ln, err := br.ReadString('\n') if err == io.EOF { break } if err != nil { log.Printf("Warning: (ignoring) reading stat cache: %v", err) break } ln = strings.TrimSpace(ln) f := strings.Split(ln, "\t") if len(f) < 3 { continue } filename, fp, putres := f[0], statFingerprint(f[1]), f[2] f = strings.Split(putres, "/") if len(f) != 2 { continue } blobrefStr := f[0] blobSize, err := strconv.ParseInt(f[1], 10, 64) if err != nil { continue } fc.m[filename] = fileInfoPutRes{ Fingerprint: fp, Result: client.PutResult{ BlobRef: blobref.Parse(blobrefStr), Size: blobSize, Skipped: true, // is this used? }, } } vlog.Printf("Flatcache read %d entries from %s", len(fc.m), filename) return fc } var _ UploadCache = (*FlatStatCache)(nil) var errCacheMiss = errors.New("not in cache") // cacheKey returns the cleaned absolute path of joining pwd and filename. func cacheKey(pwd, filename string) string { if filepath.IsAbs(filename) { return filepath.Clean(filename) } return filepath.Join(pwd, filename) } func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) (*client.PutResult, error) { c.mu.RLock() defer c.mu.RUnlock() fp := fileInfoToFingerprint(fi) key := cacheKey(pwd, filename) val, ok := c.m[key] if !ok { cachelog.Printf("cache MISS on %q: not in cache", key) return nil, errCacheMiss } if val.Fingerprint != fp { cachelog.Printf("cache MISS on %q: stats not equal:\n%#v\n%#v", key, val.Fingerprint, fp) return nil, errCacheMiss } pr := val.Result return &pr, nil } func (c *FlatStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult) { c.mu.Lock() defer c.mu.Unlock() key := cacheKey(pwd, filename) val := fileInfoPutRes{fileInfoToFingerprint(fi), *pr} cachelog.Printf("Adding to stat cache %q: %v", key, val) c.m[key] = val if c.af == nil { var err error c.af, err = os.OpenFile(c.filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { log.Printf("opening stat cache for append: %v", err) return } } // TODO: flocking. see leveldb-go. c.af.Seek(0, os.SEEK_END) c.af.Write([]byte(fmt.Sprintf("%s\t%s\t%s/%d\n", key, val.Fingerprint, val.Result.BlobRef.String(), val.Result.Size))) } type FlatHaveCache struct { mu sync.RWMutex filename string m map[string]int64 // blobref string -> size af *os.File // appending file } func NewFlatHaveCache(gen string) *FlatHaveCache { filename := filepath.Join(osutil.CacheDir(), "camput.havecache."+escapeGen(gen)) c := &FlatHaveCache{ filename: filename, m: make(map[string]int64), } f, err := os.Open(filename) if os.IsNotExist(err) { return c } if err != nil { log.Fatalf("opening camput have-cache: %v", filename, err) } br := bufio.NewReader(f) for { ln, err := br.ReadString('\n') if err == io.EOF { break } if err != nil { log.Printf("Warning: (ignoring) reading have-cache: %v", err) break } f := strings.Fields(strings.TrimSpace(ln)) if len(f) == 2 { br, sizea := f[0], f[1] if size, err := strconv.ParseInt(sizea, 10, 64); err == nil && size >= 0 { c.m[br] = size } } } return c } func (c *FlatHaveCache) StatBlobCache(br *blobref.BlobRef) (size int64, ok bool) { c.mu.RLock() defer c.mu.RUnlock() size, ok = c.m[br.String()] return } func (c *FlatHaveCache) NoteBlobExists(br *blobref.BlobRef, size int64) { c.mu.Lock() defer c.mu.Unlock() if size < 0 { panic("negative size") } k := br.String() if c.m[k] == size { // dup return } c.m[k] = size if c.af == nil { var err error c.af, err = os.OpenFile(c.filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { log.Printf("opening have-cache for append: %v", err) return } } // TODO: flocking. see leveldb-go. c.af.Seek(0, os.SEEK_END) c.af.Write([]byte(fmt.Sprintf("%s %d\n", k, size))) }