Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

force upper bound when rollsum splitting 

Change-Id: I4ec43cf42049060ccab7bbb3f8c598895d4c52d9
This commit is contained in:
mpl 2011-10-31 01:17:34 +01:00
parent 95a8780e42
commit e9650edc63
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/go/camli/schema/filewriter.go
View File

@ -31,6 +31,8 @@ import (
"camli/rollsum" "camli/rollsum"
) )
const MaxBlobSize = 1000000
var _ = log.Printf var _ = log.Printf
// WriteFileFromReader creates and uploads a "file" JSON schema // WriteFileFromReader creates and uploads a "file" JSON schema
@ -139,6 +141,7 @@ func WriteFileFromReaderRolling(bs blobserver.StatReceiver, filename string, r i
} }
func WriteFileMapRolling(bs blobserver.StatReceiver, fileMap map[string]interface{}, r io.Reader) (outbr *blobref.BlobRef, outerr os.Error) { func WriteFileMapRolling(bs blobserver.StatReceiver, fileMap map[string]interface{}, r io.Reader) (outbr *blobref.BlobRef, outerr os.Error) {
blobSize := 0
bufr := bufio.NewReader(r) bufr := bufio.NewReader(r)
spans := []span{} // the tree of spans, cut on interesting rollsum boundaries spans := []span{} // the tree of spans, cut on interesting rollsum boundaries
rs := rollsum.New() rs := rollsum.New()
@ -191,10 +194,14 @@ func WriteFileMapRolling(bs blobserver.StatReceiver, fileMap map[string]interfac
buf.WriteByte(c) buf.WriteByte(c)
n++ n++
blobSize++
rs.Roll(c) rs.Roll(c)
if !rs.OnSplit() { if !rs.OnSplit() {
continue if blobSize < MaxBlobSize {
continue
}
} }
blobSize = 0
bits := rs.Bits() bits := rs.Bits()
// Take any spans from the end of the spans slice that // Take any spans from the end of the spans slice that