2011-11-02 01:37:27 +00:00
/ *
Copyright 2011 Google Inc .
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
package index
import (
2013-12-04 19:58:59 +00:00
"bytes"
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
"errors"
2012-10-08 14:16:00 +00:00
"fmt"
2013-08-27 02:07:28 +00:00
"io"
2011-11-28 03:29:23 +00:00
"log"
2011-11-02 01:37:27 +00:00
"os"
2013-11-26 17:07:39 +00:00
"sort"
2011-11-29 19:40:15 +00:00
"strconv"
2011-11-28 03:29:23 +00:00
"strings"
2013-02-19 05:31:41 +00:00
"sync"
2011-11-02 01:37:27 +00:00
"time"
2013-08-04 02:54:30 +00:00
"camlistore.org/pkg/blob"
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
"camlistore.org/pkg/blobserver"
2013-12-03 04:01:37 +00:00
"camlistore.org/pkg/context"
2013-11-26 17:07:39 +00:00
"camlistore.org/pkg/schema"
2013-11-23 07:24:54 +00:00
"camlistore.org/pkg/sorted"
2013-12-04 05:52:00 +00:00
"camlistore.org/pkg/strutil"
2013-02-19 05:31:41 +00:00
"camlistore.org/pkg/types"
2013-11-16 23:00:30 +00:00
"camlistore.org/pkg/types/camtypes"
2011-11-02 01:37:27 +00:00
)
type Index struct {
2011-11-07 16:40:31 +00:00
* blobserver . NoImplStorage
2013-11-23 07:24:54 +00:00
s sorted . KeyValue
2011-11-07 16:40:31 +00:00
2013-08-04 02:54:30 +00:00
KeyFetcher blob . StreamingFetcher // for verifying claims
2011-11-07 16:40:31 +00:00
2013-12-24 03:07:17 +00:00
// BlobSource is used for fetching blobs when indexing files and other
// blobs types that reference other objects.
BlobSource blobserver . FetcherEnumerator
2013-11-04 22:15:24 +00:00
// deletes is a cache to keep track of the deletion status (deleted vs undeleted)
// of the blobs in the index. It makes for faster reads than the otherwise
// recursive calls on the index.
deletes * deletionCache
2013-11-16 20:55:09 +00:00
corpus * Corpus // or nil, if not being kept in memory
2011-11-02 01:37:27 +00:00
}
2013-11-19 23:28:13 +00:00
var (
_ blobserver . Storage = ( * Index ) ( nil )
_ Interface = ( * Index ) ( nil )
)
2011-11-02 01:37:27 +00:00
2013-12-25 17:49:51 +00:00
var aboutToReindex = false
// SetImpendingReindex notes that the user ran the camlistored binary with the --reindex flag.
// Because the index is about to be wiped, schema version checks should be suppressed.
func SetImpendingReindex ( ) {
// TODO: remove this function, once we refactor how indexes are created.
// They'll probably not all have their own storage constructor registered.
aboutToReindex = true
}
2013-11-19 23:28:13 +00:00
// New returns a new index using the provided key/value storage implementation.
2013-11-23 07:24:54 +00:00
func New ( s sorted . KeyValue ) * Index {
2013-11-04 22:15:24 +00:00
idx := & Index { s : s }
2013-12-25 17:49:51 +00:00
if aboutToReindex {
idx . deletes = & deletionCache {
m : make ( map [ blob . Ref ] [ ] deletion ) ,
}
return idx
}
2013-11-04 22:15:24 +00:00
schemaVersion := idx . schemaVersion ( )
2013-11-28 05:31:05 +00:00
switch {
case schemaVersion == 0 && idx . isEmpty ( ) :
// New index.
2013-11-04 22:15:24 +00:00
err := idx . s . Set ( keySchemaVersion . name , fmt . Sprintf ( "%d" , requiredSchemaVersion ) )
if err != nil {
2013-11-26 17:07:39 +00:00
panic ( fmt . Sprintf ( "Could not write index schema version %q: %v" , requiredSchemaVersion , err ) )
2013-11-04 22:15:24 +00:00
}
2013-11-28 05:31:05 +00:00
case schemaVersion != requiredSchemaVersion :
tip := ""
if os . Getenv ( "CAMLI_DEV_CAMLI_ROOT" ) != "" {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
tip = ` (For the dev server, run "devcam server --wipe" to wipe both your blobs and index) `
} else {
2013-12-24 23:46:17 +00:00
tip = "Run 'camlistored --reindex' (it might take awhile, but shows status). Alternative: 'camtool dbinit' (or just delete the file for a file based index), and then 'camtool sync --all'"
2013-11-28 05:31:05 +00:00
}
log . Fatalf ( "index schema version is %d; required one is %d. You need to reindex. %s" ,
schemaVersion , requiredSchemaVersion , tip )
2013-11-04 22:15:24 +00:00
}
if err := idx . initDeletesCache ( ) ; err != nil {
2013-11-26 17:07:39 +00:00
panic ( fmt . Sprintf ( "Could not initialize index's deletes cache: %v" , err ) )
2013-11-04 22:15:24 +00:00
}
return idx
2011-11-02 01:37:27 +00:00
}
2013-11-28 05:31:05 +00:00
func ( x * Index ) isEmpty ( ) bool {
2013-12-07 16:43:18 +00:00
iter := x . s . Find ( "" , "" )
2013-11-28 05:31:05 +00:00
hasRows := iter . Next ( )
if err := iter . Close ( ) ; err != nil {
panic ( err )
}
return ! hasRows
}
2013-12-24 03:07:17 +00:00
func ( x * Index ) Reindex ( ) error {
ctx := context . TODO ( )
wiper , ok := x . s . ( sorted . Wiper )
if ! ok {
return fmt . Errorf ( "index's storage type %T doesn't support sorted.Wiper" , x . s )
}
log . Printf ( "Wiping index storage type %T ..." , x . s )
if err := wiper . Wipe ( ) ; err != nil {
return fmt . Errorf ( "error wiping index's sorted key/value type %T: %v" , x . s , err )
}
log . Printf ( "Index wiped. Rebuilding..." )
2013-12-24 06:02:30 +00:00
reindexStart , _ := blob . Parse ( os . Getenv ( "CAMLI_REINDEX_START" ) )
2013-12-24 03:07:17 +00:00
err := x . s . Set ( keySchemaVersion . name , fmt . Sprintf ( "%d" , requiredSchemaVersion ) )
if err != nil {
return err
}
var nerrmu sync . Mutex
nerr := 0
blobc := make ( chan blob . Ref , 32 )
enumCtx := ctx . New ( )
enumErr := make ( chan error , 1 )
go func ( ) {
2014-01-09 04:36:17 +00:00
defer close ( blobc )
2013-12-24 03:07:17 +00:00
donec := enumCtx . Done ( )
var lastTick time . Time
enumErr <- blobserver . EnumerateAll ( enumCtx , x . BlobSource , func ( sb blob . SizedRef ) error {
now := time . Now ( )
if lastTick . Before ( now . Add ( - 1 * time . Second ) ) {
log . Printf ( "Reindexing at %v" , sb . Ref )
lastTick = now
}
2013-12-24 06:02:30 +00:00
if reindexStart . Valid ( ) && sb . Ref . Less ( reindexStart ) {
return nil
}
2013-12-24 03:07:17 +00:00
select {
case <- donec :
return context . ErrCanceled
case blobc <- sb . Ref :
return nil
}
} )
} ( )
const j = 4 // arbitrary concurrency level
2014-01-09 04:36:17 +00:00
var wg sync . WaitGroup
2013-12-24 03:07:17 +00:00
for i := 0 ; i < j ; i ++ {
2014-01-09 04:36:17 +00:00
wg . Add ( 1 )
2013-12-24 03:07:17 +00:00
go func ( ) {
2014-01-09 04:36:17 +00:00
defer wg . Done ( )
2013-12-24 03:07:17 +00:00
for br := range blobc {
if err := x . reindex ( br ) ; err != nil {
log . Printf ( "Error reindexing %v: %v" , br , err )
nerrmu . Lock ( )
nerr ++
nerrmu . Unlock ( )
// TODO: flag (or default?) to stop the EnumerateAll above once
// there's any error with reindexing?
}
}
} ( )
}
if err := <- enumErr ; err != nil {
return err
}
2014-01-09 04:36:17 +00:00
wg . Wait ( )
2013-12-25 17:49:51 +00:00
log . Printf ( "Index rebuild complete." )
2013-12-24 03:07:17 +00:00
nerrmu . Lock ( ) // no need to unlock
if nerr != 0 {
return fmt . Errorf ( "%d blobs failed to re-index" , nerr )
}
2013-12-25 17:49:51 +00:00
if err := x . initDeletesCache ( ) ; err != nil {
return err
}
2013-12-24 03:07:17 +00:00
return nil
}
2013-12-07 16:43:18 +00:00
func queryPrefixString ( s sorted . KeyValue , prefix string ) sorted . Iterator {
if prefix == "" {
return s . Find ( "" , "" )
2011-11-29 22:33:10 +00:00
}
2013-12-07 16:43:18 +00:00
lastByte := prefix [ len ( prefix ) - 1 ]
if lastByte == 0xff {
panic ( "unsupported query prefix ending in 0xff" )
2011-11-29 22:33:10 +00:00
}
2013-12-07 16:43:18 +00:00
end := prefix [ : len ( prefix ) - 1 ] + string ( lastByte + 1 )
return s . Find ( prefix , end )
2011-11-29 22:33:10 +00:00
}
2013-12-07 16:43:18 +00:00
func ( x * Index ) queryPrefixString ( prefix string ) sorted . Iterator {
2013-11-17 01:24:02 +00:00
return queryPrefixString ( x . s , prefix )
}
2013-12-07 16:43:18 +00:00
func queryPrefix ( s sorted . KeyValue , key * keyType , args ... interface { } ) sorted . Iterator {
2013-11-17 01:24:02 +00:00
return queryPrefixString ( s , key . Prefix ( args ... ) )
}
2013-12-07 16:43:18 +00:00
func ( x * Index ) queryPrefix ( key * keyType , args ... interface { } ) sorted . Iterator {
2013-11-17 01:24:02 +00:00
return x . queryPrefixString ( key . Prefix ( args ... ) )
}
2013-11-23 07:24:54 +00:00
func closeIterator ( it sorted . Iterator , perr * error ) {
2011-12-03 19:26:42 +00:00
err := it . Close ( )
if err != nil && * perr == nil {
* perr = err
}
}
2013-11-04 22:15:24 +00:00
// schemaVersion returns the version of schema as it is found
// in the currently used index. If not found, it returns 0.
func ( x * Index ) schemaVersion ( ) int {
schemaVersionStr , err := x . s . Get ( keySchemaVersion . name )
if err != nil {
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2013-11-04 22:15:24 +00:00
return 0
}
2013-11-26 17:07:39 +00:00
panic ( fmt . Sprintf ( "Could not get index schema version: %v" , err ) )
2013-11-04 22:15:24 +00:00
}
schemaVersion , err := strconv . Atoi ( schemaVersionStr )
if err != nil {
2013-11-26 17:07:39 +00:00
panic ( fmt . Sprintf ( "Bogus index schema version: %q" , schemaVersionStr ) )
2013-11-04 22:15:24 +00:00
}
return schemaVersion
}
2013-11-26 17:07:39 +00:00
type deletion struct {
deleter blob . Ref
when time . Time
2013-11-04 22:15:24 +00:00
}
2013-11-26 17:07:39 +00:00
type byDeletionDate [ ] deletion
func ( d byDeletionDate ) Len ( ) int { return len ( d ) }
func ( d byDeletionDate ) Swap ( i , j int ) { d [ i ] , d [ j ] = d [ j ] , d [ i ] }
func ( d byDeletionDate ) Less ( i , j int ) bool { return d [ i ] . when . Before ( d [ j ] . when ) }
2013-11-04 22:15:24 +00:00
type deletionCache struct {
sync . RWMutex
2013-11-26 17:07:39 +00:00
m map [ blob . Ref ] [ ] deletion
2013-11-04 22:15:24 +00:00
}
// initDeletesCache creates and populates the deletion status cache used by the index
// for faster calls to IsDeleted and DeletedAt. It is called by New.
func ( x * Index ) initDeletesCache ( ) error {
x . deletes = & deletionCache {
2013-11-26 17:07:39 +00:00
m : make ( map [ blob . Ref ] [ ] deletion ) ,
2013-11-04 22:15:24 +00:00
}
2012-10-08 14:16:00 +00:00
var err error
2013-11-04 22:15:24 +00:00
it := x . queryPrefix ( keyDeleted )
2012-10-08 14:16:00 +00:00
defer closeIterator ( it , & err )
for it . Next ( ) {
2013-11-26 17:07:39 +00:00
cl , ok := kvDeleted ( it . Key ( ) )
2013-11-04 22:15:24 +00:00
if ! ok {
2013-11-26 17:07:39 +00:00
return fmt . Errorf ( "Bogus keyDeleted entry key: want |\"deleted\"|<deleted blobref>|<reverse claimdate>|<deleter claim>|, got %q" , it . Key ( ) )
2013-11-04 22:15:24 +00:00
}
2013-11-26 17:07:39 +00:00
targetDeletions := append ( x . deletes . m [ cl . Target ] ,
deletion {
deleter : cl . BlobRef ,
when : cl . Date ,
} )
sort . Sort ( sort . Reverse ( byDeletionDate ( targetDeletions ) ) )
x . deletes . m [ cl . Target ] = targetDeletions
2013-11-04 22:15:24 +00:00
}
return err
}
2013-11-26 17:07:39 +00:00
func kvDeleted ( k string ) ( c camtypes . Claim , ok bool ) {
// TODO(bradfitz): garbage
keyPart := strings . Split ( k , "|" )
if len ( keyPart ) != 4 {
return
2013-11-04 22:15:24 +00:00
}
2013-11-26 17:07:39 +00:00
if keyPart [ 0 ] != "deleted" {
return
2013-11-04 22:15:24 +00:00
}
2013-11-26 17:07:39 +00:00
target , ok := blob . Parse ( keyPart [ 1 ] )
if ! ok {
return
2013-11-04 22:15:24 +00:00
}
2013-11-26 17:07:39 +00:00
claimRef , ok := blob . Parse ( keyPart [ 3 ] )
if ! ok {
return
}
date , err := time . Parse ( time . RFC3339 , unreverseTimeString ( keyPart [ 2 ] ) )
if err != nil {
return
}
return camtypes . Claim {
BlobRef : claimRef ,
Target : target ,
Date : date ,
Type : string ( schema . DeleteClaim ) ,
} , true
2013-11-04 22:15:24 +00:00
}
2013-11-10 00:20:21 +00:00
// IsDeleted reports whether the provided blobref (of a permanode or
// claim) should be considered deleted.
2013-11-04 22:15:24 +00:00
func ( x * Index ) IsDeleted ( br blob . Ref ) bool {
if x . deletes == nil {
// We still allow the slow path, in case someone creates
// their own Index without a deletes cache.
return x . isDeletedNoCache ( br )
}
x . deletes . RLock ( )
defer x . deletes . RUnlock ( )
2013-11-26 17:07:39 +00:00
return x . isDeleted ( br )
2013-11-04 22:15:24 +00:00
}
2013-11-26 17:07:39 +00:00
// The caller must hold x.deletes.mu for read.
func ( x * Index ) isDeleted ( br blob . Ref ) bool {
deletes , ok := x . deletes . m [ br ]
if ! ok {
return false
}
for _ , v := range deletes {
if ! x . isDeleted ( v . deleter ) {
return true
}
}
return false
}
// Used when the Index has no deletes cache (x.deletes is nil).
2013-11-04 22:15:24 +00:00
func ( x * Index ) isDeletedNoCache ( br blob . Ref ) bool {
var err error
it := x . queryPrefix ( keyDeleted , br )
2013-11-26 17:07:39 +00:00
for it . Next ( ) {
cl , ok := kvDeleted ( it . Key ( ) )
2013-08-04 02:54:30 +00:00
if ! ok {
2013-11-26 17:07:39 +00:00
panic ( fmt . Sprintf ( "Bogus keyDeleted entry key: want |\"deleted\"|<deleted blobref>|<reverse claimdate>|<deleter claim>|, got %q" , it . Key ( ) ) )
2012-10-08 14:16:00 +00:00
}
2013-11-26 17:07:39 +00:00
if ! x . isDeletedNoCache ( cl . BlobRef ) {
closeIterator ( it , & err )
if err != nil {
// TODO: Do better?
panic ( fmt . Sprintf ( "Could not close iterator on keyDeleted: %v" , err ) )
}
return true
2013-11-04 22:15:24 +00:00
}
}
closeIterator ( it , & err )
if err != nil {
// TODO: Do better?
2013-11-26 17:07:39 +00:00
panic ( fmt . Sprintf ( "Could not close iterator on keyDeleted: %v" , err ) )
2012-10-08 14:16:00 +00:00
}
return false
}
2013-09-15 02:48:52 +00:00
// GetRecentPermanodes sends results to dest filtered by owner, limit, and
// before. A zero value for before will default to the current time. The
// results will have duplicates supressed, with most recent permanode
// returned.
// Note, permanodes more recent than before will still be fetched from the
// index then skipped. This means runtime scales linearly with the number of
// nodes more recent than before.
2013-11-26 04:35:59 +00:00
func ( x * Index ) GetRecentPermanodes ( dest chan <- camtypes . RecentPermanode , owner blob . Ref , limit int , before time . Time ) ( err error ) {
2011-11-02 01:37:27 +00:00
defer close ( dest )
2011-11-29 19:19:32 +00:00
2013-11-17 17:41:45 +00:00
keyId , err := x . KeyId ( owner )
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2013-01-09 22:21:32 +00:00
log . Printf ( "No recent permanodes because keyId for owner %v not found" , owner )
2011-11-29 19:19:32 +00:00
return nil
}
if err != nil {
2013-01-09 22:21:32 +00:00
log . Printf ( "Error fetching keyId for owner %v: %v" , owner , err )
2011-11-29 19:19:32 +00:00
return err
}
sent := 0
var seenPermanode dupSkipper
2011-11-29 22:33:10 +00:00
2013-09-15 02:48:52 +00:00
if before . IsZero ( ) {
before = time . Now ( )
}
2013-11-26 04:35:59 +00:00
// TODO(bradfitz): handle before efficiently. don't use queryPrefix.
2011-11-29 22:33:10 +00:00
it := x . queryPrefix ( keyRecentPermanode , keyId )
2011-12-03 19:26:42 +00:00
defer closeIterator ( it , & err )
2011-11-29 19:19:32 +00:00
for it . Next ( ) {
permaStr := it . Value ( )
parts := strings . SplitN ( it . Key ( ) , "|" , 4 )
if len ( parts ) != 4 {
continue
}
2013-11-16 23:00:30 +00:00
mTime , _ := time . Parse ( time . RFC3339 , unreverseTimeString ( parts [ 2 ] ) )
2013-08-04 02:54:30 +00:00
permaRef , ok := blob . Parse ( permaStr )
if ! ok {
2011-11-29 19:19:32 +00:00
continue
}
2013-11-04 22:15:24 +00:00
if x . IsDeleted ( permaRef ) {
2012-10-08 14:16:00 +00:00
continue
}
2011-11-29 19:19:32 +00:00
if seenPermanode . Dup ( permaStr ) {
continue
}
2013-09-15 02:48:52 +00:00
// Skip entries with an mTime less than or equal to before.
if ! mTime . Before ( before ) {
2013-11-26 04:35:59 +00:00
continue
}
2013-11-16 23:00:30 +00:00
dest <- camtypes . RecentPermanode {
Permanode : permaRef ,
2011-11-29 19:19:32 +00:00
Signer : owner , // TODO(bradfitz): kinda. usually. for now.
2013-11-16 23:00:30 +00:00
LastModTime : mTime ,
2011-11-29 19:19:32 +00:00
}
sent ++
if sent == limit {
break
}
}
return nil
2011-11-02 01:37:27 +00:00
}
2013-11-17 22:54:30 +00:00
func ( x * Index ) AppendClaims ( dst [ ] camtypes . Claim , permaNode blob . Ref ,
signerFilter blob . Ref ,
attrFilter string ) ( [ ] camtypes . Claim , error ) {
2013-11-18 00:52:51 +00:00
if x . corpus != nil {
return x . corpus . AppendClaims ( dst , permaNode , signerFilter , attrFilter )
}
2013-11-17 22:54:30 +00:00
var (
keyId string
err error
2013-12-07 16:43:18 +00:00
it sorted . Iterator
2013-11-17 22:54:30 +00:00
)
if signerFilter . Valid ( ) {
keyId , err = x . KeyId ( signerFilter )
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2013-11-17 22:54:30 +00:00
return nil , nil
}
if err != nil {
return nil , err
}
it = x . queryPrefix ( keyPermanodeClaim , permaNode , keyId )
} else {
it = x . queryPrefix ( keyPermanodeClaim , permaNode )
2011-11-29 20:40:33 +00:00
}
2011-12-03 19:26:42 +00:00
defer closeIterator ( it , & err )
2013-11-17 22:54:30 +00:00
// In the common case, an attribute filter is just a plain
// token ("camliContent") unescaped. If so, fast path that
// check to skip the row before we even split it.
var mustHave string
if attrFilter != "" && urle ( attrFilter ) == attrFilter {
mustHave = attrFilter
}
2011-11-29 20:40:33 +00:00
for it . Next ( ) {
2013-11-17 22:54:30 +00:00
val := it . Value ( )
if mustHave != "" && ! strings . Contains ( val , mustHave ) {
continue
}
2013-11-30 20:04:04 +00:00
cl , ok := kvClaim ( it . Key ( ) , val , blob . Parse )
2013-11-18 00:52:51 +00:00
if ! ok {
2011-11-29 20:40:33 +00:00
continue
}
2013-11-27 22:20:28 +00:00
if x . IsDeleted ( cl . BlobRef ) {
continue
}
2013-11-18 00:52:51 +00:00
if attrFilter != "" && cl . Attr != attrFilter {
2013-11-17 22:54:30 +00:00
continue
}
2013-11-18 00:52:51 +00:00
if signerFilter . Valid ( ) && cl . Signer != signerFilter {
2011-11-29 20:40:33 +00:00
continue
}
2013-11-18 00:52:51 +00:00
dst = append ( dst , cl )
2011-11-29 20:40:33 +00:00
}
2013-11-17 22:54:30 +00:00
return dst , nil
2011-11-02 01:37:27 +00:00
}
2013-11-30 20:04:04 +00:00
func kvClaim ( k , v string , blobParse func ( string ) ( blob . Ref , bool ) ) ( c camtypes . Claim , ok bool ) {
2013-12-04 05:52:00 +00:00
const nKeyPart = 5
const nValPart = 4
var keya [ nKeyPart ] string
var vala [ nValPart ] string
keyPart := strutil . AppendSplitN ( keya [ : 0 ] , k , "|" , - 1 )
valPart := strutil . AppendSplitN ( vala [ : 0 ] , v , "|" , - 1 )
if len ( keyPart ) < nKeyPart || len ( valPart ) < nValPart {
2013-11-18 00:52:51 +00:00
return
}
2013-11-30 20:04:04 +00:00
signerRef , ok := blobParse ( valPart [ 3 ] )
2013-11-18 00:52:51 +00:00
if ! ok {
return
}
2013-11-30 20:04:04 +00:00
permaNode , ok := blobParse ( keyPart [ 1 ] )
2013-11-18 00:52:51 +00:00
if ! ok {
return
}
2013-11-30 20:04:04 +00:00
claimRef , ok := blobParse ( keyPart [ 4 ] )
2013-11-18 00:52:51 +00:00
if ! ok {
return
}
date , err := time . Parse ( time . RFC3339 , keyPart [ 3 ] )
if err != nil {
return
}
return camtypes . Claim {
BlobRef : claimRef ,
Signer : signerRef ,
Permanode : permaNode ,
Date : date ,
Type : urld ( valPart [ 0 ] ) ,
Attr : urld ( valPart [ 1 ] ) ,
Value : urld ( valPart [ 2 ] ) ,
} , true
}
2013-11-17 03:40:14 +00:00
func ( x * Index ) GetBlobMeta ( br blob . Ref ) ( camtypes . BlobMeta , error ) {
if x . corpus != nil {
return x . corpus . GetBlobMeta ( br )
}
key := "meta:" + br . String ( )
2013-01-09 03:43:09 +00:00
meta , err := x . s . Get ( key )
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2012-02-20 12:32:46 +00:00
err = os . ErrNotExist
2011-11-29 19:40:15 +00:00
}
if err != nil {
2013-11-17 03:40:14 +00:00
return camtypes . BlobMeta { } , err
2011-11-29 19:40:15 +00:00
}
pos := strings . Index ( meta , "|" )
2013-01-09 03:43:09 +00:00
if pos < 0 {
panic ( fmt . Sprintf ( "Bogus index row for key %q: got value %q" , key , meta ) )
}
2013-11-29 21:41:39 +00:00
size , err := strconv . ParseUint ( meta [ : pos ] , 10 , 32 )
if err != nil {
return camtypes . BlobMeta { } , err
}
2013-11-17 03:40:14 +00:00
mime := meta [ pos + 1 : ]
return camtypes . BlobMeta {
Ref : br ,
2013-11-29 21:41:39 +00:00
Size : uint32 ( size ) ,
2013-11-17 03:40:14 +00:00
CamliType : camliTypeFromMIME ( mime ) ,
} , nil
2011-11-02 01:37:27 +00:00
}
2013-11-17 17:41:45 +00:00
func ( x * Index ) KeyId ( signer blob . Ref ) ( string , error ) {
if x . corpus != nil {
return x . corpus . KeyId ( signer )
}
2011-11-28 03:29:23 +00:00
return x . s . Get ( "signerkeyid:" + signer . String ( ) )
}
2013-08-04 02:54:30 +00:00
func ( x * Index ) PermanodeOfSignerAttrValue ( signer blob . Ref , attr , val string ) ( permaNode blob . Ref , err error ) {
2013-11-17 17:41:45 +00:00
keyId , err := x . KeyId ( signer )
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2013-08-04 02:54:30 +00:00
return blob . Ref { } , os . ErrNotExist
2011-11-28 03:29:23 +00:00
}
2011-11-29 19:19:32 +00:00
if err != nil {
2013-08-04 02:54:30 +00:00
return blob . Ref { } , err
2011-11-29 19:19:32 +00:00
}
2011-12-04 22:47:05 +00:00
it := x . queryPrefix ( keySignerAttrValue , keyId , attr , val )
2011-12-03 19:26:42 +00:00
defer closeIterator ( it , & err )
2013-11-27 22:20:28 +00:00
for it . Next ( ) {
2013-08-04 02:54:30 +00:00
permaRef , ok := blob . Parse ( it . Value ( ) )
2013-11-04 22:15:24 +00:00
if ok && ! x . IsDeleted ( permaRef ) {
2012-10-08 14:16:00 +00:00
return permaRef , nil
}
2011-11-28 03:29:23 +00:00
}
2013-08-04 02:54:30 +00:00
return blob . Ref { } , os . ErrNotExist
2011-11-02 01:37:27 +00:00
}
2011-12-04 22:47:05 +00:00
// This is just like PermanodeOfSignerAttrValue except we return multiple and dup-suppress.
2012-11-13 23:02:12 +00:00
// If request.Query is "", it is not used in the prefix search.
2013-11-16 23:00:30 +00:00
func ( x * Index ) SearchPermanodesWithAttr ( dest chan <- blob . Ref , request * camtypes . PermanodeByAttrRequest ) ( err error ) {
2011-12-04 22:47:05 +00:00
defer close ( dest )
if request . FuzzyMatch {
// TODO(bradfitz): remove this for now? figure out how to handle it generically?
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
return errors . New ( "TODO: SearchPermanodesWithAttr: generic indexer doesn't support FuzzyMatch on PermanodeByAttrRequest" )
2011-12-04 22:47:05 +00:00
}
if request . Attribute == "" {
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
return errors . New ( "index: missing Attribute in SearchPermanodesWithAttr" )
2011-12-04 22:47:05 +00:00
}
2013-11-17 17:41:45 +00:00
keyId , err := x . KeyId ( request . Signer )
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2011-12-04 22:47:05 +00:00
return nil
}
if err != nil {
return err
}
seen := make ( map [ string ] bool )
2013-12-07 16:43:18 +00:00
var it sorted . Iterator
2012-11-13 23:02:12 +00:00
if request . Query == "" {
it = x . queryPrefix ( keySignerAttrValue , keyId , request . Attribute )
} else {
it = x . queryPrefix ( keySignerAttrValue , keyId , request . Attribute , request . Query )
}
2011-12-04 22:47:05 +00:00
defer closeIterator ( it , & err )
for it . Next ( ) {
2013-11-27 22:20:28 +00:00
cl , ok := kvSignerAttrValue ( it . Key ( ) , it . Value ( ) )
2013-08-04 02:54:30 +00:00
if ! ok {
2011-12-04 22:47:05 +00:00
continue
}
2013-11-27 22:20:28 +00:00
if x . IsDeleted ( cl . BlobRef ) {
continue
}
if x . IsDeleted ( cl . Permanode ) {
2012-10-08 14:16:00 +00:00
continue
}
2013-11-27 22:20:28 +00:00
pnstr := cl . Permanode . String ( )
2011-12-04 22:47:05 +00:00
if seen [ pnstr ] {
continue
}
seen [ pnstr ] = true
2013-11-27 22:20:28 +00:00
dest <- cl . Permanode
2011-12-04 22:47:05 +00:00
if len ( seen ) == request . MaxResults {
break
}
}
return nil
}
2013-11-27 22:20:28 +00:00
func kvSignerAttrValue ( k , v string ) ( c camtypes . Claim , ok bool ) {
// TODO(bradfitz): garbage
keyPart := strings . Split ( k , "|" )
valPart := strings . Split ( v , "|" )
if len ( keyPart ) != 6 || len ( valPart ) != 1 {
// TODO(mpl): use glog
log . Printf ( "bogus keySignerAttrValue index entry: %q = %q" , k , v )
return
}
if keyPart [ 0 ] != "signerattrvalue" {
return
}
date , err := time . Parse ( time . RFC3339 , unreverseTimeString ( keyPart [ 4 ] ) )
if err != nil {
log . Printf ( "bogus time in keySignerAttrValue index entry: %q" , keyPart [ 4 ] )
return
}
claimRef , ok := blob . Parse ( keyPart [ 5 ] )
if ! ok {
log . Printf ( "bogus claim in keySignerAttrValue index entry: %q" , keyPart [ 5 ] )
return
}
permaNode , ok := blob . Parse ( valPart [ 0 ] )
if ! ok {
log . Printf ( "bogus permanode in keySignerAttrValue index entry: %q" , valPart [ 0 ] )
return
}
return camtypes . Claim {
BlobRef : claimRef ,
Permanode : permaNode ,
Date : date ,
Attr : urld ( keyPart [ 2 ] ) ,
Value : urld ( keyPart [ 3 ] ) ,
} , true
}
2013-11-16 23:00:30 +00:00
func ( x * Index ) PathsOfSignerTarget ( signer , target blob . Ref ) ( paths [ ] * camtypes . Path , err error ) {
paths = [ ] * camtypes . Path { }
2013-11-17 17:41:45 +00:00
keyId , err := x . KeyId ( signer )
2011-12-01 18:43:57 +00:00
if err != nil {
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2011-12-01 18:43:57 +00:00
err = nil
}
return
}
2013-11-16 23:00:30 +00:00
mostRecent := make ( map [ string ] * camtypes . Path )
2013-11-27 22:20:28 +00:00
maxClaimDates := make ( map [ string ] time . Time )
2011-12-01 18:43:57 +00:00
2011-12-02 02:06:25 +00:00
it := x . queryPrefix ( keyPathBackward , keyId , target )
2011-12-03 19:26:42 +00:00
defer closeIterator ( it , & err )
2011-12-01 18:43:57 +00:00
for it . Next ( ) {
2013-11-27 22:20:28 +00:00
p , ok , active := kvPathBackward ( it . Key ( ) , it . Value ( ) )
if ! ok {
2011-12-01 18:43:57 +00:00
continue
}
2013-11-27 22:20:28 +00:00
if x . IsDeleted ( p . Claim ) {
2013-08-04 02:54:30 +00:00
continue
}
2013-11-27 22:20:28 +00:00
if x . IsDeleted ( p . Base ) {
2011-12-01 18:43:57 +00:00
continue
}
2013-11-27 22:20:28 +00:00
key := p . Base . String ( ) + "/" + p . Suffix
if p . ClaimDate . After ( maxClaimDates [ key ] ) {
maxClaimDates [ key ] = p . ClaimDate
if active {
mostRecent [ key ] = & p
2011-12-01 18:43:57 +00:00
} else {
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
delete ( mostRecent , key )
2011-12-01 18:43:57 +00:00
}
}
}
for _ , v := range mostRecent {
paths = append ( paths , v )
}
return paths , nil
2011-11-02 01:37:27 +00:00
}
2013-11-27 22:20:28 +00:00
func kvPathBackward ( k , v string ) ( p camtypes . Path , ok bool , active bool ) {
// TODO(bradfitz): garbage
keyPart := strings . Split ( k , "|" )
valPart := strings . Split ( v , "|" )
if len ( keyPart ) != 4 || len ( valPart ) != 4 {
// TODO(mpl): use glog
log . Printf ( "bogus keyPathBackward index entry: %q = %q" , k , v )
return
}
if keyPart [ 0 ] != "signertargetpath" {
return
}
target , ok := blob . Parse ( keyPart [ 2 ] )
if ! ok {
log . Printf ( "bogus target in keyPathBackward index entry: %q" , keyPart [ 2 ] )
return
}
claim , ok := blob . Parse ( keyPart [ 3 ] )
if ! ok {
log . Printf ( "bogus claim in keyPathBackward index entry: %q" , keyPart [ 3 ] )
return
}
date , err := time . Parse ( time . RFC3339 , valPart [ 0 ] )
if err != nil {
log . Printf ( "bogus date in keyPathBackward index entry: %q" , valPart [ 0 ] )
return
}
base , ok := blob . Parse ( valPart [ 1 ] )
if ! ok {
log . Printf ( "bogus base in keyPathBackward index entry: %q" , valPart [ 1 ] )
return
}
if valPart [ 2 ] == "Y" {
active = true
}
return camtypes . Path {
Claim : claim ,
Base : base ,
Target : target ,
ClaimDate : date ,
Suffix : urld ( valPart [ 3 ] ) ,
} , true , active
}
2013-11-16 23:00:30 +00:00
func ( x * Index ) PathsLookup ( signer , base blob . Ref , suffix string ) ( paths [ ] * camtypes . Path , err error ) {
paths = [ ] * camtypes . Path { }
2013-11-17 17:41:45 +00:00
keyId , err := x . KeyId ( signer )
2011-12-02 02:06:25 +00:00
if err != nil {
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2011-12-02 02:06:25 +00:00
err = nil
}
return
}
it := x . queryPrefix ( keyPathForward , keyId , base , suffix )
2011-12-03 19:26:42 +00:00
defer closeIterator ( it , & err )
2011-12-02 02:06:25 +00:00
for it . Next ( ) {
2013-11-27 22:20:28 +00:00
p , ok , active := kvPathForward ( it . Key ( ) , it . Value ( ) )
2013-08-04 02:54:30 +00:00
if ! ok {
continue
}
2013-11-27 22:20:28 +00:00
if x . IsDeleted ( p . Claim ) {
2011-12-02 02:06:25 +00:00
continue
}
2013-11-27 22:20:28 +00:00
if x . IsDeleted ( p . Target ) {
2013-08-04 02:54:30 +00:00
continue
}
2011-12-02 02:06:25 +00:00
// TODO(bradfitz): investigate what's up with deleted
// forward path claims here. Needs docs with the
// interface too, and tests.
_ = active
2013-11-27 22:20:28 +00:00
paths = append ( paths , & p )
2011-12-02 02:06:25 +00:00
}
return
2011-11-02 01:37:27 +00:00
}
2013-11-27 22:20:28 +00:00
func kvPathForward ( k , v string ) ( p camtypes . Path , ok bool , active bool ) {
// TODO(bradfitz): garbage
keyPart := strings . Split ( k , "|" )
valPart := strings . Split ( v , "|" )
if len ( keyPart ) != 6 || len ( valPart ) != 2 {
// TODO(mpl): use glog
log . Printf ( "bogus keyPathForward index entry: %q = %q" , k , v )
return
}
if keyPart [ 0 ] != "path" {
return
}
base , ok := blob . Parse ( keyPart [ 2 ] )
if ! ok {
log . Printf ( "bogus base in keyPathForward index entry: %q" , keyPart [ 2 ] )
return
}
date , err := time . Parse ( time . RFC3339 , unreverseTimeString ( keyPart [ 4 ] ) )
if err != nil {
log . Printf ( "bogus date in keyPathForward index entry: %q" , keyPart [ 4 ] )
return
}
claim , ok := blob . Parse ( keyPart [ 5 ] )
if ! ok {
log . Printf ( "bogus claim in keyPathForward index entry: %q" , keyPart [ 5 ] )
return
}
if valPart [ 0 ] == "Y" {
active = true
}
target , ok := blob . Parse ( valPart [ 1 ] )
if ! ok {
log . Printf ( "bogus target in keyPathForward index entry: %q" , valPart [ 1 ] )
return
}
return camtypes . Path {
Claim : claim ,
Base : base ,
Target : target ,
ClaimDate : date ,
Suffix : urld ( keyPart [ 3 ] ) ,
} , true , active
}
2013-11-16 23:00:30 +00:00
func ( x * Index ) PathLookup ( signer , base blob . Ref , suffix string , at time . Time ) ( * camtypes . Path , error ) {
2011-12-02 02:06:25 +00:00
paths , err := x . PathsLookup ( signer , base , suffix )
if err != nil {
return nil , err
}
var (
newest = int64 ( 0 )
atSeconds = int64 ( 0 )
2013-11-16 23:00:30 +00:00
best * camtypes . Path
2011-12-02 02:06:25 +00:00
)
2012-02-20 12:32:46 +00:00
if ! at . IsZero ( ) {
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
atSeconds = at . Unix ( )
2011-12-02 02:06:25 +00:00
}
2012-02-20 12:32:46 +00:00
2011-12-02 02:06:25 +00:00
for _ , path := range paths {
2013-11-27 22:20:28 +00:00
t := path . ClaimDate
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
secs := t . Unix ( )
2011-12-02 02:06:25 +00:00
if atSeconds != 0 && secs > atSeconds {
// Too new
continue
}
if newest > secs {
// Too old
continue
}
// Just right
newest , best = secs , path
}
if best == nil {
2012-02-20 12:32:46 +00:00
return nil , os . ErrNotExist
2011-12-02 02:06:25 +00:00
}
return best , nil
}
2013-08-04 02:54:30 +00:00
func ( x * Index ) ExistingFileSchemas ( wholeRef blob . Ref ) ( schemaRefs [ ] blob . Ref , err error ) {
2011-12-03 19:26:42 +00:00
it := x . queryPrefix ( keyWholeToFileRef , wholeRef )
defer closeIterator ( it , & err )
for it . Next ( ) {
keyPart := strings . Split ( it . Key ( ) , "|" ) [ 1 : ]
if len ( keyPart ) < 2 {
continue
}
2013-08-04 02:54:30 +00:00
ref , ok := blob . Parse ( keyPart [ 1 ] )
if ok {
2011-12-03 19:26:42 +00:00
schemaRefs = append ( schemaRefs , ref )
}
}
return schemaRefs , nil
2011-12-03 16:05:34 +00:00
}
2013-02-19 05:31:41 +00:00
func ( x * Index ) loadKey ( key string , val * string , err * error , wg * sync . WaitGroup ) {
defer wg . Done ( )
* val , * err = x . s . Get ( key )
}
2013-11-17 01:49:28 +00:00
func ( x * Index ) GetFileInfo ( fileRef blob . Ref ) ( camtypes . FileInfo , error ) {
2013-11-18 03:49:19 +00:00
if x . corpus != nil {
return x . corpus . GetFileInfo ( fileRef )
}
2013-02-19 05:31:41 +00:00
ikey := "fileinfo|" + fileRef . String ( )
tkey := "filetimes|" + fileRef . String ( )
wg := new ( sync . WaitGroup )
wg . Add ( 2 )
var iv , tv string // info value, time value
var ierr , terr error
go x . loadKey ( ikey , & iv , & ierr , wg )
go x . loadKey ( tkey , & tv , & terr , wg )
wg . Wait ( )
2013-11-23 07:24:54 +00:00
if ierr == sorted . ErrNotFound {
2013-11-17 01:49:28 +00:00
return camtypes . FileInfo { } , os . ErrNotExist
2011-12-03 21:56:05 +00:00
}
2013-02-19 05:31:41 +00:00
if ierr != nil {
2013-11-17 01:49:28 +00:00
return camtypes . FileInfo { } , ierr
2013-02-19 05:31:41 +00:00
}
valPart := strings . Split ( iv , "|" )
2011-12-03 21:56:05 +00:00
if len ( valPart ) < 3 {
2013-02-19 05:31:41 +00:00
log . Printf ( "index: bogus key %q = %q" , ikey , iv )
2013-11-17 01:49:28 +00:00
return camtypes . FileInfo { } , os . ErrNotExist
2011-12-03 21:56:05 +00:00
}
Update from r60 to [almost] Go 1.
A lot is still broken, but most stuff at least compiles now.
The directory tree has been rearranged now too. Go libraries are now
under "pkg". Fully qualified, they are e.g. "camlistore.org/pkg/jsonsign".
The go tool cannot yet fetch from arbitrary domains, but discussion is
happening now on which mechanism to use to allow that.
For now, put the camlistore root under $GOPATH/src. Typically $GOPATH
is $HOME, so Camlistore should be at $HOME/src/camlistore.org.
Then you can:
$ go build ./server/camlistored
... etc
The build.pl script is currently disabled. It'll be resurrected at
some point, but with a very different role (helping create a fake
GOPATH and running the go build command, if things are installed at
the wrong place, and/or running fileembed generators).
Many things are certainly broken.
Many things are disabled. (MySQL, all indexing, etc).
Many things need to be moved into
camlistore.org/third_party/{code.google.com,github.com} and updated
from their r60 to Go 1 versions, where applicable.
The GoMySQL stuff should be updated to use database/sql and the ziutek
library implementing database/sql/driver.
Help wanted.
Change-Id: If71217dc5c8f0e70dbe46e9504ca5131c6eeacde
2012-02-19 05:53:06 +00:00
size , err := strconv . ParseInt ( valPart [ 0 ] , 10 , 64 )
2011-12-03 21:56:05 +00:00
if err != nil {
2013-02-19 05:31:41 +00:00
log . Printf ( "index: bogus integer at position 0 in key %q = %q" , ikey , iv )
2013-11-17 01:49:28 +00:00
return camtypes . FileInfo { } , os . ErrNotExist
2011-12-03 21:56:05 +00:00
}
2013-02-19 05:31:41 +00:00
fileName := urld ( valPart [ 1 ] )
2013-11-17 01:49:28 +00:00
fi := camtypes . FileInfo {
2011-12-03 21:56:05 +00:00
Size : size ,
2013-02-19 05:31:41 +00:00
FileName : fileName ,
2013-02-18 19:16:13 +00:00
MIMEType : urld ( valPart [ 2 ] ) ,
2011-12-03 21:56:05 +00:00
}
2013-02-19 05:31:41 +00:00
if tv != "" {
times := strings . Split ( urld ( tv ) , "," )
2013-11-18 03:49:19 +00:00
updateFileInfoTimes ( & fi , times )
2013-02-19 05:31:41 +00:00
}
2011-12-03 21:56:05 +00:00
return fi , nil
2011-12-03 16:05:34 +00:00
}
2012-02-26 12:49:03 +00:00
2013-11-18 03:49:19 +00:00
func updateFileInfoTimes ( fi * camtypes . FileInfo , times [ ] string ) {
if len ( times ) == 0 {
return
}
2013-12-04 05:56:09 +00:00
fi . Time = types . ParseTime3339OrNil ( times [ 0 ] )
2013-11-18 03:49:19 +00:00
if len ( times ) == 2 {
2013-12-04 05:56:09 +00:00
fi . ModTime = types . ParseTime3339OrNil ( times [ 1 ] )
2013-11-18 03:49:19 +00:00
}
}
2013-11-26 03:46:28 +00:00
// v is "width|height"
2013-12-07 19:08:19 +00:00
func kvImageInfo ( v [ ] byte ) ( ii camtypes . ImageInfo , ok bool ) {
pipei := bytes . IndexByte ( v , '|' )
2013-11-26 03:46:28 +00:00
if pipei < 0 {
return
}
2013-12-07 19:08:19 +00:00
w , err := strutil . ParseUintBytes ( v [ : pipei ] , 10 , 16 )
2013-11-26 03:46:28 +00:00
if err != nil {
return
}
2013-12-07 19:08:19 +00:00
h , err := strutil . ParseUintBytes ( v [ pipei + 1 : ] , 10 , 16 )
2013-11-26 03:46:28 +00:00
if err != nil {
return
}
2013-11-29 19:01:41 +00:00
ii . Width = uint16 ( w )
ii . Height = uint16 ( h )
2013-11-26 03:46:28 +00:00
return ii , true
}
2013-11-17 01:49:28 +00:00
func ( x * Index ) GetImageInfo ( fileRef blob . Ref ) ( camtypes . ImageInfo , error ) {
2013-11-26 03:46:28 +00:00
if x . corpus != nil {
return x . corpus . GetImageInfo ( fileRef )
}
2013-03-06 21:54:14 +00:00
// it might be that the key does not exist because image.DecodeConfig failed earlier
// (because of unsupported JPEG features like progressive mode).
key := keyImageSize . Key ( fileRef . String ( ) )
2013-11-26 03:46:28 +00:00
v , err := x . s . Get ( key )
2013-11-23 07:24:54 +00:00
if err == sorted . ErrNotFound {
2013-03-06 21:54:14 +00:00
err = os . ErrNotExist
}
if err != nil {
2013-11-17 01:49:28 +00:00
return camtypes . ImageInfo { } , err
2013-03-06 21:54:14 +00:00
}
2013-12-07 19:08:19 +00:00
ii , ok := kvImageInfo ( [ ] byte ( v ) )
2013-11-26 03:46:28 +00:00
if ! ok {
return camtypes . ImageInfo { } , fmt . Errorf ( "index: bogus key %q = %q" , key , v )
2013-03-06 21:54:14 +00:00
}
2013-11-26 03:46:28 +00:00
return ii , nil
2013-03-06 21:54:14 +00:00
}
2013-11-16 23:00:30 +00:00
func ( x * Index ) EdgesTo ( ref blob . Ref , opts * camtypes . EdgesToOpts ) ( edges [ ] * camtypes . Edge , err error ) {
2012-11-05 09:29:42 +00:00
it := x . queryPrefix ( keyEdgeBackward , ref )
defer closeIterator ( it , & err )
2013-11-28 16:46:36 +00:00
permanodeParents := make ( map [ string ] * camtypes . Edge )
2012-11-05 09:29:42 +00:00
for it . Next ( ) {
2013-11-28 16:46:36 +00:00
edge , ok := kvEdgeBackward ( it . Key ( ) , it . Value ( ) )
if ! ok {
2012-11-05 09:29:42 +00:00
continue
}
2013-11-28 16:46:36 +00:00
if x . IsDeleted ( edge . From ) {
2012-11-05 09:29:42 +00:00
continue
}
2013-11-28 16:46:36 +00:00
if x . IsDeleted ( edge . BlobRef ) {
2012-11-05 09:29:42 +00:00
continue
}
2013-11-28 16:46:36 +00:00
edge . To = ref
if edge . FromType == "permanode" {
permanodeParents [ edge . From . String ( ) ] = edge
2012-11-05 09:29:42 +00:00
} else {
2013-11-28 16:46:36 +00:00
edges = append ( edges , edge )
2012-11-05 09:29:42 +00:00
}
}
2013-11-28 16:46:36 +00:00
for _ , e := range permanodeParents {
edges = append ( edges , e )
2012-11-05 09:29:42 +00:00
}
return edges , nil
2012-11-03 15:08:37 +00:00
}
2013-11-28 16:46:36 +00:00
func kvEdgeBackward ( k , v string ) ( edge * camtypes . Edge , ok bool ) {
// TODO(bradfitz): garbage
keyPart := strings . Split ( k , "|" )
valPart := strings . Split ( v , "|" )
if len ( keyPart ) != 4 || len ( valPart ) != 2 {
// TODO(mpl): use glog
log . Printf ( "bogus keyEdgeBackward index entry: %q = %q" , k , v )
return
}
if keyPart [ 0 ] != "edgeback" {
return
}
parentRef , ok := blob . Parse ( keyPart [ 2 ] )
if ! ok {
log . Printf ( "bogus parent in keyEdgeBackward index entry: %q" , keyPart [ 2 ] )
return
}
blobRef , ok := blob . Parse ( keyPart [ 3 ] )
if ! ok {
log . Printf ( "bogus blobref in keyEdgeBackward index entry: %q" , keyPart [ 3 ] )
return
}
return & camtypes . Edge {
From : parentRef ,
FromType : valPart [ 0 ] ,
FromTitle : valPart [ 1 ] ,
BlobRef : blobRef ,
} , true
}
2013-09-10 20:14:53 +00:00
// GetDirMembers sends on dest the children of the static directory dir.
2013-09-12 13:25:31 +00:00
func ( x * Index ) GetDirMembers ( dir blob . Ref , dest chan <- blob . Ref , limit int ) ( err error ) {
2013-09-10 20:14:53 +00:00
defer close ( dest )
sent := 0
it := x . queryPrefix ( keyStaticDirChild , dir . String ( ) )
2013-09-12 13:25:31 +00:00
defer closeIterator ( it , & err )
2013-09-10 20:14:53 +00:00
for it . Next ( ) {
keyPart := strings . Split ( it . Key ( ) , "|" )
if len ( keyPart ) != 3 {
return fmt . Errorf ( "index: bogus key keyStaticDirChild = %q" , it . Key ( ) )
}
child , ok := blob . Parse ( keyPart [ 2 ] )
if ! ok {
continue
}
dest <- child
sent ++
if sent == limit {
break
}
}
return nil
}
2013-11-17 17:41:45 +00:00
func kvBlobMeta ( k , v string ) ( bm camtypes . BlobMeta , ok bool ) {
2013-12-04 19:58:59 +00:00
refStr := k [ len ( "meta:" ) : ]
2013-11-17 17:41:45 +00:00
br , ok := blob . Parse ( refStr )
if ! ok {
return
}
pipe := strings . Index ( v , "|" )
if pipe < 0 {
return
}
2013-11-29 21:41:39 +00:00
size , err := strconv . ParseUint ( v [ : pipe ] , 10 , 32 )
2013-11-17 17:41:45 +00:00
if err != nil {
return
}
return camtypes . BlobMeta {
Ref : br ,
2013-11-29 21:41:39 +00:00
Size : uint32 ( size ) ,
2013-11-17 17:41:45 +00:00
CamliType : camliTypeFromMIME ( v [ pipe + 1 : ] ) ,
} , true
}
2013-12-04 19:58:59 +00:00
func kvBlobMeta_bytes ( k , v [ ] byte ) ( bm camtypes . BlobMeta , ok bool ) {
ref := k [ len ( "meta:" ) : ]
br , ok := blob . ParseBytes ( ref )
if ! ok {
return
}
pipe := bytes . IndexByte ( v , '|' )
if pipe < 0 {
return
}
size , err := strutil . ParseUintBytes ( v [ : pipe ] , 10 , 32 )
if err != nil {
return
}
return camtypes . BlobMeta {
Ref : br ,
Size : uint32 ( size ) ,
CamliType : camliTypeFromMIME_bytes ( v [ pipe + 1 : ] ) ,
} , true
}
2013-11-23 07:24:54 +00:00
func enumerateBlobMeta ( s sorted . KeyValue , cb func ( camtypes . BlobMeta ) error ) ( err error ) {
2013-11-17 01:24:02 +00:00
it := queryPrefixString ( s , "meta:" )
2013-10-19 00:17:35 +00:00
defer closeIterator ( it , & err )
for it . Next ( ) {
2013-11-17 17:41:45 +00:00
bm , ok := kvBlobMeta ( it . Key ( ) , it . Value ( ) )
2013-10-19 00:17:35 +00:00
if ! ok {
continue
}
2013-11-17 17:41:45 +00:00
if err := cb ( bm ) ; err != nil {
2013-11-17 01:24:02 +00:00
return err
2013-10-19 00:17:35 +00:00
}
}
2013-11-17 01:24:02 +00:00
return nil
}
2013-11-23 07:24:54 +00:00
func enumerateSignerKeyId ( s sorted . KeyValue , cb func ( blob . Ref , string ) ) ( err error ) {
2013-11-17 17:41:45 +00:00
const pfx = "signerkeyid:"
it := queryPrefixString ( s , pfx )
defer closeIterator ( it , & err )
for it . Next ( ) {
if br , ok := blob . Parse ( strings . TrimPrefix ( it . Key ( ) , pfx ) ) ; ok {
cb ( br , it . Value ( ) )
}
}
return
}
2013-11-17 01:24:02 +00:00
// EnumerateBlobMeta sends all metadata about all known blobs to ch and then closes ch.
2013-12-03 04:01:37 +00:00
func ( x * Index ) EnumerateBlobMeta ( ctx * context . Context , ch chan <- camtypes . BlobMeta ) ( err error ) {
2013-11-17 01:24:02 +00:00
if x . corpus != nil {
2013-12-12 07:34:07 +00:00
x . corpus . RLock ( )
defer x . corpus . RUnlock ( )
return x . corpus . EnumerateBlobMetaLocked ( ctx , ch )
2013-11-17 01:24:02 +00:00
}
defer close ( ch )
return enumerateBlobMeta ( x . s , func ( bm camtypes . BlobMeta ) error {
2013-12-03 04:01:37 +00:00
select {
case ch <- bm :
case <- ctx . Done ( ) :
return context . ErrCanceled
}
2013-11-17 01:24:02 +00:00
return nil
} )
2013-10-19 00:17:35 +00:00
}
2013-08-27 02:07:28 +00:00
// Storage returns the index's underlying Storage implementation.
2013-11-23 07:24:54 +00:00
func ( x * Index ) Storage ( ) sorted . KeyValue { return x . s }
2013-08-27 02:07:28 +00:00
2013-11-23 07:24:54 +00:00
// Close closes the underlying sorted.KeyValue, if the storage has a Close method.
2013-08-27 02:07:28 +00:00
// The return value is the return value of the underlying Close, or
// nil otherwise.
func ( x * Index ) Close ( ) error {
if cl , ok := x . s . ( io . Closer ) ; ok {
return cl . Close ( )
}
return nil
}
2013-11-16 18:35:18 +00:00
2013-12-04 19:58:59 +00:00
const camliTypeMIMEPrefix = "application/json; camliType="
var camliTypeMIMEPrefixBytes = [ ] byte ( camliTypeMIMEPrefix )
2013-11-16 18:35:18 +00:00
// "application/json; camliType=file" => "file"
// "image/gif" => ""
func camliTypeFromMIME ( mime string ) string {
2013-12-04 19:58:59 +00:00
if v := strings . TrimPrefix ( mime , camliTypeMIMEPrefix ) ; v != mime {
2013-11-16 18:35:18 +00:00
return v
}
return ""
}
2013-11-16 23:00:30 +00:00
2013-12-04 19:58:59 +00:00
func camliTypeFromMIME_bytes ( mime [ ] byte ) string {
if v := bytes . TrimPrefix ( mime , camliTypeMIMEPrefixBytes ) ; len ( v ) != len ( mime ) {
return strutil . StringFromBytes ( v )
}
return ""
}
2013-11-16 23:00:30 +00:00
// TODO(bradfitz): rename this? This is really about signer-attr-value
// (PermanodeOfSignerAttrValue), and not about indexed attributes in general.
func IsIndexedAttribute ( attr string ) bool {
switch attr {
case "camliRoot" , "camliImportRoot" , "tag" , "title" :
return true
}
return false
}
// IsBlobReferenceAttribute returns whether attr is an attribute whose
// value is a blob reference (e.g. camliMember) and thus something the
// indexers should keep inverted indexes on for parent/child-type
// relationships.
func IsBlobReferenceAttribute ( attr string ) bool {
switch attr {
case "camliMember" :
return true
}
return false
}
func IsFulltextAttribute ( attr string ) bool {
switch attr {
case "tag" , "title" :
return true
}
return false
}