App Engine: start of writing an IndexStorage impl for datastore.

Change-Id: I03fb6be33b570fa56a4370f6ca86da6858490510

server/go/appengine/README

@@ -0,0 +1,7 @@
+To run:
+
+$ dev_appserver.py --high_replication .
+
+Other useful flags:
+  -a 0.0.0.0     (listen on all addresses)
+  -c             (wipe the datastore)

server/go/appengine/camli/aeindex.go

@@ -19,32 +19,86 @@ limitations under the License.
 package appengine
 
 import (
-	"camlistore.org/pkg/index"
+	"errors"
+
 	"camlistore.org/pkg/blobserver"
+	"camlistore.org/pkg/index"
 	"camlistore.org/pkg/jsonconfig"
+
+	"appengine"
+	"appengine/datastore"
 )
 
+var (
+	indexRowKind = "IndexRow"
+)
+
+// A row of the index.  Keyed by "<namespace>|<keyname>"
+type indexRowEnt struct {
+	value []byte
+}
+
+type indexStorage struct {
+	ns string
+}
+
+func (is *indexStorage) key(c appengine.Context, key string) *datastore.Key {
+	return datastore.NewKey(c, indexRowKind, is.ns + "|" + key, 0, nil)
+}
+
+func (is *indexStorage) BeginBatch() index.BatchMutation {
+	panic("TODO: impl")
+}
+
+func (is *indexStorage) CommitBatch(bm index.BatchMutation) error {
+	return errors.New("TODO: impl")
+}
+
+func (is *indexStorage) Get(key string) (string, error) {
+	c := getContext()
+	defer putContext(c)
+
+	panic("TODO: impl")
+}
+
+func (is *indexStorage) Set(key, value string) error {
+	c := getContext()
+	defer putContext(c)
+
+	panic("TODO: impl")
+}
+
+func (is *indexStorage) Delete(key string) error {
+	c := getContext()
+	defer putContext(c)
+
+	panic("TODO: impl")
+}
+
+func (is *indexStorage) Find(key string) index.Iterator {
+	panic("TODO: impl")
+}
+
 func indexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
-	//sto := &appengineIndexStorage{}
-	ns := config.OptionalString("namespace", "")
+	is := &indexStorage{}
+	var (
+		blobPrefix = config.RequiredString("blobSource")
+		ns         = config.OptionalString("namespace", "")
+	)
 	if err := config.Validate(); err != nil {
 		return nil, err
 	}
-	_ = ns
-
-	var sto index.IndexStorage
-	// TODO
-        ix := index.New(sto)
-	return ix, nil
-
-	/*
-        ix.BlobSource = sto
-	// Good enough, for now:
-	ix.KeyFetcher = ix.BlobSource
-	sto.namespace, err = sanitizeNamespace(ns)
+	sto, err := ld.GetStorage(blobPrefix)
 	if err != nil {
 		return nil, err
 	}
-	return sto, nil
-	 */
+	is.ns, err = sanitizeNamespace(ns)
+	if err != nil {
+		return nil, err
+	}
+
+	ix := index.New(is)
+	ix.BlobSource = sto
+	ix.KeyFetcher = ix.BlobSource // TODO(bradfitz): global search? something else?
+	return ix, nil
 }

server/go/appengine/camli/contextpool.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2013 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package appengine
+
+import (
+	"sync"
+
+	"appengine"
+)
+
+type ContextPool struct {
+	mu sync.Mutex
+}
+
+func (p *ContextPool) Get() appengine.Context {
+	panic("TODO")
+}
+
+func (p *ContextPool) Put(c appengine.Context) {
+	panic("TODO")
+}

server/go/appengine/camli/main.go

@@ -32,7 +32,7 @@ import (
 	_ "camlistore.org/pkg/blobserver/shard"
 	_ "camlistore.org/pkg/server" // handlers: UI, publish, thumbnailing, etc
 
-	// TODO(bradfitz): uncomment these and deal with symlinks + config setup
+	// TODO(bradfitz): uncomment these config setup
 	// Both require an App Engine context to make HTTP requests too.
 	//_ "camlistore.org/pkg/blobserver/remote"
 	//_ "camlistore.org/pkg/blobserver/s3"
@@ -59,6 +59,16 @@ func (li *lazyInit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+var ctxPool ContextPool
+
+func getContext() appengine.Context {
+	return ctxPool.Get()
+}
+
+func putContext(c appengine.Context) {
+	ctxPool.Put(c)
+}
+
 var root = new(lazyInit)
 
 func init() {

server/go/appengine/config.json

@@ -73,7 +73,8 @@
     "/indexer/": {
       "handler": "storage-aeindex",
       "handlerArgs": {
-        "namespace": "idx1"
+        "namespace": "idx1",
+        "blobSource": "/bs/"
       }
     },