diff --git a/pkg/server/sync.go b/pkg/server/sync.go
index 05c893b21..6bdb8ad37 100644
--- a/pkg/server/sync.go
+++ b/pkg/server/sync.go
@@ -41,6 +41,8 @@ type SyncHandler struct {
fromName, fromqName, toName string
from, fromq, to blobserver.Storage
+ idle bool // if true, the handler does nothing other than providing the discovery.
+
copierPoolSize int
lk sync.Mutex // protects following
@@ -62,9 +64,17 @@ func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handle
to := conf.RequiredString("to")
fullSync := conf.OptionalBool("fullSyncOnStart", false)
blockFullSync := conf.OptionalBool("blockingFullSyncOnStart", false)
+ idle := conf.OptionalBool("idle", false)
if err = conf.Validate(); err != nil {
return
}
+ if idle {
+ synch, err := createIdleSyncHandler(from, to)
+ if err != nil {
+ return nil, err
+ }
+ return synch, nil
+ }
fromBs, err := ld.GetStorage(from)
if err != nil {
return
@@ -142,6 +152,16 @@ func createSyncHandler(fromName, toName string, from blobserver.StorageQueueCrea
return h, nil
}
+func createIdleSyncHandler(fromName, toName string) (*SyncHandler, error) {
+ h := &SyncHandler{
+ fromName: fromName,
+ toName: toName,
+ idle: true,
+ status: "disabled",
+ }
+ return h, nil
+}
+
func (sh *SyncHandler) discoveryMap() map[string]interface{} {
// TODO(mpl): more status info
return map[string]interface{}{
@@ -156,6 +176,9 @@ func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
fmt.Fprintf(rw, "
%s to %s Sync Status
Current status: %s
",
sh.fromName, sh.toName, html.EscapeString(sh.status))
+ if sh.idle {
+ return
+ }
fmt.Fprintf(rw, "Stats:
")
fmt.Fprintf(rw, "- Blobs copied: %d
", sh.totalCopies)
diff --git a/pkg/serverconfig/genconfig.go b/pkg/serverconfig/genconfig.go
index 695f3b940..08a228501 100644
--- a/pkg/serverconfig/genconfig.go
+++ b/pkg/serverconfig/genconfig.go
@@ -424,12 +424,25 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json
}
if haveIndex {
+ syncArgs := map[string]interface{}{
+ "from": "/bs/",
+ "to": params.indexerPath,
+ }
+ // TODO(mpl): Brad says the cond should be dest == /index-*.
+ // But what about when dest is index-mem and we have a local disk;
+ // don't we want to have an active synchandler to do the fullSyncOnStart?
+ // Anyway, that condition works for now.
+ if params.blobPath == "" {
+ // When our primary blob store is remote (s3 or google cloud),
+ // i.e not an efficient replication source, we do not want the
+ // synchandler to mirror to the indexer. But we still want a
+ // synchandler to provide the discovery for e.g tools like
+ // camtool sync. See http://camlistore.org/issue/201
+ syncArgs["idle"] = true
+ }
m["/sync/"] = map[string]interface{}{
- "handler": "sync",
- "handlerArgs": map[string]interface{}{
- "from": "/bs/",
- "to": params.indexerPath,
- },
+ "handler": "sync",
+ "handlerArgs": syncArgs,
}
m["/bs-and-index/"] = map[string]interface{}{
@@ -579,8 +592,13 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
}
nolocaldisk := blobPath == ""
- if nolocaldisk && s3 == "" && googlecloudstorage == "" {
- return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
+ if nolocaldisk {
+ if s3 == "" && googlecloudstorage == "" {
+ return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
+ }
+ if s3 != "" && googlecloudstorage != "" {
+ return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
+ }
}
if shareHandler && shareHandlerPath == "" {
diff --git a/pkg/serverconfig/testdata/google_nolocaldisk-want.json b/pkg/serverconfig/testdata/google_nolocaldisk-want.json
index a976bf599..bf372bb14 100644
--- a/pkg/serverconfig/testdata/google_nolocaldisk-want.json
+++ b/pkg/serverconfig/testdata/google_nolocaldisk-want.json
@@ -41,7 +41,8 @@
"handler": "sync",
"handlerArgs": {
"from": "/bs/",
- "to": "/index-mem/"
+ "to": "/index-mem/",
+ "idle": true
}
},
diff --git a/pkg/serverconfig/testdata/s3_google_nolocaldisk.err b/pkg/serverconfig/testdata/s3_google_nolocaldisk.err
new file mode 100644
index 000000000..01dd0e633
--- /dev/null
+++ b/pkg/serverconfig/testdata/s3_google_nolocaldisk.err
@@ -0,0 +1 @@
+Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.
\ No newline at end of file
diff --git a/pkg/serverconfig/testdata/s3_google_nolocaldisk.json b/pkg/serverconfig/testdata/s3_google_nolocaldisk.json
new file mode 100644
index 000000000..0fe116850
--- /dev/null
+++ b/pkg/serverconfig/testdata/s3_google_nolocaldisk.json
@@ -0,0 +1,13 @@
+{
+ "listen": "localhost:3179",
+ "https": false,
+ "auth": "userpass:camlistore:pass3179",
+ "identity": "26F5ABDA",
+ "identitySecretRing": "/path/to/secring",
+ "memIndex": true,
+ "s3": "key:secret:bucket",
+ "googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName",
+ "replicateTo": [],
+ "publish": {},
+ "shareHandlerPath": "/share/"
+}
diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk-want.json b/pkg/serverconfig/testdata/s3_nolocaldisk-want.json
index f5610c182..e411d399a 100644
--- a/pkg/serverconfig/testdata/s3_nolocaldisk-want.json
+++ b/pkg/serverconfig/testdata/s3_nolocaldisk-want.json
@@ -41,7 +41,8 @@
"handler": "sync",
"handlerArgs": {
"from": "/bs/",
- "to": "/index-mem/"
+ "to": "/index-mem/",
+ "idle": true
}
},
diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json
new file mode 100644
index 000000000..3dd09dd2e
--- /dev/null
+++ b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json
@@ -0,0 +1,115 @@
+{
+ "listen": "localhost:3179",
+ "auth": "userpass:camlistore:pass3179",
+ "https": false,
+ "prefixes": {
+ "/": {
+ "handler": "root",
+ "handlerArgs": {
+ "blobRoot": "/bs-and-maybe-also-index/",
+ "searchRoot": "/my-search/",
+ "statusRoot": "/status/",
+ "stealth": false
+ }
+ },
+
+ "/ui/": {
+ "handler": "ui",
+ "handlerArgs": {
+ "jsonSignRoot": "/sighelper/",
+ "cache": "/cache/",
+ "scaledImage": "lrucache"
+ }
+ },
+
+ "/setup/": {
+ "handler": "setup"
+ },
+
+ "/status/": {
+ "handler": "status"
+ },
+
+ "/share/": {
+ "handler": "share",
+ "handlerArgs": {
+ "blobRoot": "/bs/"
+ }
+ },
+
+ "/sync/": {
+ "handler": "sync",
+ "handlerArgs": {
+ "from": "/bs/",
+ "to": "/index-mysql/",
+ "idle": true
+ }
+ },
+
+ "/sighelper/": {
+ "handler": "jsonsign",
+ "handlerArgs": {
+ "secretRing": "/path/to/secring",
+ "keyId": "26F5ABDA",
+ "publicKeyDest": "/bs-and-index/"
+ }
+ },
+
+ "/bs-and-index/": {
+ "handler": "storage-replica",
+ "handlerArgs": {
+ "backends": ["/bs/", "/index-mysql/"]
+ }
+ },
+
+ "/bs-and-maybe-also-index/": {
+ "handler": "storage-cond",
+ "handlerArgs": {
+ "write": {
+ "if": "isSchema",
+ "then": "/bs-and-index/",
+ "else": "/bs/"
+ },
+ "read": "/bs/"
+ }
+ },
+
+ "/bs/": {
+ "handler": "storage-s3",
+ "handlerArgs": {
+ "aws_access_key": "key",
+ "aws_secret_access_key": "secret",
+ "bucket": "bucket"
+ }
+ },
+
+ "/cache/": {
+ "handler": "storage-filesystem",
+ "handlerArgs": {
+ "path": "/tmp/camli-cache"
+ }
+ },
+
+ "/index-mysql/": {
+ "enabled": true,
+ "handler": "storage-mysqlindexer",
+ "handlerArgs": {
+ "blobSource": "/bs/",
+ "database": "camlitest",
+ "host": "localhost",
+ "password": "password",
+ "user": "user"
+ }
+ },
+
+ "/my-search/": {
+ "handler": "search",
+ "handlerArgs": {
+ "index": "/index-mysql/",
+ "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4"
+ }
+ }
+
+ }
+
+}
diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk_mysql.json b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql.json
new file mode 100644
index 000000000..eb0d88c84
--- /dev/null
+++ b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql.json
@@ -0,0 +1,13 @@
+{
+ "listen": "localhost:3179",
+ "https": false,
+ "auth": "userpass:camlistore:pass3179",
+ "identity": "26F5ABDA",
+ "identitySecretRing": "/path/to/secring",
+ "dbname": "camlitest",
+ "mysql": "user@localhost:password",
+ "s3": "key:secret:bucket",
+ "replicateTo": [],
+ "publish": {},
+ "shareHandlerPath": "/share/"
+}