Merge "serverconfig: idle synchandler when no localdisk as primary storage"

pkg/server/sync.go

@@ -41,6 +41,8 @@ type SyncHandler struct {
 	fromName, fromqName, toName string
 	from, fromq, to             blobserver.Storage
 
+	idle bool // if true, the handler does nothing other than providing the discovery.
+
 	copierPoolSize int
 
 	lk             sync.Mutex // protects following
@@ -62,9 +64,17 @@ func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handle
 	to := conf.RequiredString("to")
 	fullSync := conf.OptionalBool("fullSyncOnStart", false)
 	blockFullSync := conf.OptionalBool("blockingFullSyncOnStart", false)
+	idle := conf.OptionalBool("idle", false)
 	if err = conf.Validate(); err != nil {
 		return
 	}
+	if idle {
+		synch, err := createIdleSyncHandler(from, to)
+		if err != nil {
+			return nil, err
+		}
+		return synch, nil
+	}
 	fromBs, err := ld.GetStorage(from)
 	if err != nil {
 		return
@@ -142,6 +152,16 @@ func createSyncHandler(fromName, toName string, from blobserver.StorageQueueCrea
 	return h, nil
 }
 
+func createIdleSyncHandler(fromName, toName string) (*SyncHandler, error) {
+	h := &SyncHandler{
+		fromName: fromName,
+		toName:   toName,
+		idle:     true,
+		status:   "disabled",
+	}
+	return h, nil
+}
+
 func (sh *SyncHandler) discoveryMap() map[string]interface{} {
 	// TODO(mpl): more status info
 	return map[string]interface{}{
@@ -156,6 +176,9 @@ func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 
 	fmt.Fprintf(rw, "<h1>%s to %s Sync Status</h1><p><b>Current status: </b>%s</p>",
 		sh.fromName, sh.toName, html.EscapeString(sh.status))
+	if sh.idle {
+		return
+	}
 
 	fmt.Fprintf(rw, "<h2>Stats:</h2><ul>")
 	fmt.Fprintf(rw, "<li>Blobs copied: %d</li>", sh.totalCopies)

pkg/serverconfig/genconfig.go

@@ -424,12 +424,25 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json
 	}
 
 	if haveIndex {
+		syncArgs := map[string]interface{}{
+			"from": "/bs/",
+			"to":   params.indexerPath,
+		}
+		// TODO(mpl): Brad says the cond should be dest == /index-*.
+		// But what about when dest is index-mem and we have a local disk;
+		// don't we want to have an active synchandler to do the fullSyncOnStart?
+		// Anyway, that condition works for now.
+		if params.blobPath == "" {
+			// When our primary blob store is remote (s3 or google cloud),
+			// i.e not an efficient replication source, we do not want the
+			// synchandler to mirror to the indexer. But we still want a
+			// synchandler to provide the discovery for e.g tools like
+			// camtool sync. See http://camlistore.org/issue/201
+			syncArgs["idle"] = true
+		}
 		m["/sync/"] = map[string]interface{}{
-			"handler": "sync",
+			"handler":     "sync",
-			"handlerArgs": map[string]interface{}{
+			"handlerArgs": syncArgs,
-				"from": "/bs/",
-				"to":   params.indexerPath,
-			},
 		}
 
 		m["/bs-and-index/"] = map[string]interface{}{
@@ -579,8 +592,13 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 	}
 
 	nolocaldisk := blobPath == ""
-	if nolocaldisk && s3 == "" && googlecloudstorage == "" {
+	if nolocaldisk {
-		return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
+		if s3 == "" && googlecloudstorage == "" {
+			return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
+		}
+		if s3 != "" && googlecloudstorage != "" {
+			return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
+		}
 	}
 
 	if shareHandler && shareHandlerPath == "" {

pkg/serverconfig/testdata/google_nolocaldisk-want.json

@@ -41,7 +41,8 @@
 			"handler": "sync",
 			"handlerArgs": {
 				"from": "/bs/",
-				"to": "/index-mem/"
+				"to": "/index-mem/",
+				"idle": true
 			}
 		},
 

pkg/serverconfig/testdata/s3_google_nolocaldisk.err

@@ -0,0 +1 @@
+Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.

pkg/serverconfig/testdata/s3_google_nolocaldisk.json

@@ -0,0 +1,13 @@
+{
+	"listen": "localhost:3179",
+	"https": false,
+	"auth": "userpass:camlistore:pass3179",
+	"identity": "26F5ABDA",
+	"identitySecretRing": "/path/to/secring",
+	"memIndex": true,
+	"s3": "key:secret:bucket",
+	"googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName",
+	"replicateTo": [],
+	"publish": {},
+	"shareHandlerPath": "/share/"
+}

pkg/serverconfig/testdata/s3_nolocaldisk-want.json

@@ -41,7 +41,8 @@
 			"handler": "sync",
 			"handlerArgs": {
 				"from": "/bs/",
-				"to": "/index-mem/"
+				"to": "/index-mem/",
+				"idle": true
 			}
 		},
 	

pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json

@@ -0,0 +1,115 @@
+{
+	"listen": "localhost:3179",
+	"auth": "userpass:camlistore:pass3179",
+	"https": false,
+	"prefixes": {
+		"/": {
+			"handler": "root",
+			"handlerArgs": {
+				"blobRoot": "/bs-and-maybe-also-index/",
+				"searchRoot": "/my-search/",
+				"statusRoot": "/status/",
+				"stealth": false
+			}
+		},
+
+		"/ui/": {
+			"handler": "ui",
+			"handlerArgs": {
+				"jsonSignRoot": "/sighelper/",
+				"cache": "/cache/",
+				"scaledImage": "lrucache"
+			}
+		},
+	
+ 		"/setup/": {
+			"handler": "setup"
+        },
+
+ 		"/status/": {
+			"handler": "status"
+		},
+
+ 		"/share/": {
+			"handler": "share",
+			"handlerArgs": {
+				"blobRoot": "/bs/"
+			}
+		},
+
+ 		"/sync/": {
+			"handler": "sync",
+			"handlerArgs": {
+				"from": "/bs/",
+				"to": "/index-mysql/",
+				"idle": true
+			}
+		},
+	
+		"/sighelper/": {
+			"handler": "jsonsign",
+			"handlerArgs": {
+				"secretRing": "/path/to/secring",
+				"keyId": "26F5ABDA",
+				"publicKeyDest": "/bs-and-index/"
+			}
+		},
+	
+		"/bs-and-index/": {
+			"handler": "storage-replica",
+			"handlerArgs": {
+				"backends": ["/bs/", "/index-mysql/"]
+			}
+		},
+	
+		"/bs-and-maybe-also-index/": {
+			"handler": "storage-cond",
+			"handlerArgs": {
+				"write": {
+					"if": "isSchema",
+					"then": "/bs-and-index/",
+					"else": "/bs/"
+				},
+				"read": "/bs/"
+			}
+		},
+	
+		"/bs/": {
+			"handler": "storage-s3",
+			"handlerArgs": {
+				"aws_access_key": "key",
+				"aws_secret_access_key": "secret",
+				"bucket": "bucket"
+			}
+		},
+	
+		"/cache/": {
+			"handler": "storage-filesystem",
+			"handlerArgs": {
+				"path": "/tmp/camli-cache"
+			}
+		},
+	
+		"/index-mysql/": {
+			"enabled": true,
+			"handler": "storage-mysqlindexer",
+			"handlerArgs": {
+				"blobSource": "/bs/",
+				"database": "camlitest",
+				"host": "localhost",
+				"password": "password",
+				"user": "user"
+			}
+		},
+	
+		"/my-search/": {
+			"handler": "search",
+			"handlerArgs": {
+				"index": "/index-mysql/",
+				"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4"
+			}
+		}
+
+	}
+
+}

pkg/serverconfig/testdata/s3_nolocaldisk_mysql.json

@@ -0,0 +1,13 @@
+{
+	"listen": "localhost:3179",
+	"https": false,
+	"auth": "userpass:camlistore:pass3179",
+	"identity": "26F5ABDA",
+	"identitySecretRing": "/path/to/secring",
+	"dbname": "camlitest",
+	"mysql": "user@localhost:password",
+	"s3": "key:secret:bucket",
+	"replicateTo": [],
+	"publish": {},
+	"shareHandlerPath": "/share/"
+}