pkg/server: add recovery option on web UI and instance page

When the blobpacked index gets corrupted/destroyed, it needs to be
rebuilt and hence camlistored refuses to start without the -recovery
flag being set if it detects it is needed.

This is a problem on GCE, because camlistored is handled by systemd, and
so the only way to restart camlistored with -recovery is to edit the
systemd service, then issue 'systemctl daemon-reload', and 'systemctl
restart camlistored'. Then revert the change for the next time
camlistored restarts. This is not user-friendly.

This change adds a check on camlistored startup based on the presence
and value (as a boolean) of the "camlistore-recovery" key as an instance
attribute. Therefore, the user only has to add the
("camlistore-recovery", "true") attribute in the Custom metadata of
their Google Cloud instance page to switch to recovery mode. (And
restart the instance).

As an additional feature (for non-GCE users), this change also adds the
option to restart the server in recovery mode from the status page of
the web UI.

Change-Id: I44f5ca293ddd0a0033fc5d9c2edca1bac0ee9c8f

pkg/blobserver/blobpacked/blobpacked.go

@@ -99,6 +99,7 @@ import (
 	"camlistore.org/pkg/blob"
 	"camlistore.org/pkg/blobserver"
 	"camlistore.org/pkg/constants"
+	"camlistore.org/pkg/env"
 	"camlistore.org/pkg/pools"
 	"camlistore.org/pkg/schema"
 	"camlistore.org/pkg/sorted"
@@ -291,6 +292,9 @@ func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (blobserver.Storag
 	// is recorded. This is probably a corrupt state, and the user likely
 	// wants to recover.
 	if !sto.anyMeta() && sto.anyZipPacks() {
+		if env.OnGCE() {
+			log.Fatal("Error: blobpacked storage detects non-zero packed zips, but no metadata. Please switch to recovery mode: add the \"camlistore-recovery = true\" key/value to the Custom metadata of your instance. And restart the instance.")
+		}
 		log.Fatal("Error: blobpacked storage detects non-zero packed zips, but no metadata. Please re-start in recovery mode with -recovery.")
 	}
 

pkg/server/status.go

@@ -261,7 +261,8 @@ func (sh *StatusHandler) serveStatusHTML(rw http.ResponseWriter, req *http.Reque
 	f("<h2>Admin</h2>")
 	f("<ul>")
 	f("  <li><form method='post' action='restart' onsubmit='return confirm(\"Really restart now?\")'><button>restart server</button>")
-	f("<input type='checkbox' name='reindex'> reindex</form></li>")
+	f("<input type='checkbox' name='reindex'> reindex <input type='checkbox' name='recovery'> recovery</form></li>")
+	f("</form></li>")
 	if env.OnGCE() {
 		console, err := sh.googleCloudConsole()
 		if err != nil {
@@ -313,6 +314,7 @@ func (sh *StatusHandler) serveRestart(rw http.ResponseWriter, req *http.Request)
 	}
 
 	reindex := (req.FormValue("reindex") == "on")
+	recovery := (req.FormValue("recovery") == "on")
 
 	log.Println("Restarting camlistored")
 	rw.Header().Set("Connection", "close")
@@ -320,7 +322,7 @@ func (sh *StatusHandler) serveRestart(rw http.ResponseWriter, req *http.Request)
 	if f, ok := rw.(http.Flusher); ok {
 		f.Flush()
 	}
-	osutil.RestartProcess(fmt.Sprintf("-reindex=%t", reindex))
+	osutil.RestartProcess(fmt.Sprintf("-reindex=%t", reindex), fmt.Sprintf("-recovery=%t", recovery))
 }
 
 var cgoEnabled bool

server/camlistored/camlistored.go

@@ -670,6 +670,33 @@ func setupLogging() io.Closer {
 	return maybeSetupGoogleCloudLogging()
 }
 
+func checkRecovery() {
+	if *flagRecovery {
+		blobpacked.SetRecovery()
+		return
+	}
+	if !env.OnGCE() {
+		return
+	}
+	recovery, err := metadata.InstanceAttributeValue("camlistore-recovery")
+	if err != nil {
+		if _, ok := err.(metadata.NotDefinedError); !ok {
+			log.Printf("error getting camlistore-recovery: %v", err)
+		}
+		return
+	}
+	if recovery == "" {
+		return
+	}
+	doRecovery, err := strconv.ParseBool(recovery)
+	if err != nil {
+		log.Printf("invalid bool value for \"camlistore-recovery\": %v", err)
+	}
+	if doRecovery {
+		blobpacked.SetRecovery()
+	}
+}
+
 // main wraps Main so tests (which generate their own func main) can still run Main.
 func main() {
 	Main(nil, nil)
@@ -694,9 +721,7 @@ func Main(up chan<- struct{}, down <-chan struct{}) {
 		}
 		return
 	}
-	if *flagRecovery {
-		blobpacked.SetRecovery()
-	}
+	checkRecovery()
 
 	// In case we're running in a Docker container with no
 	// filesytem from which to load the root CAs, this