serverconfig & camlistored: simplify low/high-level config bootstrapping.

Change-Id: I22e003ff2eaa29b09a5c4cbb59afa1ca9bd624df

config/dev-blobserver-config.json

@@ -1,4 +1,5 @@
 { "_for-emacs": "-*- mode: js2;-*-",
+  "handlerConfig": true,
   "baseURL": ["_env", "http://localhost:${CAMLI_PORT}"],
   "password": ["_env", "${CAMLI_PASSWORD}"],
 

config/dev-indexer-config.json

@@ -1,4 +1,5 @@
 { "_for-emacs": "-*- mode: js2;-*-",
+  "handlerConfig": true,
   "baseURL": ["_env", "http://localhost:${CAMLI_PORT}"],
   "password": ["_env", "${CAMLI_PASSWORD}"],
   "prefixes": {

config/dev-server-config.json

@@ -1,4 +1,5 @@
 { "_for-emacs": "-*- mode: js2;-*-",
+  "handlerConfig": true,
   "baseURL": ["_env", "${CAMLI_BASEURL}"],
   "auth": ["_env", "${CAMLI_AUTH}"],
   "https": ["_env", "${CAMLI_TLS}", false],

config/dev-tls-config.json

@@ -1,4 +1,5 @@
 { "_for-emacs": "-*- mode: js2;-*-",
+  "handlerConfig": true,
   "baseURL": ["_env", "http://localhost:${CAMLI_PORT}"],
   "password": ["_env", "${CAMLI_PASSWORD}"],
   "prefixes": {

pkg/serverconfig/genconfig.go

@@ -166,33 +166,35 @@ func genLowLevelPrefixes(params *configPrefixesParams) jsonconfig.Obj {
 	return prefixes
 }
 
-// TODO(mpl): check the high level config for invalid keywords. with validate maybe?
 func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
-	obj := jsonconfig.Obj{}
-	baseUrl := conf.RequiredString("listen")
-	if baseUrl == "" {
-		return nil, fmt.Errorf("\"listen\" missing in user config file")
-	}
-	tls := conf.RequiredBool("TLS")
-	scheme := "http"
-	if tls {
-		scheme = "https"
-	}
-	auth := conf.RequiredString("auth")
-	if auth == "" {
-		return nil, fmt.Errorf("\"auth\" missing in user config file")
+	var (
+		baseUrl    = conf.RequiredString("listen")
+		tlsOn      = conf.OptionalBool("TLS", false)
+		auth       = conf.RequiredString("auth")
+		dbname     = conf.OptionalString("dbname", "")
+		secretRing = conf.OptionalString("secring", "")
+		blobPath   = conf.RequiredString("blobPath")
+		keyId      = conf.OptionalString("keyid", "")
+		mysql      = conf.OptionalString("mysql", "")
+		mongo      = conf.OptionalString("mongo", "")
+		_          = conf.OptionalList("replicateTo")
+		_          = conf.OptionalString("s3", "")
+	)
+	if err := conf.Validate(); err != nil {
+		return nil, err
 	}
 
-	obj["baseURL"] = scheme + "://" + baseUrl
-	obj["https"] = tls
-	obj["auth"] = auth
-	if tls {
-		// TODO(mpl): probably need other default paths
+	obj := jsonconfig.Obj{}
+	scheme := "http"
+	if tlsOn {
+		scheme = "https"
 		obj["TLSCertFile"] = "config/selfgen_cert.pem"
 		obj["TLSKeyFile"] = "config/selfgen_key.pem"
 	}
+	obj["baseURL"] = scheme + "://" + baseUrl
+	obj["https"] = tlsOn
+	obj["auth"] = auth
 
-	dbname := conf.OptionalString("dbname", "")
 	if dbname == "" {
 		username := os.Getenv("USER")
 		if username == "" {
@@ -201,7 +203,6 @@ func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 		dbname = "camli" + username
 	}
 
-	secretRing := conf.OptionalString("secring", "")
 	if secretRing == "" {
 		secretRing = filepath.Join(osutil.HomeDir(), ".camli", "secring.gpg")
 		_, err = os.Stat(secretRing)
@@ -210,16 +211,11 @@ func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 		}
 	}
 
-	keyId := conf.OptionalString("keyid", "")
 	if keyId == "" {
 		// TODO(mpl): where do we get a default keyId from? Brad?
 		keyId = "26F5ABDA"
 	}
 
-	blobPath := conf.RequiredString("blobPath")
-	if blobPath == "" {
-		return nil, fmt.Errorf("\"blobPath\" not defined in config")
-	}
 	indexerPath := "/index-mem/"
 
 	prefixesParams := &configPrefixesParams{
@@ -237,8 +233,6 @@ func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 
 	addUiConfig(&prefixes, "/ui/")
 
-	mysql := conf.OptionalString("mysql", "")
-	mongo := conf.OptionalString("mongo", "")
 	if mongo != "" && mysql != "" {
 		return nil, fmt.Errorf("Cannot have both mysql and mongo in config, pick one")
 	}
@@ -254,9 +248,9 @@ func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 
 	obj["prefixes"] = (map[string]interface{})(prefixes)
 
-	// TODO(mpl): configPath
 	lowLevelConf = &Config{
 		jsonconfig.Obj: obj,
+		configPath:     conf.configPath,
 	}
 	return lowLevelConf, nil
 }

pkg/serverconfig/serverconfig.go

@@ -280,15 +280,29 @@ type Config struct {
 	configPath string // Filesystem path
 }
 
-func Load(configPath string) (*Config, error) {
-	obj, err := jsonconfig.ReadFile(configPath)
+// Load returns a low-level "handler config" from the provided filename.
+// If the config file doesn't contain a top-level JSON key of "handlerConfig"
+// with boolean value true, the configuration is assumed to be a high-level
+// "user config" file, and transformed into a low-level config.
+func Load(filename string) (*Config, error) {
+	obj, err := jsonconfig.ReadFile(filename)
 	if err != nil {
 		return nil, err
 	}
 	conf := &Config{
 		Obj:        obj,
-		configPath: configPath,
+		configPath: filename,
 	}
+
+	if lowLevel := obj.OptionalBool("handlerConfig", false); !lowLevel {
+		conf, err = GenLowLevelConfig(conf)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"Failed to transform user config file %q into internal handler configuration: %v",
+				filename, err)
+		}
+	}
+
 	return conf, nil
 }
 

server/camlistored/camlistored.go

@@ -122,35 +122,30 @@ func genSelfTLS(listen string) error {
 	return nil
 }
 
-// checkConfigFile tests if the given config file is a
-// valid path to an existing file, and returns the same
-// value if yes, an error if not.
-// If file is the empty string, it checks for the existence
-// of a default config and creates it if absent. It returns
-// newfile as the path to that file, or an error if a
-// problem occured.
-func checkConfigFile(file string) (newfile string, err error) {
-	newfile = file
-	if newfile == "" {
-		newfile = osutil.UserServerConfigPath()
-		_, err := os.Stat(newfile)
-		if err != nil {
-			log.Printf("Can't open default conf file, now creating a new one at %s \n", newfile)
-			err = newDefaultConfigFile(newfile)
-			if err != nil {
-				return "", err
-			}
+// findConfigFile returns the absolute path of the user's
+// config file.
+// The provided file may be absolute or relative
+// to the user's configuration directory.
+// If file is empty, a default high-level config is written
+// for the user.
+func findConfigFile(file string) (absPath string, err error) {
+	switch {
+	case file == "":
+		absPath = osutil.UserServerConfigPath()
+		_, err = os.Stat(absPath)
+		if os.IsNotExist(err) {
+			os.MkdirAll(osutil.CamliConfigDir(), 0700)
+			log.Printf("Generating template config file %s", absPath)
+			err = newDefaultConfigFile(absPath)
 		}
-		return newfile, nil
+		return
+	case filepath.IsAbs(file):
+		absPath = file
+	default:
+		absPath = filepath.Join(osutil.CamliConfigDir(), file)
 	}
-	if !filepath.IsAbs(newfile) {
-		newfile = filepath.Join(osutil.CamliConfigDir(), newfile)
-	}
-	_, err = os.Stat(newfile)
-	if err != nil {
-		return "", fmt.Errorf("can't stat %s: %q", file, err)
-	}
-	return newfile, nil
+	_, err = os.Stat(absPath)
+	return
 }
 
 // TODO: "auth": "localtcp". See http://code.google.com/p/camlistore/issues/detail?id=50
@@ -168,12 +163,12 @@ func newDefaultConfigFile(path string) error {
 	"replicateTo": []
 }
 `
-	blobDir := filepath.Join(osutil.HomeDir(), "var", "camlistore", "blobs")
+	blobDir := osutil.CamliBlobRoot()
 	if err := os.MkdirAll(blobDir, 0700); err != nil {
 		return fmt.Errorf("Could not create default blobs directory: %v", err)
 	}
 	serverConf = strings.Replace(serverConf, "%BLOBPATH%", blobDir, 1)
-	secRing := filepath.Join(osutil.HomeDir(), ".camli", "secring.gpg")
+	secRing := filepath.Join(osutil.CamliConfigDir(), "secring.gpg")
 	serverConf = strings.Replace(serverConf, "%SECRING%", secRing, 1)
 	if err := ioutil.WriteFile(path, []byte(serverConf), 0700); err != nil {
 		return fmt.Errorf("Could not create or write default server config: %v", err)
@@ -218,17 +213,14 @@ func setupTLS(ws *webserver.Server, config *serverconfig.Config, listen string)
 func main() {
 	flag.Parse()
 
-	file, err := checkConfigFile(*flagConfigFile)
+	fileName, err := findConfigFile(*flagConfigFile)
 	if err != nil {
-		exitf("Problem with config file: %q", err)
+		exitf("Error finding config file %q: %v", fileName, err)
 	}
-	conf, err := serverconfig.Load(file)
+	log.Printf("Using config file %s", fileName)
+	config, err := serverconfig.Load(fileName)
 	if err != nil {
-		exitf("Could not load server config file %v: %v", file, err)
-	}
-	config, err := serverconfig.GenLowLevelConfig(conf)
-	if err != nil {
-		exitf("Could not gen low level server config: %v", err)
+		exitf("Could not load server config: %v", err)
 	}
 
 	ws := webserver.New()