From 7f630657d5a0cb4d96522671306e2c4b493398e4 Mon Sep 17 00:00:00 2001 From: mpl Date: Thu, 15 Mar 2012 13:31:06 +0100 Subject: [PATCH] generate low level configuration Change-Id: I43d1610bdc386954dea724b4b38e184bf16e2d34 --- pkg/jsonconfig/jsonconfig.go | 9 +- pkg/serverconfig/genconfig.go | 262 ++++++++++++++++++++ pkg/serverconfig/serverconfig.go | 3 - pkg/serverconfig/serverconfig_test.go | 100 ++++++++ pkg/serverconfig/testdata/default-want.json | 90 +++++++ pkg/serverconfig/testdata/default.json | 11 + pkg/webserver/webserver.go | 25 +- server/camlistored/camlistored.go | 60 +++-- 8 files changed, 526 insertions(+), 34 deletions(-) create mode 100644 pkg/serverconfig/genconfig.go create mode 100644 pkg/serverconfig/serverconfig_test.go create mode 100644 pkg/serverconfig/testdata/default-want.json create mode 100644 pkg/serverconfig/testdata/default.json diff --git a/pkg/jsonconfig/jsonconfig.go b/pkg/jsonconfig/jsonconfig.go index 31a4689c5..e6585ce52 100644 --- a/pkg/jsonconfig/jsonconfig.go +++ b/pkg/jsonconfig/jsonconfig.go @@ -162,8 +162,13 @@ func (jc Obj) int(key string, def *int) int { } b, ok := ei.(float64) if !ok { - jc.appendError(fmt.Errorf("Expected config key %q to be a number", key)) - return 0 + // TODO(mpl): float or int? or both allowed? use a switch? + c, ok := ei.(int) + if !ok { + jc.appendError(fmt.Errorf("Expected config key %q to be a number", key)) + return 0 + } + return int(c) } return int(b) } diff --git a/pkg/serverconfig/genconfig.go b/pkg/serverconfig/genconfig.go new file mode 100644 index 000000000..4f60eb5cd --- /dev/null +++ b/pkg/serverconfig/genconfig.go @@ -0,0 +1,262 @@ +/* +Copyright 2012 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serverconfig + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "camlistore.org/pkg/jsonconfig" + "camlistore.org/pkg/osutil" +) + +// various parameters derived from the high-level user config +// and needed to set up the low-level config. +type configPrefixesParams struct { + secretRing string + keyId string + indexerPath string + blobPath string +} + +func addUiConfig(prefixes *jsonconfig.Obj, uiPrefix string, published ...interface{}) { + ob := map[string]interface{}{} + ob["handler"] = "ui" + handlerArgs := map[string]interface{}{ + "blobRoot": "/bs-and-maybe-also-index/", + "searchRoot": "/my-search/", + "jsonSignRoot": "/sighelper/", + "cache": "/cache/", + "scaledImage": "lrucache", + } + if len(published) > 0 { + handlerArgs["publishRoots"] = published + } + ob["handlerArgs"] = handlerArgs + (*prefixes)[uiPrefix] = ob +} + +// TODO(mpl): add auth info +func addMongoConfig(prefixes *jsonconfig.Obj, dbname string, servers string) { + ob := map[string]interface{}{} + ob["enabled"] = true + ob["handler"] = "storage-mongodbindexer" + ob["handlerArgs"] = map[string]interface{}{ + "servers": servers, + "database": dbname, + "blobSource": "/bs/", + } + (*prefixes)["/index-mongo/"] = ob +} + +func addMysqlConfig(prefixes *jsonconfig.Obj, dbname string, dbinfo string) { + fields := strings.Split(dbinfo, "@") + if len(fields) != 2 { + exitFailure("Malformed mysql config string. Want: \"user@host:password\"") + } + user := fields[0] + fields = strings.Split(fields[1], ":") + if len(fields) != 2 { + exitFailure("Malformed mysql config string. Want: \"user@host:password\"") + } + ob := map[string]interface{}{} + ob["enabled"] = true + ob["handler"] = "storage-mysqlindexer" + ob["handlerArgs"] = map[string]interface{}{ + "host": fields[0], + "user": user, + "password": fields[1], + "database": dbname, + "blobSource": "/bs/", + } + (*prefixes)["/index-mysql/"] = ob +} + +func addMemindexConfig(prefixes *jsonconfig.Obj) { + ob := map[string]interface{}{} + ob["handler"] = "storage-memory-only-dev-indexer" + ob["handlerArgs"] = map[string]interface{}{ + "blobSource": "/bs/", + } + (*prefixes)["/index-mem/"] = ob +} + +func genLowLevelPrefixes(params *configPrefixesParams) jsonconfig.Obj { + prefixes := map[string]interface{}{} + + ob := map[string]interface{}{} + ob["handler"] = "root" + ob["handlerArgs"] = map[string]interface{}{"stealth": false} + prefixes["/"] = ob + + ob = map[string]interface{}{} + ob["handler"] = "sync" + ob["handlerArgs"] = map[string]interface{}{ + "from": "/bs/", + "to": params.indexerPath, + } + prefixes["/sync/"] = ob + + ob = map[string]interface{}{} + ob["handler"] = "jsonsign" + ob["handlerArgs"] = map[string]interface{}{ + "secretRing": params.secretRing, + "keyId": params.keyId, + "publicKeyDest": "/bs/", + } + prefixes["/sighelper/"] = ob + + ob = map[string]interface{}{} + ob["handler"] = "storage-replica" + ob["handlerArgs"] = map[string]interface{}{ + "backends": []interface{}{"/bs/", params.indexerPath}, + } + prefixes["/bs-and-index/"] = ob + + ob = map[string]interface{}{} + ob["handler"] = "storage-cond" + ob["handlerArgs"] = map[string]interface{}{ + "write": map[string]interface{}{ + "if": "isSchema", + "then": "/bs-and-index/", + "else": "/bs/", + }, + "read": "/bs/", + } + prefixes["/bs-and-maybe-also-index/"] = ob + + ob = map[string]interface{}{} + ob["handler"] = "storage-filesystem" + ob["handlerArgs"] = map[string]interface{}{ + "path": params.blobPath, + } + prefixes["/bs/"] = ob + + ob = map[string]interface{}{} + ob["handler"] = "storage-filesystem" + ob["handlerArgs"] = map[string]interface{}{ + "path": filepath.Join(params.blobPath, "/cache"), + } + prefixes["/cache/"] = ob + + ob = map[string]interface{}{} + ob["handler"] = "search" + ob["handlerArgs"] = map[string]interface{}{ + "index": params.indexerPath, + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + } + prefixes["/my-search/"] = ob + + return prefixes +} + +// TODO(mpl): check the high level config for invalid keywords. with validate maybe? +func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { + obj := jsonconfig.Obj{} + baseUrl := conf.RequiredString("listen") + if baseUrl == "" { + return nil, fmt.Errorf("\"listen\" missing in user config file") + } + tls := conf.RequiredBool("TLS") + scheme := "http" + if tls { + scheme = "https" + } + auth := conf.RequiredString("auth") + if auth == "" { + return nil, fmt.Errorf("\"auth\" missing in user config file") + } + + obj["baseURL"] = scheme + "://" + baseUrl + obj["https"] = tls + obj["auth"] = auth + if tls { + // TODO(mpl): probably need other default paths + obj["TLSCertFile"] = "config/selfgen_cert.pem" + obj["TLSKeyFile"] = "config/selfgen_key.pem" + } + + dbname := conf.OptionalString("dbname", "") + if dbname == "" { + username := os.Getenv("USER") + if username == "" { + return nil, fmt.Errorf("USER env var not set; needed to define dbname") + } + dbname = "camli" + username + } + + secretRing := conf.OptionalString("secring", "") + if secretRing == "" { + secretRing = filepath.Join(osutil.HomeDir(), ".camli", "secring.gpg") + _, err = os.Stat(secretRing) + if err != nil { + return nil, fmt.Errorf("\"secring\" not set in config, and no default secret ring at %s", secretRing) + } + } + + keyId := conf.OptionalString("keyid", "") + if keyId == "" { + // TODO(mpl): where do we get a default keyId from? Brad? + keyId = "26F5ABDA" + } + + blobPath := conf.RequiredString("blobPath") + if blobPath == "" { + return nil, fmt.Errorf("\"blobPath\" not defined in config") + } + indexerPath := "/index-mem/" + + prefixesParams := &configPrefixesParams{ + secretRing: secretRing, + keyId: keyId, + indexerPath: indexerPath, + blobPath: blobPath, + } + + prefixes := genLowLevelPrefixes(prefixesParams) + cacheDir := filepath.Join(blobPath, "/cache") + if err := os.MkdirAll(cacheDir, 0700); err != nil { + return nil, fmt.Errorf("Could not create blobs dir %s: %v", cacheDir, err) + } + + addUiConfig(&prefixes, "/ui/") + + mysql := conf.OptionalString("mysql", "") + mongo := conf.OptionalString("mongo", "") + if mongo != "" && mysql != "" { + return nil, fmt.Errorf("Cannot have both mysql and mongo in config, pick one") + } + if mysql != "" { + addMysqlConfig(&prefixes, dbname, mysql) + } else { + if mongo != "" { + addMongoConfig(&prefixes, dbname, mongo) + } else { + addMemindexConfig(&prefixes) + } + } + + obj["prefixes"] = (map[string]interface{})(prefixes) + + // TODO(mpl): configPath + lowLevelConf = &Config{ + jsonconfig.Obj: obj, + } + return lowLevelConf, nil +} diff --git a/pkg/serverconfig/serverconfig.go b/pkg/serverconfig/serverconfig.go index 62a3047f3..374fe5f2b 100644 --- a/pkg/serverconfig/serverconfig.go +++ b/pkg/serverconfig/serverconfig.go @@ -312,9 +312,6 @@ func (config *Config) InstallHandlers(hi HandlerInstaller, baseURL string, conte if err != nil { return fmt.Errorf("error while configuring auth: %v", err) } - if url := config.OptionalString("baseURL", ""); url != "" { - baseURL = url - } prefixes := config.RequiredObject("prefixes") if err := config.Validate(); err != nil { return fmt.Errorf("configuration error in root object's keys: %v", err) diff --git a/pkg/serverconfig/serverconfig_test.go b/pkg/serverconfig/serverconfig_test.go new file mode 100644 index 000000000..08f934880 --- /dev/null +++ b/pkg/serverconfig/serverconfig_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2012 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serverconfig_test + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "camlistore.org/pkg/jsonconfig" + "camlistore.org/pkg/serverconfig" +) + +func prettyPrint(i interface{}, indent int) { + switch ei := i.(type) { + case jsonconfig.Obj: + for k, v := range ei { + fmt.Printf("\n") + fmt.Printf("%s: ", k) + prettyPrint(v, indent) + } + fmt.Printf("\n") + case map[string]interface{}: + indent++ + for k, v := range ei { + fmt.Printf("\n") + for i := 0; i < indent; i++ { + fmt.Printf(" ") + } + fmt.Printf("%s: ", k) + prettyPrint(v, indent) + } + case []interface{}: + fmt.Printf(" ") + for _, v := range ei { + prettyPrint(v, indent) + } + default: + fmt.Printf("%v, ", i) + } +} + +func TestConfigs(t *testing.T) { + dir, err := os.Open("testdata") + if err != nil { + t.Fatal(err) + } + names, err := dir.Readdirnames(-1) + if err != nil { + t.Fatal(err) + } + for _, name := range names { + if strings.HasSuffix(name, ".json") { + if strings.HasSuffix(name, "-want.json") { + continue + } + testConfig(filepath.Join("testdata", name), t) + } + } +} + +func testConfig(name string, t *testing.T) { + obj, err := jsonconfig.ReadFile("testdata/default.json") + if err != nil { + t.Fatal(err) + } + lowLevelConf, err := serverconfig.GenLowLevelConfig(&serverconfig.Config{jsonconfig.Obj: obj}) + if err != nil { + t.Fatal(err) + } + wantConf, err := jsonconfig.ReadFile("testdata/default-want.json") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(lowLevelConf.Obj, wantConf) { + fmt.Printf("Configurations differ:\n") + fmt.Printf("Generated:") + prettyPrint(lowLevelConf.Obj, 0) + fmt.Printf("\nWant:") + prettyPrint(wantConf, 0) + t.Fail() + } +} diff --git a/pkg/serverconfig/testdata/default-want.json b/pkg/serverconfig/testdata/default-want.json new file mode 100644 index 000000000..cb67349ac --- /dev/null +++ b/pkg/serverconfig/testdata/default-want.json @@ -0,0 +1,90 @@ +{ + "baseURL": "http://localhost:3179", + "auth": "userpass:camlistore:pass3179", + "https": false, + "prefixes": { + "/": { + "handler": "root", + "handlerArgs": { + "stealth": false + } + }, + + "/ui/": { + "handler": "ui", + "handlerArgs": { + "blobRoot": "/bs-and-maybe-also-index/", + "searchRoot": "/my-search/", + "jsonSignRoot": "/sighelper/", + "cache": "/cache/", + "scaledImage": "lrucache" + } + }, + + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "to": "/index-mem/" + } + }, + + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "secretRing": "~/.camli/secring", + "keyId": "26F5ABDA", + "publicKeyDest": "/bs/" + } + }, + + "/bs-and-index/": { + "handler": "storage-replica", + "handlerArgs": { + "backends": ["/bs/", "/index-mem/"] + } + }, + + "/bs-and-maybe-also-index/": { + "handler": "storage-cond", + "handlerArgs": { + "write": { + "if": "isSchema", + "then": "/bs-and-index/", + "else": "/bs/" + }, + "read": "/bs/" + } + }, + + "/bs/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/tmp/blobs" + } + }, + + "/cache/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/tmp/blobs/cache" + } + }, + + "/index-mem/": { + "handler": "storage-memory-only-dev-indexer", + "handlerArgs": { + "blobSource": "/bs/" + } + }, + + "/my-search/": { + "handler": "search", + "handlerArgs": { + "index": "/index-mem/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + } + } + } + +} diff --git a/pkg/serverconfig/testdata/default.json b/pkg/serverconfig/testdata/default.json new file mode 100644 index 000000000..d3fcf0fc1 --- /dev/null +++ b/pkg/serverconfig/testdata/default.json @@ -0,0 +1,11 @@ +{ + "listen": "localhost:3179", + "TLS": false, + "auth": "userpass:camlistore:pass3179", + "blobPath": "/tmp/blobs", + "secring": "~/.camli/secring", + "mysql": "", + "mongo": "", + "s3": "", + "replicateTo": [] +} diff --git a/pkg/webserver/webserver.go b/pkg/webserver/webserver.go index e9604e8cc..3fd60bc8e 100644 --- a/pkg/webserver/webserver.go +++ b/pkg/webserver/webserver.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "crypto/tls" "flag" + "fmt" "io" "log" "net" @@ -30,7 +31,7 @@ import ( "time" ) -var Listen = flag.String("listen", "0.0.0.0:2856", "host:port to listen on, or :0 to auto-select") +var Listen = flag.String("listen", "", "host:port to listen on, or :0 to auto-select") type HandlerPicker func(req *http.Request) (http.HandlerFunc, bool) @@ -96,21 +97,29 @@ func (s *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) { s.mux.ServeHTTP(rw, req) } -func (s *Server) Listen() error { +// Listen starts listening on the given host:port string. +// If listen is empty the *Listen flag will be used instead. +func (s *Server) Listen(listen string) error { if s.listener != nil { return nil } doLog := os.Getenv("TESTING_PORT_WRITE_FD") == "" // Don't make noise during unit tests - base := s.BaseURL() - if doLog { - log.Printf("Starting to listen on %s\n", base) + if listen == "" { + if *Listen == "" { + return fmt.Errorf("Cannot start listening: host:port needs to be provided with the -listen flag") + } + listen = *Listen } var err error - s.listener, err = net.Listen("tcp", *Listen) + s.listener, err = net.Listen("tcp", listen) if err != nil { - log.Fatalf("Failed to listen on %s: %v", *Listen, err) + log.Fatalf("Failed to listen on %s: %v", listen, err) + } + base := s.BaseURL() + if doLog { + log.Printf("Starting to listen on %s\n", base) } if s.enableTLS { @@ -135,7 +144,7 @@ func (s *Server) Listen() error { } func (s *Server) Serve() { - if err := s.Listen(); err != nil { + if err := s.Listen(""); err != nil { log.Fatalf("Listen error: %v", err) } go runTestHarnessIntegration(s.listener) diff --git a/server/camlistored/camlistored.go b/server/camlistored/camlistored.go index 43e6d1763..00c93271c 100644 --- a/server/camlistored/camlistored.go +++ b/server/camlistored/camlistored.go @@ -27,6 +27,7 @@ import ( "io/ioutil" "log" "math/big" + "net" "os" "path/filepath" "runtime" @@ -73,7 +74,7 @@ func exitf(pattern string, args ...interface{}) { } // Mostly copied from $GOROOT/src/pkg/crypto/tls/generate_cert.go -func genSelfTLS() error { +func genSelfTLS(listen string) error { priv, err := rsa.GenerateKey(rand.Reader, 1024) if err != nil { return fmt.Errorf("failed to generate private key: %s", err) @@ -81,13 +82,10 @@ func genSelfTLS() error { now := time.Now() - baseurl := os.Getenv("CAMLI_BASEURL") - if baseurl == "" { - return fmt.Errorf("CAMLI_BASEURL is not set") + hostname, _, err := net.SplitHostPort(listen) + if err != nil { + return fmt.Errorf("splitting listen failed: %q", err) } - split := strings.Split(baseurl, ":") - hostname := split[1] - hostname = hostname[2:len(hostname)] template := x509.Certificate{ SerialNumber: new(big.Int).SetInt64(0), @@ -155,16 +153,19 @@ func checkConfigFile(file string) (newfile string, err error) { return newfile, nil } +// TODO: "auth": "localtcp". See http://code.google.com/p/camlistore/issues/detail?id=50 func newDefaultConfigFile(path string) error { serverConf := `{ - "listen": "localhost:3179", - "TLS": false, - "blobPath": "%BLOBPATH%", - "mysql": "", - "mongo": "", - "s3": "", - "replicateTo": [] + "listen": "localhost:3179", + "TLS": false, + "auth": "userpass:camlistore:pass3179", + "blobPath": "%BLOBPATH%", + "secring": "%SECRING%", + "mysql": "", + "mongo": "", + "s3": "", + "replicateTo": [] } ` blobDir := filepath.Join(osutil.HomeDir(), "var", "camlistore", "blobs") @@ -172,13 +173,16 @@ func newDefaultConfigFile(path string) error { return fmt.Errorf("Could not create default blobs directory: %v", err) } serverConf = strings.Replace(serverConf, "%BLOBPATH%", blobDir, 1) + secRing := filepath.Join(osutil.HomeDir(), ".camli", "secring.gpg") + serverConf = strings.Replace(serverConf, "%SECRING%", secRing, 1) if err := ioutil.WriteFile(path, []byte(serverConf), 0700); err != nil { return fmt.Errorf("Could not create or write default server config: %v", err) } return nil } -func setupTLS(ws *webserver.Server, config *serverconfig.Config) { + +func setupTLS(ws *webserver.Server, config *serverconfig.Config, listen string) { cert, key := config.OptionalString("TLSCertFile", ""), config.OptionalString("TLSKeyFile", "") if !config.OptionalBool("https", true) { return @@ -192,7 +196,7 @@ func setupTLS(ws *webserver.Server, config *serverconfig.Config) { _, err2 := os.Stat(key) if err1 != nil || err2 != nil { if os.IsNotExist(err1) || os.IsNotExist(err2) { - if err := genSelfTLS(); err != nil { + if err := genSelfTLS(listen); err != nil { exitf("Could not generate self-signed TLS cert: %q", err) } } else { @@ -201,7 +205,7 @@ func setupTLS(ws *webserver.Server, config *serverconfig.Config) { } } if cert == "" && key == "" { - err := genSelfTLS() + err := genSelfTLS(listen) if err != nil { exitf("Could not generate self signed creds: %q", err) } @@ -218,22 +222,36 @@ func main() { if err != nil { exitf("Problem with config file: %q", err) } - config, err := serverconfig.Load(file) + conf, err := serverconfig.Load(file) if err != nil { exitf("Could not load server config file %v: %v", file, err) } + config, err := serverconfig.GenLowLevelConfig(conf) + if err != nil { + exitf("Could not gen low level server config: %v", err) + } ws := webserver.New() - baseURL := ws.BaseURL() + baseURL := config.RequiredString("baseURL") + listen := *(webserver.Listen) + if listen == "" { + // if command line was empty, use value in config + listen = strings.TrimLeft(baseURL, "http://") + listen = strings.TrimLeft(listen, "https://") + } else { + // else command line takes precedence + scheme := strings.Split(baseURL, "://")[0] + baseURL = scheme + "://" + listen + } - setupTLS(ws, config) + setupTLS(ws, config, listen) err = config.InstallHandlers(ws, baseURL, nil) if err != nil { exitf("Error parsing config: %v", err) } - ws.Listen() + ws.Listen(listen) if config.UIPath != "" { uiURL := ws.BaseURL() + config.UIPath