From 0ca7712691dcc24c0119aa226fc71583d0ca215a Mon Sep 17 00:00:00 2001 From: Iain Peet Date: Fri, 24 Jun 2011 16:02:51 -0400 Subject: [PATCH] Implemented config file includes. I've implemented the previously stubbed out "_file" config expression. The root JSON object of the included file becomes the value of the field. In addition to searching the CWD and the config dir, the include will search directories listed in the CAMLI_INCLUDE_PATH environment variable. The include path is useful because it gives us the ability to have default configs in git which can be overridden by custom configs which are provided in another path directory which gets searched first. I ended up working on this because, as I was thinking about how to approach supporting Google Storage, I realized that we would need to have secrets in includes; this would be a pain when combined with the current configs and runscripts in GIT. So, one thing lead to another, and now this is done. Change-Id: Ib256859326dfceefed90d0fadf83a76adffd538b --- lib/go/camli/client/config.go | 36 ++------ lib/go/camli/jsonconfig/eval.go | 97 +++++++++++++++++++--- lib/go/camli/jsonconfig/jsonconfig.go | 10 +++ lib/go/camli/osutil/paths.go | 35 ++++++++ lib/go/camli/osutil/paths_test.go | 114 ++++++++++++++++++++++++++ server/go/camlistored/camlistored.go | 34 ++------ 6 files changed, 255 insertions(+), 71 deletions(-) create mode 100644 lib/go/camli/osutil/paths_test.go diff --git a/lib/go/camli/client/config.go b/lib/go/camli/client/config.go index e78bb6a44..19bf53949 100644 --- a/lib/go/camli/client/config.go +++ b/lib/go/camli/client/config.go @@ -18,18 +18,14 @@ package client import ( "flag" - "fmt" - "json" "log" "io/ioutil" "os" "path/filepath" "strings" "sync" - "syscall" "camli/blobref" - "camli/errorutil" "camli/jsonconfig" "camli/jsonsign" "camli/osutil" @@ -49,34 +45,12 @@ var config = make(map[string]interface{}) func parseConfig() { configPath := ConfigFilePath() - f, err := os.Open(configPath) - switch { - case err != nil && err.(*os.PathError).Error.(os.Errno) == syscall.ENOENT: - // TODO: write empty file? - return - case err != nil: - log.Printf("Error opening config file %q: %v", ConfigFilePath(), err) - return - } - defer f.Close() - dj := json.NewDecoder(f) - if err := dj.Decode(&config); err != nil { - extra := "" - if serr, ok := err.(*json.SyntaxError); ok { - if _, serr := f.Seek(0, os.SEEK_SET); serr != nil { - log.Fatalf("seek error: %v", serr) - } - line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset) - extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s", - line, col, serr.Offset, highlight) - } - log.Fatalf("error parsing JSON object in config file %s%s\n%v", - ConfigFilePath(), extra, err) - } - if err := jsonconfig.EvaluateExpressions(config); err != nil { - log.Fatalf("error expanding JSON config expressions in %s: %v", configPath, err) - } + var err os.Error + if config, err = jsonconfig.ReadFile(configPath); err != nil { + log.Fatal(err.String()) + return + } } func cleanServer(server string) string { diff --git a/lib/go/camli/jsonconfig/eval.go b/lib/go/camli/jsonconfig/eval.go index 9c0377555..7028a63db 100644 --- a/lib/go/camli/jsonconfig/eval.go +++ b/lib/go/camli/jsonconfig/eval.go @@ -17,14 +17,77 @@ limitations under the License. package jsonconfig import ( - "os" + "container/vector" "fmt" + "json" + "log" + "os" + "path/filepath" "regexp" + + "camli/errorutil" + "camli/osutil" ) + +// State for config parsing and expression evalutaion +type configParser struct { + RootJson Obj + + touchedFiles map[string]bool + includeStack vector.StringVector +} + +// Validates variable names for config _env expresssions var envPattern = regexp.MustCompile(`\$\{[A-Za-z0-9_]+\}`) -func EvaluateExpressions(m map[string]interface{}) os.Error { +// Decodes and evaluates a json config file, watching for include cycles. +func (c *configParser) recursiveReadJson(configPath string) (decodedObject map[string]interface{}, err os.Error) { + + configPath, err = filepath.Abs(configPath) + if err != nil { + return nil, fmt.Errorf("Failed to expand absolute path for %s", configPath) + } + if c.touchedFiles[configPath] { + return nil, fmt.Errorf("configParser include cycle detected reading config: %v", + configPath) + } + c.touchedFiles[configPath] = true + + c.includeStack.Push(configPath) + defer c.includeStack.Pop() + + var f *os.File + if f, err = os.Open(configPath); err != nil { + return nil, fmt.Errorf("Failed to open config: %s, %v", configPath, err) + } + defer f.Close() + + decodedObject = make(map[string]interface{}) + dj := json.NewDecoder(f) + if err = dj.Decode(&decodedObject); err != nil { + extra := "" + if serr, ok := err.(*json.SyntaxError); ok { + if _, serr := f.Seek(0, os.SEEK_SET); serr != nil { + log.Fatalf("seek error: %v", serr) + } + line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset) + extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s", + line, col, serr.Offset, highlight) + } + return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v", + f.Name(), extra, err) + } + + if err = c.evaluateExpressions(decodedObject); err != nil { + return nil, fmt.Errorf("error expanding JSON config expressions in %s:\n%v", + f.Name(), err) + } + + return decodedObject, nil +} + +func (c *configParser) evaluateExpressions(m map[string]interface{}) os.Error { for k, ei := range m { switch subval := ei.(type) { case string: @@ -37,24 +100,24 @@ func EvaluateExpressions(m map[string]interface{}) os.Error { if len(subval) == 0 { continue } - var expander func(v []interface{}) (interface{}, os.Error) + var expander func(c *configParser, v []interface{}) (interface{}, os.Error) if firstString, ok := subval[0].(string); ok { switch firstString { case "_env": - expander = expandEnv - case "_file": - expander = expandFile + expander = (*configParser).expandEnv + case "_fileobj": + expander = (*configParser).expandFile } } if expander != nil { - newval, err := expander(subval[1:]) + newval, err := expander(c, subval[1:]) if err != nil { return err } m[k] = newval } case map[string]interface{}: - if err := EvaluateExpressions(subval); err != nil { + if err := c.evaluateExpressions(subval); err != nil { return err } default: @@ -67,7 +130,7 @@ func EvaluateExpressions(m map[string]interface{}) os.Error { // Permit either: // ["_env", "VARIABLE"] (required to be set) // or ["_env", "VARIABLE", "default_value"] -func expandEnv(v []interface{}) (interface{}, os.Error) { +func (c *configParser) expandEnv(v []interface{}) (interface{}, os.Error) { hasDefault := false def := "" if len(v) < 1 || len(v) > 2 { @@ -99,6 +162,18 @@ func expandEnv(v []interface{}) (interface{}, os.Error) { return expanded, err } -func expandFile(v []interface{}) (interface{}, os.Error) { - return "", os.NewError("_file not implemented") +func (c *configParser) expandFile(v []interface{}) (exp interface{}, err os.Error) { + if len(v) != 1 { + return "", fmt.Errorf("_file expansion expected 1 arg, got %d", len(v)) + } + var incPath string + if incPath, err = osutil.FindCamliInclude(v[0].(string)); err != nil { + return "", fmt.Errorf("Included config does not exist: %v", v[0]) + } + if exp, err = c.recursiveReadJson(incPath); err != nil { + return "", fmt.Errorf("In file included from %s:\n%v", + c.includeStack.Last(), err) + } + return exp, nil } + diff --git a/lib/go/camli/jsonconfig/jsonconfig.go b/lib/go/camli/jsonconfig/jsonconfig.go index cfb9c2290..87c51f6ab 100644 --- a/lib/go/camli/jsonconfig/jsonconfig.go +++ b/lib/go/camli/jsonconfig/jsonconfig.go @@ -27,6 +27,16 @@ import ( // Obj is a JSON configuration map. type Obj map[string]interface{} +// Reads json config data from the specified open file, expanding +// all expressions +func ReadFile(configPath string) (Obj, os.Error) { + var c configParser + var err os.Error + c.touchedFiles = make(map[string]bool) + c.RootJson, err = c.recursiveReadJson(configPath) + return c.RootJson, err +} + func (jc Obj) RequiredObject(key string) Obj { return jc.obj(key, false) } diff --git a/lib/go/camli/osutil/paths.go b/lib/go/camli/osutil/paths.go index a0e011dcc..e240a7f03 100644 --- a/lib/go/camli/osutil/paths.go +++ b/lib/go/camli/osutil/paths.go @@ -20,6 +20,7 @@ import ( "os" "path/filepath" "runtime" + "strings" ) func HomeDir() string { @@ -43,3 +44,37 @@ func UserServerConfigPath() string { return filepath.Join(CamliConfigDir(), "serverconfig") } +// Find the correct absolute path corresponding to a relative path, +// searching the following sequence of directories: +// 1. Working Directory +// 2. CAMLI_CONFIG_DIR (deprecated, will complain if this is on env) +// 3. (windows only) APPDATA/camli +// 4. All directories in CAMLI_INCLUDE_PATH (standard PATH form for OS) +func FindCamliInclude(configFile string) (absPath string, err os.Error) { + // Try to open as absolute / relative to CWD + _, err = os.Stat(configFile) + if err == nil { + return configFile, nil; + } + if filepath.IsAbs(configFile) { + // End of the line for absolute path + return "", err; + } + + // Try the config dir + configDir := CamliConfigDir(); + if _, err = os.Stat(filepath.Join(configDir, configFile)); err == nil { + return filepath.Join(configDir, configFile), nil + } + + // Finally, search CAMLI_INCLUDE_PATH + p := os.Getenv("CAMLI_INCLUDE_PATH"); + for _, d := range strings.Split(p, string(filepath.ListSeparator)) { + if _, err = os.Stat(filepath.Join(d, configFile)); err == nil { + return filepath.Join(d, configFile), nil + } + } + + return "", os.ENOENT +} + diff --git a/lib/go/camli/osutil/paths_test.go b/lib/go/camli/osutil/paths_test.go new file mode 100644 index 000000000..f28f0e1b6 --- /dev/null +++ b/lib/go/camli/osutil/paths_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package osutil + +import ( + "fmt" + "os" + "syscall" + "testing" +) + +// Creates a file with the content "test" at path +func createTestInclude(path string) os.Error { + // Create a config file for OpenCamliInclude to play with + cf, e := os.Create(path) + if e != nil { + return e + } + fmt.Fprintf(cf, "test") + return cf.Close() +} + +// Calls OpenCamliInclude to open path, and checks that it containts "test" +func checkOpen(t *testing.T, path string) { + found, e := FindCamliInclude(path) + if e != nil { + t.Errorf("Failed to find %v", path) + return + } + var file *os.File + file, e = os.Open(found) + if e != nil { + t.Errorf("Failed to open %v", path) + } else { + var d [10]byte + if n, _ := file.Read(d[:]); n != 4 { + t.Errorf("Read incorrect number of chars from test.config, wrong file?") + } + if string(d[0:4]) != "test" { + t.Errorf("Wrong test file content: %v", string(d[0:4])) + } + file.Close() + } +} + +// Test for error when file doesn't exist +func TestOpenCamliIncludeNoFile(t *testing.T) { + // Test that error occurs if no such file + const notExist = "this_config_doesnt_exist.config" + _, e := FindCamliInclude(notExist) + if e == nil { + t.Errorf("Successfully opened config which doesn't exist: %v", notExist) + } +} + +// Test for when a file exists in CWD +func TestOpenCamliIncludeCWD(t *testing.T) { + const path string = "TestOpenCamliIncludeCWD.config" + if e := createTestInclude(path); e != nil { + t.Errorf("Couldn't create test config file, aborting test: %v", e) + return + } + defer syscall.Remove(path) + + checkOpen(t, path) +} + +// Test for when a file exists in CAMLI_CONFIG_DIR +func TestOpenCamliIncludeDir(t *testing.T) { + const name string = "TestOpenCamliIncludeDir.config" + if e := createTestInclude("/tmp/" + name); e != nil { + t.Errorf("Couldn't create test config file, aborting test: %v", e) + return + } + defer os.Remove("/tmp/" + name) + os.Setenv("CAMLI_CONFIG_DIR", "/tmp") + defer os.Setenv("CAMLI_CONFIG_DIR", "") + + checkOpen(t, name) +} + +// Test for when a file exits in CAMLI_INCLUDE_PATH +func TestOpenCamliIncludePath(t *testing.T) { + const name string = "TestOpenCamliIncludePath.config" + if e := createTestInclude("/tmp/" + name); e != nil { + t.Errorf("Couldn't create test config file, aborting test: %v", e) + return + } + defer syscall.Unlink("/tmp/" + name) + defer os.Setenv("CAMLI_INCLUDE_PATH", "") + + os.Setenv("CAMLI_INCLUDE_PATH", "/tmp") + checkOpen(t, name) + + os.Setenv("CAMLI_INCLUDE_PATH", "/not/a/camli/config/dir:/tmp") + checkOpen(t, name) + + os.Setenv("CAMLI_INCLUDE_PATH", "/not/a/camli/config/dir:/tmp:/another/fake/camli/dir") + checkOpen(t, name) +} diff --git a/server/go/camlistored/camlistored.go b/server/go/camlistored/camlistored.go index f6809c696..fa1959112 100644 --- a/server/go/camlistored/camlistored.go +++ b/server/go/camlistored/camlistored.go @@ -20,7 +20,6 @@ import ( "flag" "fmt" "http" - "json" "log" "path/filepath" "strings" @@ -29,7 +28,6 @@ import ( "camli/auth" "camli/blobserver" "camli/blobserver/handlers" - "camli/errorutil" "camli/httputil" "camli/jsonconfig" "camli/osutil" @@ -45,7 +43,6 @@ import ( _ "camli/mysqlindexer" // indexer, but uses storage interface // Handlers: _ "camli/search" - ) var flagConfigFile = flag.String("configfile", "serverconfig", @@ -162,36 +159,15 @@ func main() { if !filepath.IsAbs(configPath) { configPath = filepath.Join(osutil.CamliConfigDir(), configPath) } - f, err := os.Open(configPath) + + config, err := jsonconfig.ReadFile(configPath) if err != nil { - exitFailure("error opening %s: %v", configPath, err) - } - defer f.Close() - dj := json.NewDecoder(f) - rootjson := make(map[string]interface{}) - if err = dj.Decode(&rootjson); err != nil { - extra := "" - if serr, ok := err.(*json.SyntaxError); ok { - if _, serr := f.Seek(0, os.SEEK_SET); serr != nil { - log.Fatalf("seek error: %v", serr) - } - line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset) - extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s", - line, col, serr.Offset, highlight) - } - exitFailure("error parsing JSON object in config file %s%s\n%v", - osutil.UserServerConfigPath(), extra, err) - } - if err := jsonconfig.EvaluateExpressions(rootjson); err != nil { - exitFailure("error expanding JSON config expressions in %s: %v", configPath, err) + exitFailure("%v", err) } ws := webserver.New() baseURL := ws.BaseURL() - // Root configuration - config := jsonconfig.Obj(rootjson) - { cert, key := config.OptionalString("TLSCertFile", ""), config.OptionalString("TLSKeyFile", "") if (cert != "") != (key != "") { @@ -207,7 +183,7 @@ func main() { baseURL = url } prefixes := config.RequiredObject("prefixes") - if err := config.Validate(); err != nil { + if err = config.Validate(); err != nil { exitFailure("configuration error in root object's keys in %s: %v", configPath, err) } @@ -227,7 +203,7 @@ func main() { } pmap, ok := vei.(map[string]interface{}) if !ok { - exitFailure("prefix %q value isn't an object", prefix) + exitFailure("prefix %q value is a %T, not an object", prefix, vei) } pconf := jsonconfig.Obj(pmap) handlerType := pconf.RequiredString("handler")