diff --git a/lib/go/camli/client/config.go b/lib/go/camli/client/config.go index e78bb6a44..19bf53949 100644 --- a/lib/go/camli/client/config.go +++ b/lib/go/camli/client/config.go @@ -18,18 +18,14 @@ package client import ( "flag" - "fmt" - "json" "log" "io/ioutil" "os" "path/filepath" "strings" "sync" - "syscall" "camli/blobref" - "camli/errorutil" "camli/jsonconfig" "camli/jsonsign" "camli/osutil" @@ -49,34 +45,12 @@ var config = make(map[string]interface{}) func parseConfig() { configPath := ConfigFilePath() - f, err := os.Open(configPath) - switch { - case err != nil && err.(*os.PathError).Error.(os.Errno) == syscall.ENOENT: - // TODO: write empty file? - return - case err != nil: - log.Printf("Error opening config file %q: %v", ConfigFilePath(), err) - return - } - defer f.Close() - dj := json.NewDecoder(f) - if err := dj.Decode(&config); err != nil { - extra := "" - if serr, ok := err.(*json.SyntaxError); ok { - if _, serr := f.Seek(0, os.SEEK_SET); serr != nil { - log.Fatalf("seek error: %v", serr) - } - line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset) - extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s", - line, col, serr.Offset, highlight) - } - log.Fatalf("error parsing JSON object in config file %s%s\n%v", - ConfigFilePath(), extra, err) - } - if err := jsonconfig.EvaluateExpressions(config); err != nil { - log.Fatalf("error expanding JSON config expressions in %s: %v", configPath, err) - } + var err os.Error + if config, err = jsonconfig.ReadFile(configPath); err != nil { + log.Fatal(err.String()) + return + } } func cleanServer(server string) string { diff --git a/lib/go/camli/jsonconfig/eval.go b/lib/go/camli/jsonconfig/eval.go index 9c0377555..7028a63db 100644 --- a/lib/go/camli/jsonconfig/eval.go +++ b/lib/go/camli/jsonconfig/eval.go @@ -17,14 +17,77 @@ limitations under the License. package jsonconfig import ( - "os" + "container/vector" "fmt" + "json" + "log" + "os" + "path/filepath" "regexp" + + "camli/errorutil" + "camli/osutil" ) + +// State for config parsing and expression evalutaion +type configParser struct { + RootJson Obj + + touchedFiles map[string]bool + includeStack vector.StringVector +} + +// Validates variable names for config _env expresssions var envPattern = regexp.MustCompile(`\$\{[A-Za-z0-9_]+\}`) -func EvaluateExpressions(m map[string]interface{}) os.Error { +// Decodes and evaluates a json config file, watching for include cycles. +func (c *configParser) recursiveReadJson(configPath string) (decodedObject map[string]interface{}, err os.Error) { + + configPath, err = filepath.Abs(configPath) + if err != nil { + return nil, fmt.Errorf("Failed to expand absolute path for %s", configPath) + } + if c.touchedFiles[configPath] { + return nil, fmt.Errorf("configParser include cycle detected reading config: %v", + configPath) + } + c.touchedFiles[configPath] = true + + c.includeStack.Push(configPath) + defer c.includeStack.Pop() + + var f *os.File + if f, err = os.Open(configPath); err != nil { + return nil, fmt.Errorf("Failed to open config: %s, %v", configPath, err) + } + defer f.Close() + + decodedObject = make(map[string]interface{}) + dj := json.NewDecoder(f) + if err = dj.Decode(&decodedObject); err != nil { + extra := "" + if serr, ok := err.(*json.SyntaxError); ok { + if _, serr := f.Seek(0, os.SEEK_SET); serr != nil { + log.Fatalf("seek error: %v", serr) + } + line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset) + extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s", + line, col, serr.Offset, highlight) + } + return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v", + f.Name(), extra, err) + } + + if err = c.evaluateExpressions(decodedObject); err != nil { + return nil, fmt.Errorf("error expanding JSON config expressions in %s:\n%v", + f.Name(), err) + } + + return decodedObject, nil +} + +func (c *configParser) evaluateExpressions(m map[string]interface{}) os.Error { for k, ei := range m { switch subval := ei.(type) { case string: @@ -37,24 +100,24 @@ func EvaluateExpressions(m map[string]interface{}) os.Error { if len(subval) == 0 { continue } - var expander func(v []interface{}) (interface{}, os.Error) + var expander func(c *configParser, v []interface{}) (interface{}, os.Error) if firstString, ok := subval[0].(string); ok { switch firstString { case "_env": - expander = expandEnv - case "_file": - expander = expandFile + expander = (*configParser).expandEnv + case "_fileobj": + expander = (*configParser).expandFile } } if expander != nil { - newval, err := expander(subval[1:]) + newval, err := expander(c, subval[1:]) if err != nil { return err } m[k] = newval } case map[string]interface{}: - if err := EvaluateExpressions(subval); err != nil { + if err := c.evaluateExpressions(subval); err != nil { return err } default: @@ -67,7 +130,7 @@ func EvaluateExpressions(m map[string]interface{}) os.Error { // Permit either: // ["_env", "VARIABLE"] (required to be set) // or ["_env", "VARIABLE", "default_value"] -func expandEnv(v []interface{}) (interface{}, os.Error) { +func (c *configParser) expandEnv(v []interface{}) (interface{}, os.Error) { hasDefault := false def := "" if len(v) < 1 || len(v) > 2 { @@ -99,6 +162,18 @@ func expandEnv(v []interface{}) (interface{}, os.Error) { return expanded, err } -func expandFile(v []interface{}) (interface{}, os.Error) { - return "", os.NewError("_file not implemented") +func (c *configParser) expandFile(v []interface{}) (exp interface{}, err os.Error) { + if len(v) != 1 { + return "", fmt.Errorf("_file expansion expected 1 arg, got %d", len(v)) + } + var incPath string + if incPath, err = osutil.FindCamliInclude(v[0].(string)); err != nil { + return "", fmt.Errorf("Included config does not exist: %v", v[0]) + } + if exp, err = c.recursiveReadJson(incPath); err != nil { + return "", fmt.Errorf("In file included from %s:\n%v", + c.includeStack.Last(), err) + } + return exp, nil } + diff --git a/lib/go/camli/jsonconfig/jsonconfig.go b/lib/go/camli/jsonconfig/jsonconfig.go index cfb9c2290..87c51f6ab 100644 --- a/lib/go/camli/jsonconfig/jsonconfig.go +++ b/lib/go/camli/jsonconfig/jsonconfig.go @@ -27,6 +27,16 @@ import ( // Obj is a JSON configuration map. type Obj map[string]interface{} +// Reads json config data from the specified open file, expanding +// all expressions +func ReadFile(configPath string) (Obj, os.Error) { + var c configParser + var err os.Error + c.touchedFiles = make(map[string]bool) + c.RootJson, err = c.recursiveReadJson(configPath) + return c.RootJson, err +} + func (jc Obj) RequiredObject(key string) Obj { return jc.obj(key, false) } diff --git a/lib/go/camli/osutil/paths.go b/lib/go/camli/osutil/paths.go index a0e011dcc..e240a7f03 100644 --- a/lib/go/camli/osutil/paths.go +++ b/lib/go/camli/osutil/paths.go @@ -20,6 +20,7 @@ import ( "os" "path/filepath" "runtime" + "strings" ) func HomeDir() string { @@ -43,3 +44,37 @@ func UserServerConfigPath() string { return filepath.Join(CamliConfigDir(), "serverconfig") } +// Find the correct absolute path corresponding to a relative path, +// searching the following sequence of directories: +// 1. Working Directory +// 2. CAMLI_CONFIG_DIR (deprecated, will complain if this is on env) +// 3. (windows only) APPDATA/camli +// 4. All directories in CAMLI_INCLUDE_PATH (standard PATH form for OS) +func FindCamliInclude(configFile string) (absPath string, err os.Error) { + // Try to open as absolute / relative to CWD + _, err = os.Stat(configFile) + if err == nil { + return configFile, nil; + } + if filepath.IsAbs(configFile) { + // End of the line for absolute path + return "", err; + } + + // Try the config dir + configDir := CamliConfigDir(); + if _, err = os.Stat(filepath.Join(configDir, configFile)); err == nil { + return filepath.Join(configDir, configFile), nil + } + + // Finally, search CAMLI_INCLUDE_PATH + p := os.Getenv("CAMLI_INCLUDE_PATH"); + for _, d := range strings.Split(p, string(filepath.ListSeparator)) { + if _, err = os.Stat(filepath.Join(d, configFile)); err == nil { + return filepath.Join(d, configFile), nil + } + } + + return "", os.ENOENT +} + diff --git a/lib/go/camli/osutil/paths_test.go b/lib/go/camli/osutil/paths_test.go new file mode 100644 index 000000000..f28f0e1b6 --- /dev/null +++ b/lib/go/camli/osutil/paths_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package osutil + +import ( + "fmt" + "os" + "syscall" + "testing" +) + +// Creates a file with the content "test" at path +func createTestInclude(path string) os.Error { + // Create a config file for OpenCamliInclude to play with + cf, e := os.Create(path) + if e != nil { + return e + } + fmt.Fprintf(cf, "test") + return cf.Close() +} + +// Calls OpenCamliInclude to open path, and checks that it containts "test" +func checkOpen(t *testing.T, path string) { + found, e := FindCamliInclude(path) + if e != nil { + t.Errorf("Failed to find %v", path) + return + } + var file *os.File + file, e = os.Open(found) + if e != nil { + t.Errorf("Failed to open %v", path) + } else { + var d [10]byte + if n, _ := file.Read(d[:]); n != 4 { + t.Errorf("Read incorrect number of chars from test.config, wrong file?") + } + if string(d[0:4]) != "test" { + t.Errorf("Wrong test file content: %v", string(d[0:4])) + } + file.Close() + } +} + +// Test for error when file doesn't exist +func TestOpenCamliIncludeNoFile(t *testing.T) { + // Test that error occurs if no such file + const notExist = "this_config_doesnt_exist.config" + _, e := FindCamliInclude(notExist) + if e == nil { + t.Errorf("Successfully opened config which doesn't exist: %v", notExist) + } +} + +// Test for when a file exists in CWD +func TestOpenCamliIncludeCWD(t *testing.T) { + const path string = "TestOpenCamliIncludeCWD.config" + if e := createTestInclude(path); e != nil { + t.Errorf("Couldn't create test config file, aborting test: %v", e) + return + } + defer syscall.Remove(path) + + checkOpen(t, path) +} + +// Test for when a file exists in CAMLI_CONFIG_DIR +func TestOpenCamliIncludeDir(t *testing.T) { + const name string = "TestOpenCamliIncludeDir.config" + if e := createTestInclude("/tmp/" + name); e != nil { + t.Errorf("Couldn't create test config file, aborting test: %v", e) + return + } + defer os.Remove("/tmp/" + name) + os.Setenv("CAMLI_CONFIG_DIR", "/tmp") + defer os.Setenv("CAMLI_CONFIG_DIR", "") + + checkOpen(t, name) +} + +// Test for when a file exits in CAMLI_INCLUDE_PATH +func TestOpenCamliIncludePath(t *testing.T) { + const name string = "TestOpenCamliIncludePath.config" + if e := createTestInclude("/tmp/" + name); e != nil { + t.Errorf("Couldn't create test config file, aborting test: %v", e) + return + } + defer syscall.Unlink("/tmp/" + name) + defer os.Setenv("CAMLI_INCLUDE_PATH", "") + + os.Setenv("CAMLI_INCLUDE_PATH", "/tmp") + checkOpen(t, name) + + os.Setenv("CAMLI_INCLUDE_PATH", "/not/a/camli/config/dir:/tmp") + checkOpen(t, name) + + os.Setenv("CAMLI_INCLUDE_PATH", "/not/a/camli/config/dir:/tmp:/another/fake/camli/dir") + checkOpen(t, name) +} diff --git a/server/go/camlistored/camlistored.go b/server/go/camlistored/camlistored.go index f6809c696..fa1959112 100644 --- a/server/go/camlistored/camlistored.go +++ b/server/go/camlistored/camlistored.go @@ -20,7 +20,6 @@ import ( "flag" "fmt" "http" - "json" "log" "path/filepath" "strings" @@ -29,7 +28,6 @@ import ( "camli/auth" "camli/blobserver" "camli/blobserver/handlers" - "camli/errorutil" "camli/httputil" "camli/jsonconfig" "camli/osutil" @@ -45,7 +43,6 @@ import ( _ "camli/mysqlindexer" // indexer, but uses storage interface // Handlers: _ "camli/search" - ) var flagConfigFile = flag.String("configfile", "serverconfig", @@ -162,36 +159,15 @@ func main() { if !filepath.IsAbs(configPath) { configPath = filepath.Join(osutil.CamliConfigDir(), configPath) } - f, err := os.Open(configPath) + + config, err := jsonconfig.ReadFile(configPath) if err != nil { - exitFailure("error opening %s: %v", configPath, err) - } - defer f.Close() - dj := json.NewDecoder(f) - rootjson := make(map[string]interface{}) - if err = dj.Decode(&rootjson); err != nil { - extra := "" - if serr, ok := err.(*json.SyntaxError); ok { - if _, serr := f.Seek(0, os.SEEK_SET); serr != nil { - log.Fatalf("seek error: %v", serr) - } - line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset) - extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s", - line, col, serr.Offset, highlight) - } - exitFailure("error parsing JSON object in config file %s%s\n%v", - osutil.UserServerConfigPath(), extra, err) - } - if err := jsonconfig.EvaluateExpressions(rootjson); err != nil { - exitFailure("error expanding JSON config expressions in %s: %v", configPath, err) + exitFailure("%v", err) } ws := webserver.New() baseURL := ws.BaseURL() - // Root configuration - config := jsonconfig.Obj(rootjson) - { cert, key := config.OptionalString("TLSCertFile", ""), config.OptionalString("TLSKeyFile", "") if (cert != "") != (key != "") { @@ -207,7 +183,7 @@ func main() { baseURL = url } prefixes := config.RequiredObject("prefixes") - if err := config.Validate(); err != nil { + if err = config.Validate(); err != nil { exitFailure("configuration error in root object's keys in %s: %v", configPath, err) } @@ -227,7 +203,7 @@ func main() { } pmap, ok := vei.(map[string]interface{}) if !ok { - exitFailure("prefix %q value isn't an object", prefix) + exitFailure("prefix %q value is a %T, not an object", prefix, vei) } pconf := jsonconfig.Obj(pmap) handlerType := pconf.RequiredString("handler")