Merge "Implemented config file includes."

lib/go/camli/client/config.go

@@ -18,18 +18,14 @@ package client
 
 import (
 	"flag"
-	"fmt"
-	"json"
 	"log"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
-	"syscall"
 
 	"camli/blobref"
-	"camli/errorutil"
 	"camli/jsonconfig"
 	"camli/jsonsign"
 	"camli/osutil"
@@ -49,34 +45,12 @@ var config = make(map[string]interface{})
 
 func parseConfig() {
 	configPath := ConfigFilePath()
-	f, err := os.Open(configPath)
-	switch {
-	case err != nil && err.(*os.PathError).Error.(os.Errno) == syscall.ENOENT:
-		// TODO: write empty file?
-		return
-	case err != nil:
-		log.Printf("Error opening config file %q: %v", ConfigFilePath(), err)
-		return
-	}
-	defer f.Close()
-	dj := json.NewDecoder(f)
-	if err := dj.Decode(&config); err != nil {
-		extra := ""
-		if serr, ok := err.(*json.SyntaxError); ok {
-			if _, serr := f.Seek(0, os.SEEK_SET); serr != nil {
-				log.Fatalf("seek error: %v", serr)
-			}
-			line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset)
-			extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
-				line, col, serr.Offset, highlight)
-		}
-		log.Fatalf("error parsing JSON object in config file %s%s\n%v",
-			ConfigFilePath(), extra, err)
-	}
-	if err := jsonconfig.EvaluateExpressions(config); err != nil {
-		log.Fatalf("error expanding JSON config expressions in %s: %v", configPath, err)
-	}
 
+	var err os.Error
+	if config, err = jsonconfig.ReadFile(configPath); err != nil {
+		log.Fatal(err.String())
+		return
+	}
 }
 
 func cleanServer(server string) string {

lib/go/camli/jsonconfig/eval.go

@@ -17,14 +17,77 @@ limitations under the License.
 package jsonconfig
 
 import (
-	"os"
+	"container/vector"
 	"fmt"
+	"json"
+	"log"
+	"os"
+	"path/filepath"
 	"regexp"
+
+	"camli/errorutil"
+	"camli/osutil"
 )
 
+
+// State for config parsing and expression evalutaion
+type configParser struct {
+	RootJson     Obj
+
+	touchedFiles map[string]bool
+	includeStack vector.StringVector
+}
+
+// Validates variable names for config _env expresssions
 var envPattern = regexp.MustCompile(`\$\{[A-Za-z0-9_]+\}`)
 
-func EvaluateExpressions(m map[string]interface{}) os.Error {
+// Decodes and evaluates a json config file, watching for include cycles.
+func (c *configParser) recursiveReadJson(configPath string) (decodedObject map[string]interface{}, err os.Error) {
+
+	configPath, err = filepath.Abs(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to expand absolute path for %s", configPath)
+	}
+	if c.touchedFiles[configPath] {
+		return nil, fmt.Errorf("configParser include cycle detected reading config: %v",
+			configPath)
+	}
+	c.touchedFiles[configPath] = true
+
+	c.includeStack.Push(configPath)
+	defer c.includeStack.Pop()
+
+	var f *os.File
+	if f, err = os.Open(configPath); err != nil {
+		return nil, fmt.Errorf("Failed to open config: %s, %v", configPath, err)
+	}
+	defer f.Close()
+
+	decodedObject = make(map[string]interface{})
+	dj := json.NewDecoder(f)
+	if err = dj.Decode(&decodedObject); err != nil {
+		extra := ""
+		if serr, ok := err.(*json.SyntaxError); ok {
+			if _, serr := f.Seek(0, os.SEEK_SET); serr != nil {
+				log.Fatalf("seek error: %v", serr)
+			}
+			line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset)
+			extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
+				line, col, serr.Offset, highlight)
+		}
+		return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v",
+			f.Name(), extra, err)
+	}
+
+	if err = c.evaluateExpressions(decodedObject); err != nil {
+		return nil, fmt.Errorf("error expanding JSON config expressions in %s:\n%v",
+			f.Name(), err)
+	}
+
+	return decodedObject, nil
+}
+
+func (c *configParser) evaluateExpressions(m map[string]interface{}) os.Error {
 	for k, ei := range m {
 		switch subval := ei.(type) {
 		case string:
@@ -37,24 +100,24 @@ func EvaluateExpressions(m map[string]interface{}) os.Error {
 			if len(subval) == 0 {
 				continue
 			}
-			var expander func(v []interface{}) (interface{}, os.Error)
+			var expander func(c *configParser, v []interface{}) (interface{}, os.Error)
 			if firstString, ok := subval[0].(string); ok {
 				switch firstString {
 				case "_env":
-					expander = expandEnv
-				case "_file":
-					expander = expandFile
+					expander = (*configParser).expandEnv
+				case "_fileobj":
+					expander = (*configParser).expandFile
 				}
 			}
 			if expander != nil {
-				newval, err := expander(subval[1:])
+				newval, err := expander(c, subval[1:])
 				if err != nil {
 					return err
 				}
 				m[k] = newval
 			}
 		case map[string]interface{}:
-			if err := EvaluateExpressions(subval); err != nil {
+			if err := c.evaluateExpressions(subval); err != nil {
 				return err
 			}
 		default:
@@ -67,7 +130,7 @@ func EvaluateExpressions(m map[string]interface{}) os.Error {
 // Permit either:
 //    ["_env", "VARIABLE"] (required to be set)
 // or ["_env", "VARIABLE", "default_value"]
-func expandEnv(v []interface{}) (interface{}, os.Error) {
+func (c *configParser) expandEnv(v []interface{}) (interface{}, os.Error) {
 	hasDefault := false
 	def := ""
 	if len(v) < 1 || len(v) > 2 {
@@ -99,6 +162,18 @@ func expandEnv(v []interface{}) (interface{}, os.Error) {
 	return expanded, err
 }
 
-func expandFile(v []interface{}) (interface{}, os.Error) {
-	return "", os.NewError("_file not implemented")
+func (c *configParser) expandFile(v []interface{}) (exp interface{}, err os.Error) {
+	if len(v) != 1 {
+		return "", fmt.Errorf("_file expansion expected 1 arg, got %d", len(v))
+	}
+	var incPath string
+	if incPath, err = osutil.FindCamliInclude(v[0].(string)); err != nil {
+		return "", fmt.Errorf("Included config does not exist: %v", v[0])
+	}
+	if exp, err = c.recursiveReadJson(incPath); err != nil {
+		return "", fmt.Errorf("In file included from %s:\n%v",
+			c.includeStack.Last(), err)
+	}
+	return exp, nil
 }
+

lib/go/camli/jsonconfig/jsonconfig.go

@@ -27,6 +27,16 @@ import (
 // Obj is a JSON configuration map.
 type Obj map[string]interface{}
 
+// Reads json config data from the specified open file, expanding
+// all expressions 
+func ReadFile(configPath string) (Obj, os.Error) {
+	var c configParser
+	var err os.Error
+	c.touchedFiles = make(map[string]bool)
+	c.RootJson, err = c.recursiveReadJson(configPath)
+	return c.RootJson, err
+}
+
 func (jc Obj) RequiredObject(key string) Obj {
 	return jc.obj(key, false)
 }

lib/go/camli/osutil/paths.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 )
 
 func HomeDir() string {
@@ -43,3 +44,37 @@ func UserServerConfigPath() string {
 	return filepath.Join(CamliConfigDir(), "serverconfig")
 }
 
+// Find the correct absolute path corresponding to a relative path, 
+// searching the following sequence of directories:
+// 1. Working Directory
+// 2. CAMLI_CONFIG_DIR (deprecated, will complain if this is on env)
+// 3. (windows only) APPDATA/camli
+// 4. All directories in CAMLI_INCLUDE_PATH (standard PATH form for OS)
+func FindCamliInclude(configFile string) (absPath string, err os.Error) {
+	// Try to open as absolute / relative to CWD
+	_, err = os.Stat(configFile)
+	if err == nil {
+		return configFile, nil;
+	}
+	if filepath.IsAbs(configFile) {
+		// End of the line for absolute path
+		return "", err;
+	}
+
+	// Try the config dir
+	configDir := CamliConfigDir();
+	if _, err = os.Stat(filepath.Join(configDir, configFile)); err == nil {
+		return filepath.Join(configDir, configFile), nil
+	}
+
+	// Finally, search CAMLI_INCLUDE_PATH
+	p := os.Getenv("CAMLI_INCLUDE_PATH");
+	for _, d := range strings.Split(p, string(filepath.ListSeparator)) {
+		if _, err = os.Stat(filepath.Join(d, configFile)); err == nil {
+			return filepath.Join(d, configFile), nil
+		}
+	}
+
+	return "", os.ENOENT
+}
+

lib/go/camli/osutil/paths_test.go

@@ -0,0 +1,114 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package osutil
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"testing"
+)
+
+// Creates a file with the content "test" at path
+func createTestInclude(path string) os.Error {
+	// Create a config file for OpenCamliInclude to play with
+	cf, e := os.Create(path)
+	if e != nil {
+		return e
+	}
+	fmt.Fprintf(cf, "test")
+	return cf.Close()
+}
+
+// Calls OpenCamliInclude to open path, and checks that it containts "test"
+func checkOpen(t *testing.T, path string) {
+	found, e := FindCamliInclude(path)
+	if e != nil {
+		t.Errorf("Failed to find %v", path)
+		return
+	}
+	var file *os.File
+	file, e = os.Open(found)
+	if e != nil {
+		t.Errorf("Failed to open %v", path)
+	} else {
+		var d [10]byte
+		if n, _ := file.Read(d[:]); n != 4 {
+			t.Errorf("Read incorrect number of chars from test.config, wrong file?")
+		}
+		if string(d[0:4]) != "test" {
+			t.Errorf("Wrong test file content: %v", string(d[0:4]))
+		}
+		file.Close()
+	}
+}
+
+// Test for error when file doesn't exist
+func TestOpenCamliIncludeNoFile(t *testing.T) {
+	// Test that error occurs if no such file
+	const notExist = "this_config_doesnt_exist.config"
+	_, e := FindCamliInclude(notExist)
+	if e == nil {
+		t.Errorf("Successfully opened config which doesn't exist: %v", notExist)
+	}
+}
+
+// Test for when a file exists in CWD
+func TestOpenCamliIncludeCWD(t *testing.T) {
+	const path string = "TestOpenCamliIncludeCWD.config"
+	if e := createTestInclude(path); e != nil {
+		t.Errorf("Couldn't create test config file, aborting test: %v", e)
+		return
+	}
+	defer syscall.Remove(path)
+
+	checkOpen(t, path)
+}
+
+// Test for when a file exists in CAMLI_CONFIG_DIR
+func TestOpenCamliIncludeDir(t *testing.T) {
+	const name string = "TestOpenCamliIncludeDir.config"
+	if e := createTestInclude("/tmp/" + name); e != nil {
+		t.Errorf("Couldn't create test config file, aborting test: %v", e)
+		return
+	}
+	defer os.Remove("/tmp/" + name)
+	os.Setenv("CAMLI_CONFIG_DIR", "/tmp")
+	defer os.Setenv("CAMLI_CONFIG_DIR", "")
+
+	checkOpen(t, name)
+}
+
+// Test for when a file exits in CAMLI_INCLUDE_PATH
+func TestOpenCamliIncludePath(t *testing.T) {
+	const name string = "TestOpenCamliIncludePath.config"
+	if e := createTestInclude("/tmp/" + name); e != nil {
+		t.Errorf("Couldn't create test config file, aborting test: %v", e)
+		return
+	}
+	defer syscall.Unlink("/tmp/" + name)
+	defer os.Setenv("CAMLI_INCLUDE_PATH", "")
+
+	os.Setenv("CAMLI_INCLUDE_PATH", "/tmp")
+	checkOpen(t, name)
+
+	os.Setenv("CAMLI_INCLUDE_PATH", "/not/a/camli/config/dir:/tmp")
+	checkOpen(t, name)
+
+	os.Setenv("CAMLI_INCLUDE_PATH", "/not/a/camli/config/dir:/tmp:/another/fake/camli/dir")
+	checkOpen(t, name)
+}

server/go/camlistored/camlistored.go

@@ -20,7 +20,6 @@ import (
 	"flag"
 	"fmt"
 	"http"
-	"json"
 	"log"
 	"path/filepath"
 	"strings"
@@ -29,7 +28,6 @@ import (
 	"camli/auth"
 	"camli/blobserver"
 	"camli/blobserver/handlers"
-	"camli/errorutil"
 	"camli/httputil"
 	"camli/jsonconfig"
 	"camli/osutil"
@@ -45,7 +43,6 @@ import (
 	_ "camli/mysqlindexer" // indexer, but uses storage interface
 	// Handlers:
 	_ "camli/search"
-
 )
 
 var flagConfigFile = flag.String("configfile", "serverconfig",
@@ -162,36 +159,15 @@ func main() {
 	if !filepath.IsAbs(configPath) {
 		configPath = filepath.Join(osutil.CamliConfigDir(), configPath)
 	}
-	f, err := os.Open(configPath)
+
+	config, err := jsonconfig.ReadFile(configPath)
 	if err != nil {
-		exitFailure("error opening %s: %v", configPath, err)
-	}
-	defer f.Close()
-	dj := json.NewDecoder(f)
-	rootjson := make(map[string]interface{})
-	if err = dj.Decode(&rootjson); err != nil {
-		extra := ""
-		if serr, ok := err.(*json.SyntaxError); ok {
-			if _, serr := f.Seek(0, os.SEEK_SET); serr != nil {
-				log.Fatalf("seek error: %v", serr)
-			}
-			line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset)
-			extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
-				line, col, serr.Offset, highlight)
-		}
-		exitFailure("error parsing JSON object in config file %s%s\n%v",
-			osutil.UserServerConfigPath(), extra, err)
-	}
-	if err := jsonconfig.EvaluateExpressions(rootjson); err != nil {
-		exitFailure("error expanding JSON config expressions in %s: %v", configPath, err)
+		exitFailure("%v", err)
 	}
 
 	ws := webserver.New()
 	baseURL := ws.BaseURL()
 
-	// Root configuration
-	config := jsonconfig.Obj(rootjson)
-
 	{
 		cert, key := config.OptionalString("TLSCertFile", ""), config.OptionalString("TLSKeyFile", "")
 		if (cert != "") != (key != "") {
@@ -207,7 +183,7 @@ func main() {
 		baseURL = url
 	}
 	prefixes := config.RequiredObject("prefixes")
-	if err := config.Validate(); err != nil {
+	if err = config.Validate(); err != nil {
 		exitFailure("configuration error in root object's keys in %s: %v", configPath, err)
 	}
 
@@ -227,7 +203,7 @@ func main() {
 		}
 		pmap, ok := vei.(map[string]interface{})
 		if !ok {
-			exitFailure("prefix %q value isn't an object", prefix)
+			exitFailure("prefix %q value is a %T, not an object", prefix,	vei)
 		}
 		pconf := jsonconfig.Obj(pmap)
 		handlerType := pconf.RequiredString("handler")