diff --git a/config/dev-server-config.json b/config/dev-server-config.json index d6be71099..1070cfb89 100644 --- a/config/dev-server-config.json +++ b/config/dev-server-config.json @@ -19,9 +19,9 @@ "/hello/": { "handler": "app", + "enabled": ["_env", "${CAMLI_HELLO_ENABLED}"], "handlerArgs": { "program": "hello", - "baseURL": "http://localhost:3178/", "appConfig": { "word": "world" } diff --git a/dev/devcam/server.go b/dev/devcam/server.go index 4d395f142..0db479c68 100644 --- a/dev/devcam/server.go +++ b/dev/devcam/server.go @@ -55,7 +55,8 @@ type serverCmd struct { fullClosure bool mini bool - publish bool + publish bool // whether to start the publish handlers + hello bool // whether to build and start the hello demo app openBrowser bool flickrAPIKey string @@ -82,6 +83,7 @@ func init() { flags.BoolVar(&cmd.wipe, "wipe", false, "Wipe the blobs on disk and the indexer.") flags.BoolVar(&cmd.debug, "debug", false, "Enable http debugging.") flags.BoolVar(&cmd.publish, "publish", true, "Enable publish handlers") + flags.BoolVar(&cmd.hello, "hello", false, "Enable hello (demo) app") flags.BoolVar(&cmd.mini, "mini", false, "Enable minimal mode, where all optional features are disabled. (Currently just publishing)") flags.BoolVar(&cmd.mongo, "mongo", false, "Use mongodb as the indexer. Excludes -mysql, -postgres, -sqlite.") @@ -195,6 +197,7 @@ func (c *serverCmd) setEnvVars() error { setenv("CAMLI_KVINDEX_ENABLED", "false") setenv("CAMLI_PUBLISH_ENABLED", strconv.FormatBool(c.publish)) + setenv("CAMLI_HELLO_ENABLED", strconv.FormatBool(c.hello)) switch { case c.mongo: setenv("CAMLI_MONGO_ENABLED", "true") @@ -366,6 +369,7 @@ func (c *serverCmd) setFullClosure() error { func (c *serverCmd) RunCommand(args []string) error { if c.mini { c.publish = false + c.hello = false } err := c.checkFlags(args) if err != nil { @@ -373,10 +377,14 @@ func (c *serverCmd) RunCommand(args []string) error { } if !*noBuild { withSqlite = c.sqlite - for _, name := range []string{ + targets := []string{ filepath.Join("server", "camlistored"), filepath.Join("cmd", "camtool"), - } { + } + if c.hello { + targets = append(targets, filepath.Join("app", "hello")) + } + for _, name := range targets { err := build(name) if err != nil { return fmt.Errorf("Could not build %v: %v", name, err) diff --git a/doc/app-environment.text b/doc/app-environment.text deleted file mode 100644 index a9e07e772..000000000 --- a/doc/app-environment.text +++ /dev/null @@ -1,24 +0,0 @@ -Camlistore applications run with the following environment variables set: - -CAMLI_APP_BASEURL (string): - URL prefix of the application's root, always ending in a trailing slash. Examples: - https://foo.org:3178/pub/ - https://foo.org/pub/ - http://192.168.0.1/ - http://192.168.0.1:1234/ - -CAMLI_APP_CONFIG_URL (string): - URL containing JSON configuration for the app. The body of this URL comes from the - "appConfig" part of the config file. - -CAMLI_AUTH (string): - Username and password (username:password) that the app should use to authenticate - over HTTP basic auth with the Camlistore server. Basic auth is unencrypted, hence - it should only be used with HTTPS or in a secure (local loopback) environment. - -CAMLI_SERVER (string): - URL prefix of Camlistore's root, always ending in a trailing slash. Examples: - https://foo.org:3178/pub/ - https://foo.org/pub/ - http://192.168.0.1/ - http://192.168.0.1:1234/ diff --git a/doc/app-environment.txt b/doc/app-environment.txt new file mode 100644 index 000000000..e5d5b97fa --- /dev/null +++ b/doc/app-environment.txt @@ -0,0 +1,31 @@ +Camlistore applications run with the following environment variables set: + +CAMLI_API_HOST (string): + URL prefix of the Camlistore server which the app should use to make API calls. + It always ends in a trailing slash. Examples: + https://foo.org:3178/pub/ + https://foo.org/pub/ + http://192.168.0.1/ + http://192.168.0.1:1234/ + +CAMLI_APP_BACKEND_URL (string): + URL of the application's process, always ending in a trailing slash. That path + represents the top-most path that requests will hit. The path usually matches + the path as visible in the outside world when camlistored is proxying an app, + but that is not guaranteed. Examples: + https://foo.org:3178/pub/ + https://foo.org/pub/ + http://192.168.0.1/ + http://192.168.0.1:1234/ + +CAMLI_APP_CONFIG_URL (string): + URL containing JSON configuration for the app. The app should once, upon + startup, fetch this URL (using CAMLI_AUTH) to retrieve its configuration data. + The response JSON is the contents of the app's "appConfig" part of the config + file. + +CAMLI_AUTH (string): + Username and password (username:password) that the app should use to + authenticate over HTTP basic auth with the Camlistore server. Basic auth is + unencrypted, hence it should only be used with HTTPS or in a secure (local + loopback) environment. diff --git a/pkg/app/app.go b/pkg/app/app.go index 3baa58908..da8e4ce5e 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -32,9 +32,9 @@ import ( // Client returns a client from pkg/client, configured by environment variables // for applications, and ready to be used to connect to the Camlistore server. func Client() (*client.Client, error) { - server := os.Getenv("CAMLI_SERVER") + server := os.Getenv("CAMLI_API_HOST") if server == "" { - return nil, errors.New("CAMLI_SERVER var not set") + return nil, errors.New("CAMLI_API_HOST var not set") } authString := os.Getenv("CAMLI_AUTH") if authString == "" { @@ -54,9 +54,9 @@ func Client() (*client.Client, error) { // ListenAddress returns the host:[port] network address, derived from the environment, // that the application should listen on. func ListenAddress() (string, error) { - baseURL := os.Getenv("CAMLI_APP_BASEURL") + baseURL := os.Getenv("CAMLI_APP_BACKEND_URL") if baseURL == "" { - return "", errors.New("CAMLI_APP_BASEURL is undefined") + return "", errors.New("CAMLI_APP_BACKEND_URL is undefined") } defaultPort := "80" noScheme := strings.TrimPrefix(baseURL, "http://") @@ -66,7 +66,7 @@ func ListenAddress() (string, error) { } hostPortPrefix := strings.SplitN(noScheme, "/", 2) if len(hostPortPrefix) != 2 { - return "", fmt.Errorf("invalid CAMLI_APP_BASEURL: %q (no trailing slash?)", baseURL) + return "", fmt.Errorf("invalid CAMLI_APP_BACKEND_URL: %q (no trailing slash?)", baseURL) } if !strings.Contains(hostPortPrefix[0], ":") { return fmt.Sprintf("%s:%s", hostPortPrefix[0], defaultPort), nil diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 493bcf771..5b3f87cc7 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -98,7 +98,7 @@ func TestListenAddress(t *testing.T) { }, } for _, v := range tests { - os.Setenv("CAMLI_APP_BASEURL", v.baseURL) + os.Setenv("CAMLI_APP_BACKEND_URL", v.baseURL) got, err := ListenAddress() if v.wantErr { if err == nil { diff --git a/pkg/server/app/app.go b/pkg/server/app/app.go index 2562c7743..aded8aaa2 100644 --- a/pkg/server/app/app.go +++ b/pkg/server/app/app.go @@ -21,6 +21,7 @@ package app import ( "fmt" "log" + "net" "net/http" "net/http/httputil" "net/url" @@ -35,9 +36,9 @@ import ( "camlistore.org/pkg/osutil" ) -// AppHandler acts as a reverse proxy for a server application started by +// Handler acts as a reverse proxy for a server application started by // Camlistore. It can also serve some extra JSON configuration to the app. -type AppHandler struct { +type Handler struct { name string // Name of the app's program. envVars map[string]string // Variables set in the app's process environment. See pkg/app/vars.txt. @@ -47,7 +48,7 @@ type AppHandler struct { proxy *httputil.ReverseProxy // For redirecting requests to the app. } -func (a *AppHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { +func (a *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if camhttputil.PathSuffix(req) == "config.json" { if a.auth.AllowedAccess(req)&auth.OpGet == auth.OpGet { camhttputil.ReturnJSON(rw, a.appConfig) @@ -63,20 +64,62 @@ func (a *AppHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { a.proxy.ServeHTTP(rw, req) } -// New returns a configured AppHandler that Camlistore can use during server initialization -// as a handler that proxies request to an app. It is also used to start the app. -// The conf object has the following members, related to the vars described in doc/app-environment.text: -// "program", string, required. Name of the app's program. -// "baseURL", string, required. See CAMLI_APP_BASEURL. -// "server", string, optional, overrides the camliBaseURL argument. See CAMLI_SERVER. -// "appConfig", object, optional. Additional configuration that the app can request from Camlistore. -func New(conf jsonconfig.Obj, serverBaseURL string) (*AppHandler, error) { - name := conf.RequiredString("program") - server := conf.OptionalString("server", serverBaseURL) - if server == "" { - return nil, fmt.Errorf("could not initialize AppHandler for %q: Camlistore baseURL is unknown", name) +// randPortBackendURL picks a random free port to listen on, and combines it +// with apiHost and appHandlerPrefix to create the appBackendURL that the app +// will listen on, and that the app handler will proxy to. +func randPortBackendURL(apiHost, appHandlerPrefix string) (string, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return "", err } - baseURL := conf.RequiredString("baseURL") + listener, err := net.ListenTCP("tcp", addr) + if err != nil { + return "", fmt.Errorf("could not listen to find random port: %v", err) + } + randAddr := listener.Addr().(*net.TCPAddr) + if err := listener.Close(); err != nil { + return "", fmt.Errorf("could not close random listener: %v", err) + } + + scheme := "https://" + noScheme := strings.TrimPrefix(apiHost, scheme) + if strings.HasPrefix(noScheme, "http://") { + scheme = "http://" + noScheme = strings.TrimPrefix(noScheme, scheme) + } + hostPortPrefix := strings.SplitN(noScheme, "/", 2) + if len(hostPortPrefix) != 2 { + return "", fmt.Errorf("invalid apiHost: %q (no trailing slash?)", apiHost) + } + var host string + if strings.Contains(hostPortPrefix[0], "]") { + // we've got some IPv6 probably + hostPort := strings.Split(hostPortPrefix[0], "]") + host = hostPort[0] + "]" + } else { + hostPort := strings.Split(hostPortPrefix[0], ":") + host = hostPort[0] + } + return fmt.Sprintf("%s%s:%d%s", scheme, host, randAddr.Port, appHandlerPrefix), nil +} + +// NewHandler returns a Handler that proxies requests to an app. Start() on the +// Handler starts the app. +// The apiHost must end in a slash and is the camlistored API server for the app +// process to hit. +// The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted. +// It must end in a slash, and be at minimum "/". +// The conf object has the following members, related to the vars described in +// doc/app-environment.txt: +// "program", string, required. File name of the app's program executable. Either +// an absolute path or the name of a file located in your PATH or in the bin +// directory of the Camlistore source tree. +// "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL. +// "appConfig", object, optional. Additional configuration that the app can request from Camlistore. +func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) { + // TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available. + name := conf.RequiredString("program") + backendURL := conf.OptionalString("backendURL", "") appConfig := conf.OptionalObject("appConfig") // TODO(mpl): add an auth token in the extra config of the dev server config, // that the hello app can use to setup a status handler than only responds @@ -85,24 +128,41 @@ func New(conf jsonconfig.Obj, serverBaseURL string) (*AppHandler, error) { return nil, err } + if apiHost == "" { + return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name) + } + if appHandlerPrefix == "" { + return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name) + } + + if backendURL == "" { + var err error + // If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL + // now (instead of letting the app itself do it), because we need to know it in advance in order + // to set the app handler's proxy. + backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix) + if err != nil { + return nil, err + } + } + username, password := auth.RandToken(20), auth.RandToken(20) camliAuth := username + ":" + password basicAuth := auth.NewBasicAuth(username, password) envVars := map[string]string{ - "CAMLI_SERVER": server, - "CAMLI_AUTH": camliAuth, - "CAMLI_APP_BASEURL": baseURL, + "CAMLI_API_HOST": apiHost, + "CAMLI_AUTH": camliAuth, + "CAMLI_APP_BACKEND_URL": backendURL, } if appConfig != nil { - appConfigURL := fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(server, "/"), name, "config.json") - envVars["CAMLI_APP_CONFIG_URL"] = appConfigURL + envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json" } - proxyURL, err := url.Parse(baseURL) + proxyURL, err := url.Parse(backendURL) if err != nil { - return nil, fmt.Errorf("could not parse baseURL %q: %v", baseURL, err) + return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err) } - return &AppHandler{ + return &Handler{ name: name, envVars: envVars, auth: basicAuth, @@ -111,7 +171,7 @@ func New(conf jsonconfig.Obj, serverBaseURL string) (*AppHandler, error) { }, nil } -func (a *AppHandler) Start() error { +func (a *Handler) Start() error { name := a.name if name == "" { return fmt.Errorf("invalid app name: %q", name) @@ -159,13 +219,22 @@ func (a *AppHandler) Start() error { return nil } -func (a *AppHandler) Name() string { +// ProgramName returns the name of the app's binary. It may be a file name in +// PATH or in the bin directory of the Camlistore source tree, or an absolute +// path. +func (a *Handler) ProgramName() string { return a.name } // AuthMode returns the app handler's auth mode, which is also the auth that the // app's client will be configured with. This mode should be registered with // the server's auth modes, for the app to have access to the server's resources. -func (a *AppHandler) AuthMode() auth.AuthMode { +func (a *Handler) AuthMode() auth.AuthMode { return a.auth } + +// AppConfig returns the optional configuration parameters object that the app +// can request from the app handler. It can be nil. +func (a *Handler) AppConfig() map[string]interface{} { + return a.appConfig +} diff --git a/pkg/server/app/app_test.go b/pkg/server/app/app_test.go new file mode 100644 index 000000000..ce7131f0b --- /dev/null +++ b/pkg/server/app/app_test.go @@ -0,0 +1,150 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "regexp" + "testing" +) + +func TestRandPortBackendURL(t *testing.T) { + tests := []struct { + apiHost string + appHandlerPrefix string + wantBackendURL string + wantErr bool + }{ + { + apiHost: "http://foo.com/", + appHandlerPrefix: "/pics/", + wantBackendURL: "http://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "https://foo.com/", + appHandlerPrefix: "/pics/", + wantBackendURL: "https://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "http://foo.com:8080/", + appHandlerPrefix: "/pics/", + wantBackendURL: "http://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "https://foo.com:8080/", + appHandlerPrefix: "/pics/", + wantBackendURL: "https://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "http://foo.com:/", + appHandlerPrefix: "/pics/", + wantBackendURL: "http://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "https://foo.com:/", + appHandlerPrefix: "/pics/", + wantBackendURL: "https://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "http://foo.com/bar/", + appHandlerPrefix: "/pics/", + wantBackendURL: "http://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "https://foo.com/bar/", + appHandlerPrefix: "/pics/", + wantBackendURL: "https://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "http://foo.com:8080/bar/", + appHandlerPrefix: "/pics/", + wantBackendURL: "http://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "https://foo.com:8080/bar/", + appHandlerPrefix: "/pics/", + wantBackendURL: "https://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "http://foo.com:/bar/", + appHandlerPrefix: "/pics/", + wantBackendURL: "http://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "https://foo.com:/bar/", + appHandlerPrefix: "/pics/", + wantBackendURL: "https://foo.com:[0-9]+/pics/", + }, + + { + apiHost: "http://[::1]:80/", + appHandlerPrefix: "/pics/", + wantBackendURL: `http://\[::1\]:[0-9]+/pics/`, + }, + + { + apiHost: "https://[::1]:80/", + appHandlerPrefix: "/pics/", + wantBackendURL: `https://\[::1\]:[0-9]+/pics/`, + }, + + { + apiHost: "http://[::1]/", + appHandlerPrefix: "/pics/", + wantBackendURL: `http://\[::1\]:[0-9]+/pics/`, + }, + + { + apiHost: "https://[::1]/", + appHandlerPrefix: "/pics/", + wantBackendURL: `https://\[::1\]:[0-9]+/pics/`, + }, + + { + apiHost: "http://[::1]:/", + appHandlerPrefix: "/pics/", + wantBackendURL: `http://\[::1\]:[0-9]+/pics/`, + }, + + { + apiHost: "https://[::1]:/", + appHandlerPrefix: "/pics/", + wantBackendURL: `https://\[::1\]:[0-9]+/pics/`, + }, + } + for _, v := range tests { + got, err := randPortBackendURL(v.apiHost, v.appHandlerPrefix) + if err != nil { + t.Error(err) + continue + } + reg := regexp.MustCompile(v.wantBackendURL) + if !reg.MatchString(got) { + t.Errorf("got: %v for %v, want: %v", got, v.apiHost, v.wantBackendURL) + } + } +} diff --git a/pkg/serverinit/serverinit.go b/pkg/serverinit/serverinit.go index 030a61e86..16526a81a 100644 --- a/pkg/serverinit/serverinit.go +++ b/pkg/serverinit/serverinit.go @@ -336,7 +336,7 @@ func (hl *handlerLoader) setupHandler(prefix string) { var hh http.Handler if h.htype == "app" { - ap, err := app.New(h.conf, hl.baseURL+"/") + ap, err := app.NewHandler(h.conf, hl.baseURL+"/", prefix) if err != nil { exitFailure("error setting up app for prefix %q: %v", h.prefix, err) } @@ -390,7 +390,7 @@ type Config struct { // apps is the list of server apps configured during InstallHandlers, // and that should be started after camlistored has started serving. - apps []*app.AppHandler + apps []*app.Handler } // detectConfigChange returns an informative error if conf contains obsolete keys. @@ -545,7 +545,7 @@ func (config *Config) InstallHandlers(hi HandlerInstaller, baseURL string, reind // methods. // And register apps that will be started later. for pfx, handler := range hl.handler { - if starter, ok := handler.(*app.AppHandler); ok { + if starter, ok := handler.(*app.Handler); ok { config.apps = append(config.apps, starter) } if in, ok := handler.(blobserver.HandlerIniter); ok { @@ -571,7 +571,7 @@ func (config *Config) InstallHandlers(hi HandlerInstaller, baseURL string, reind func (config *Config) StartApps() error { for _, ap := range config.apps { if err := ap.Start(); err != nil { - return fmt.Errorf("error starting app %v: %v", ap.Name(), err) + return fmt.Errorf("error starting app %v: %v", ap.ProgramName(), err) } } return nil