Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apps: more generic app handler work, to prepare for publisher 

http://camlistore.org/issue/365

Change-Id: I3c34ba1f09debc3aaaf68ad9fb11b595470b6b5d
This commit is contained in:
mpl 2014-06-13 23:03:49 +02:00
parent 964b62ae89
commit 9723b5c814
9 changed files with 299 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/dev-server-config.json
View File

@ -19,9 +19,9 @@
"/hello/": {
"handler": "app",
"enabled": ["_env", "${CAMLI_HELLO_ENABLED}"],
"handlerArgs": {
"program": "hello",
"baseURL": "http://localhost:3178/",
"appConfig": {
"word": "world"
}

14
dev/devcam/server.go
View File

@ -55,7 +55,8 @@ type serverCmd struct {
fullClosure bool
mini bool
publish bool
publish bool // whether to start the publish handlers
hello bool // whether to build and start the hello demo app
openBrowser bool
flickrAPIKey string
@ -82,6 +83,7 @@ func init() {
flags.BoolVar(&cmd.wipe, "wipe", false, "Wipe the blobs on disk and the indexer.")
flags.BoolVar(&cmd.debug, "debug", false, "Enable http debugging.")
flags.BoolVar(&cmd.publish, "publish", true, "Enable publish handlers")
flags.BoolVar(&cmd.hello, "hello", false, "Enable hello (demo) app")
flags.BoolVar(&cmd.mini, "mini", false, "Enable minimal mode, where all optional features are disabled. (Currently just publishing)")
flags.BoolVar(&cmd.mongo, "mongo", false, "Use mongodb as the indexer. Excludes -mysql, -postgres, -sqlite.")
@ -195,6 +197,7 @@ func (c *serverCmd) setEnvVars() error {
setenv("CAMLI_KVINDEX_ENABLED", "false")
setenv("CAMLI_PUBLISH_ENABLED", strconv.FormatBool(c.publish))
setenv("CAMLI_HELLO_ENABLED", strconv.FormatBool(c.hello))
switch {
case c.mongo:
setenv("CAMLI_MONGO_ENABLED", "true")
@ -366,6 +369,7 @@ func (c *serverCmd) setFullClosure() error {
func (c *serverCmd) RunCommand(args []string) error {
if c.mini {
c.publish = false
c.hello = false
}
err := c.checkFlags(args)
if err != nil {
@ -373,10 +377,14 @@ func (c *serverCmd) RunCommand(args []string) error {
}
if !*noBuild {
withSqlite = c.sqlite
for _, name := range []string{
targets := []string{
filepath.Join("server", "camlistored"),
filepath.Join("cmd", "camtool"),
} {
}
if c.hello {
targets = append(targets, filepath.Join("app", "hello"))
}
for _, name := range targets {
err := build(name)
if err != nil {
return fmt.Errorf("Could not build %v: %v", name, err)

24
doc/app-environment.text
View File

@ -1,24 +0,0 @@
Camlistore applications run with the following environment variables set:
CAMLI_APP_BASEURL (string):
URL prefix of the application's root, always ending in a trailing slash. Examples:
https://foo.org:3178/pub/
https://foo.org/pub/
http://192.168.0.1/
http://192.168.0.1:1234/
CAMLI_APP_CONFIG_URL (string):
URL containing JSON configuration for the app. The body of this URL comes from the
"appConfig" part of the config file.
CAMLI_AUTH (string):
Username and password (username:password) that the app should use to authenticate
over HTTP basic auth with the Camlistore server. Basic auth is unencrypted, hence
it should only be used with HTTPS or in a secure (local loopback) environment.
CAMLI_SERVER (string):
URL prefix of Camlistore's root, always ending in a trailing slash. Examples:
https://foo.org:3178/pub/
https://foo.org/pub/
http://192.168.0.1/
http://192.168.0.1:1234/

31
doc/app-environment.txt Normal file
View File

@ -0,0 +1,31 @@
Camlistore applications run with the following environment variables set:
CAMLI_API_HOST (string):
URL prefix of the Camlistore server which the app should use to make API calls.
It always ends in a trailing slash. Examples:
https://foo.org:3178/pub/
https://foo.org/pub/
http://192.168.0.1/
http://192.168.0.1:1234/
CAMLI_APP_BACKEND_URL (string):
URL of the application's process, always ending in a trailing slash. That path
represents the top-most path that requests will hit. The path usually matches
the path as visible in the outside world when camlistored is proxying an app,
but that is not guaranteed. Examples:
https://foo.org:3178/pub/
https://foo.org/pub/
http://192.168.0.1/
http://192.168.0.1:1234/
CAMLI_APP_CONFIG_URL (string):
URL containing JSON configuration for the app. The app should once, upon
startup, fetch this URL (using CAMLI_AUTH) to retrieve its configuration data.
The response JSON is the contents of the app's "appConfig" part of the config
file.
CAMLI_AUTH (string):
Username and password (username:password) that the app should use to
authenticate over HTTP basic auth with the Camlistore server. Basic auth is
unencrypted, hence it should only be used with HTTPS or in a secure (local
loopback) environment.

10
pkg/app/app.go
View File

@ -32,9 +32,9 @@ import (
// Client returns a client from pkg/client, configured by environment variables
// for applications, and ready to be used to connect to the Camlistore server.
func Client() (*client.Client, error) {
server := os.Getenv("CAMLI_SERVER")
server := os.Getenv("CAMLI_API_HOST")
if server == "" {
return nil, errors.New("CAMLI_SERVER var not set")
return nil, errors.New("CAMLI_API_HOST var not set")
}
authString := os.Getenv("CAMLI_AUTH")
if authString == "" {
@ -54,9 +54,9 @@ func Client() (*client.Client, error) {
// ListenAddress returns the host:[port] network address, derived from the environment,
// that the application should listen on.
func ListenAddress() (string, error) {
baseURL := os.Getenv("CAMLI_APP_BASEURL")
baseURL := os.Getenv("CAMLI_APP_BACKEND_URL")
if baseURL == "" {
return "", errors.New("CAMLI_APP_BASEURL is undefined")
return "", errors.New("CAMLI_APP_BACKEND_URL is undefined")
}
defaultPort := "80"
noScheme := strings.TrimPrefix(baseURL, "http://")
@ -66,7 +66,7 @@ func ListenAddress() (string, error) {
}
hostPortPrefix := strings.SplitN(noScheme, "/", 2)
if len(hostPortPrefix) != 2 {
return "", fmt.Errorf("invalid CAMLI_APP_BASEURL: %q (no trailing slash?)", baseURL)
return "", fmt.Errorf("invalid CAMLI_APP_BACKEND_URL: %q (no trailing slash?)", baseURL)
}
if !strings.Contains(hostPortPrefix[0], ":") {
return fmt.Sprintf("%s:%s", hostPortPrefix[0], defaultPort), nil

2
pkg/app/app_test.go
View File

@ -98,7 +98,7 @@ func TestListenAddress(t *testing.T) {
},
}
for _, v := range tests {
os.Setenv("CAMLI_APP_BASEURL", v.baseURL)
os.Setenv("CAMLI_APP_BACKEND_URL", v.baseURL)
got, err := ListenAddress()
if v.wantErr {
if err == nil {

123
pkg/server/app/app.go
View File

@ -21,6 +21,7 @@ package app
import (
"fmt"
"log"
"net"
"net/http"
"net/http/httputil"
"net/url"
@ -35,9 +36,9 @@ import (
"camlistore.org/pkg/osutil"
)
// AppHandler acts as a reverse proxy for a server application started by
// Handler acts as a reverse proxy for a server application started by
// Camlistore. It can also serve some extra JSON configuration to the app.
type AppHandler struct {
type Handler struct {
name string // Name of the app's program.
envVars map[string]string // Variables set in the app's process environment. See pkg/app/vars.txt.
@ -47,7 +48,7 @@ type AppHandler struct {
proxy *httputil.ReverseProxy // For redirecting requests to the app.
}
func (a *AppHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
func (a *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if camhttputil.PathSuffix(req) == "config.json" {
if a.auth.AllowedAccess(req)&auth.OpGet == auth.OpGet {
camhttputil.ReturnJSON(rw, a.appConfig)
@ -63,20 +64,62 @@ func (a *AppHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
a.proxy.ServeHTTP(rw, req)
}
// New returns a configured AppHandler that Camlistore can use during server initialization
// as a handler that proxies request to an app. It is also used to start the app.
// The conf object has the following members, related to the vars described in doc/app-environment.text:
// "program", string, required. Name of the app's program.
// "baseURL", string, required. See CAMLI_APP_BASEURL.
// "server", string, optional, overrides the camliBaseURL argument. See CAMLI_SERVER.
// "appConfig", object, optional. Additional configuration that the app can request from Camlistore.
func New(conf jsonconfig.Obj, serverBaseURL string) (*AppHandler, error) {
name := conf.RequiredString("program")
server := conf.OptionalString("server", serverBaseURL)
if server == "" {
return nil, fmt.Errorf("could not initialize AppHandler for %q: Camlistore baseURL is unknown", name)
// randPortBackendURL picks a random free port to listen on, and combines it
// with apiHost and appHandlerPrefix to create the appBackendURL that the app
// will listen on, and that the app handler will proxy to.
func randPortBackendURL(apiHost, appHandlerPrefix string) (string, error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return "", err
}
baseURL := conf.RequiredString("baseURL")
listener, err := net.ListenTCP("tcp", addr)
if err != nil {
return "", fmt.Errorf("could not listen to find random port: %v", err)
}
randAddr := listener.Addr().(*net.TCPAddr)
if err := listener.Close(); err != nil {
return "", fmt.Errorf("could not close random listener: %v", err)
}
scheme := "https://"
noScheme := strings.TrimPrefix(apiHost, scheme)
if strings.HasPrefix(noScheme, "http://") {
scheme = "http://"
noScheme = strings.TrimPrefix(noScheme, scheme)
}
hostPortPrefix := strings.SplitN(noScheme, "/", 2)
if len(hostPortPrefix) != 2 {
return "", fmt.Errorf("invalid apiHost: %q (no trailing slash?)", apiHost)
}
var host string
if strings.Contains(hostPortPrefix[0], "]") {
// we've got some IPv6 probably
hostPort := strings.Split(hostPortPrefix[0], "]")
host = hostPort[0] + "]"
} else {
hostPort := strings.Split(hostPortPrefix[0], ":")
host = hostPort[0]
}
return fmt.Sprintf("%s%s:%d%s", scheme, host, randAddr.Port, appHandlerPrefix), nil
}
// NewHandler returns a Handler that proxies requests to an app. Start() on the
// Handler starts the app.
// The apiHost must end in a slash and is the camlistored API server for the app
// process to hit.
// The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted.
// It must end in a slash, and be at minimum "/".
// The conf object has the following members, related to the vars described in
// doc/app-environment.txt:
// "program", string, required. File name of the app's program executable. Either
// an absolute path or the name of a file located in your PATH or in the bin
// directory of the Camlistore source tree.
// "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL.
// "appConfig", object, optional. Additional configuration that the app can request from Camlistore.
func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) {
// TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available.
name := conf.RequiredString("program")
backendURL := conf.OptionalString("backendURL", "")
appConfig := conf.OptionalObject("appConfig")
// TODO(mpl): add an auth token in the extra config of the dev server config,
// that the hello app can use to setup a status handler than only responds
@ -85,24 +128,41 @@ func New(conf jsonconfig.Obj, serverBaseURL string) (*AppHandler, error) {
return nil, err
}
if apiHost == "" {
return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name)
}
if appHandlerPrefix == "" {
return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name)
}
if backendURL == "" {
var err error
// If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL
// now (instead of letting the app itself do it), because we need to know it in advance in order
// to set the app handler's proxy.
backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix)
if err != nil {
return nil, err
}
}
username, password := auth.RandToken(20), auth.RandToken(20)
camliAuth := username + ":" + password
basicAuth := auth.NewBasicAuth(username, password)
envVars := map[string]string{
"CAMLI_SERVER": server,
"CAMLI_AUTH": camliAuth,
"CAMLI_APP_BASEURL": baseURL,
"CAMLI_API_HOST": apiHost,
"CAMLI_AUTH": camliAuth,
"CAMLI_APP_BACKEND_URL": backendURL,
}
if appConfig != nil {
appConfigURL := fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(server, "/"), name, "config.json")
envVars["CAMLI_APP_CONFIG_URL"] = appConfigURL
envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json"
}
proxyURL, err := url.Parse(baseURL)
proxyURL, err := url.Parse(backendURL)
if err != nil {
return nil, fmt.Errorf("could not parse baseURL %q: %v", baseURL, err)
return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err)
}
return &AppHandler{
return &Handler{
name: name,
envVars: envVars,
auth: basicAuth,
@ -111,7 +171,7 @@ func New(conf jsonconfig.Obj, serverBaseURL string) (*AppHandler, error) {
}, nil
}
func (a *AppHandler) Start() error {
func (a *Handler) Start() error {
name := a.name
if name == "" {
return fmt.Errorf("invalid app name: %q", name)
@ -159,13 +219,22 @@ func (a *AppHandler) Start() error {
return nil
}
func (a *AppHandler) Name() string {
// ProgramName returns the name of the app's binary. It may be a file name in
// PATH or in the bin directory of the Camlistore source tree, or an absolute
// path.
func (a *Handler) ProgramName() string {
return a.name
}
// AuthMode returns the app handler's auth mode, which is also the auth that the
// app's client will be configured with. This mode should be registered with
// the server's auth modes, for the app to have access to the server's resources.
func (a *AppHandler) AuthMode() auth.AuthMode {
func (a *Handler) AuthMode() auth.AuthMode {
return a.auth
}
// AppConfig returns the optional configuration parameters object that the app
// can request from the app handler. It can be nil.
func (a *Handler) AppConfig() map[string]interface{} {
return a.appConfig
}

150
pkg/server/app/app_test.go Normal file
View File

@ -0,0 +1,150 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"regexp"
"testing"
)
func TestRandPortBackendURL(t *testing.T) {
tests := []struct {
apiHost string
appHandlerPrefix string
wantBackendURL string
wantErr bool
}{
{
apiHost: "http://foo.com/",
appHandlerPrefix: "/pics/",
wantBackendURL: "http://foo.com:[0-9]+/pics/",
},
{
apiHost: "https://foo.com/",
appHandlerPrefix: "/pics/",
wantBackendURL: "https://foo.com:[0-9]+/pics/",
},
{
apiHost: "http://foo.com:8080/",
appHandlerPrefix: "/pics/",
wantBackendURL: "http://foo.com:[0-9]+/pics/",
},
{
apiHost: "https://foo.com:8080/",
appHandlerPrefix: "/pics/",
wantBackendURL: "https://foo.com:[0-9]+/pics/",
},
{
apiHost: "http://foo.com:/",
appHandlerPrefix: "/pics/",
wantBackendURL: "http://foo.com:[0-9]+/pics/",
},
{
apiHost: "https://foo.com:/",
appHandlerPrefix: "/pics/",
wantBackendURL: "https://foo.com:[0-9]+/pics/",
},
{
apiHost: "http://foo.com/bar/",
appHandlerPrefix: "/pics/",
wantBackendURL: "http://foo.com:[0-9]+/pics/",
},
{
apiHost: "https://foo.com/bar/",
appHandlerPrefix: "/pics/",
wantBackendURL: "https://foo.com:[0-9]+/pics/",
},
{
apiHost: "http://foo.com:8080/bar/",
appHandlerPrefix: "/pics/",
wantBackendURL: "http://foo.com:[0-9]+/pics/",
},
{
apiHost: "https://foo.com:8080/bar/",
appHandlerPrefix: "/pics/",
wantBackendURL: "https://foo.com:[0-9]+/pics/",
},
{
apiHost: "http://foo.com:/bar/",
appHandlerPrefix: "/pics/",
wantBackendURL: "http://foo.com:[0-9]+/pics/",
},
{
apiHost: "https://foo.com:/bar/",
appHandlerPrefix: "/pics/",
wantBackendURL: "https://foo.com:[0-9]+/pics/",
},
{
apiHost: "http://[::1]:80/",
appHandlerPrefix: "/pics/",
wantBackendURL: `http://\[::1\]:[0-9]+/pics/`,
},
{
apiHost: "https://[::1]:80/",
appHandlerPrefix: "/pics/",
wantBackendURL: `https://\[::1\]:[0-9]+/pics/`,
},
{
apiHost: "http://[::1]/",
appHandlerPrefix: "/pics/",
wantBackendURL: `http://\[::1\]:[0-9]+/pics/`,
},
{
apiHost: "https://[::1]/",
appHandlerPrefix: "/pics/",
wantBackendURL: `https://\[::1\]:[0-9]+/pics/`,
},
{
apiHost: "http://[::1]:/",
appHandlerPrefix: "/pics/",
wantBackendURL: `http://\[::1\]:[0-9]+/pics/`,
},
{
apiHost: "https://[::1]:/",
appHandlerPrefix: "/pics/",
wantBackendURL: `https://\[::1\]:[0-9]+/pics/`,
},
}
for _, v := range tests {
got, err := randPortBackendURL(v.apiHost, v.appHandlerPrefix)
if err != nil {
t.Error(err)
continue
}
reg := regexp.MustCompile(v.wantBackendURL)
if !reg.MatchString(got) {
t.Errorf("got: %v for %v, want: %v", got, v.apiHost, v.wantBackendURL)
}
}
}

8
pkg/serverinit/serverinit.go
View File

@ -336,7 +336,7 @@ func (hl *handlerLoader) setupHandler(prefix string) {
var hh http.Handler
if h.htype == "app" {
ap, err := app.New(h.conf, hl.baseURL+"/")
ap, err := app.NewHandler(h.conf, hl.baseURL+"/", prefix)
if err != nil {
exitFailure("error setting up app for prefix %q: %v", h.prefix, err)
}
@ -390,7 +390,7 @@ type Config struct {
// apps is the list of server apps configured during InstallHandlers,
// and that should be started after camlistored has started serving.
apps []*app.AppHandler
apps []*app.Handler
}
// detectConfigChange returns an informative error if conf contains obsolete keys.
@ -545,7 +545,7 @@ func (config *Config) InstallHandlers(hi HandlerInstaller, baseURL string, reind
// methods.
// And register apps that will be started later.
for pfx, handler := range hl.handler {
if starter, ok := handler.(*app.AppHandler); ok {
if starter, ok := handler.(*app.Handler); ok {
config.apps = append(config.apps, starter)
}
if in, ok := handler.(blobserver.HandlerIniter); ok {
@ -571,7 +571,7 @@ func (config *Config) InstallHandlers(hi HandlerInstaller, baseURL string, reind
func (config *Config) StartApps() error {
for _, ap := range config.apps {
if err := ap.Start(); err != nil {
return fmt.Errorf("error starting app %v: %v", ap.Name(), err)
return fmt.Errorf("error starting app %v: %v", ap.ProgramName(), err)
}
}
return nil