2012-03-15 12:31:06 +00:00
/ *
Copyright 2012 Google Inc .
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
2014-01-23 16:18:22 +00:00
package serverinit
2012-03-15 12:31:06 +00:00
import (
2014-04-01 16:03:20 +00:00
"encoding/json"
2012-08-04 01:12:39 +00:00
"errors"
2012-03-15 12:31:06 +00:00
"fmt"
2014-04-01 16:03:20 +00:00
"io/ioutil"
"log"
2013-08-09 00:50:50 +00:00
"net/url"
2012-03-15 12:31:06 +00:00
"os"
"path/filepath"
2014-03-14 19:08:43 +00:00
"sort"
2012-03-15 12:31:06 +00:00
"strings"
2013-08-04 02:54:30 +00:00
"camlistore.org/pkg/blob"
2012-03-15 12:31:06 +00:00
"camlistore.org/pkg/jsonconfig"
2012-04-12 18:39:53 +00:00
"camlistore.org/pkg/jsonsign"
2013-09-01 16:50:35 +00:00
"camlistore.org/pkg/osutil"
2014-01-23 22:40:12 +00:00
"camlistore.org/pkg/types/serverconfig"
2012-03-15 12:31:06 +00:00
)
// various parameters derived from the high-level user config
// and needed to set up the low-level config.
type configPrefixesParams struct {
2013-07-16 15:55:20 +00:00
secretRing string
keyId string
2014-04-01 16:03:20 +00:00
haveIndex bool
haveSQLite bool
2013-07-16 15:55:20 +00:00
blobPath string
2013-11-25 10:23:50 +00:00
packBlobs bool
2013-08-04 02:54:30 +00:00
searchOwner blob . Ref
2013-07-16 15:55:20 +00:00
shareHandlerPath string
2013-11-19 04:53:46 +00:00
flickr string
2014-03-04 21:25:10 +00:00
picasa string
2013-11-28 19:09:16 +00:00
memoryIndex bool
2014-03-17 15:42:19 +00:00
indexFileDir string // if sqlite or kvfile, its directory. else "".
2012-03-15 12:31:06 +00:00
}
2013-08-25 17:25:30 +00:00
var (
tempDir = os . TempDir
noMkdir bool // for tests to not call os.Mkdir
)
2013-01-11 18:52:22 +00:00
2014-06-16 16:05:39 +00:00
type tlsOpts struct {
httpsCert string
httpsKey string
}
2013-06-28 15:15:48 +00:00
func addPublishedConfig ( prefixes jsonconfig . Obj ,
2014-01-23 22:40:12 +00:00
published map [ string ] * serverconfig . Publish ,
2014-06-16 16:05:39 +00:00
sourceRoot string , tlsO * tlsOpts ) ( [ ] string , error ) {
2014-03-14 19:08:43 +00:00
var pubPrefixes [ ] string
2012-04-22 15:33:22 +00:00
for k , v := range published {
2014-06-14 20:14:34 +00:00
if v . CamliRoot == "" {
return nil , fmt . Errorf ( "Missing \"camliRoot\" key in configuration for %s." , k )
2012-04-22 15:33:22 +00:00
}
2014-01-23 22:40:12 +00:00
if v . GoTemplate == "" {
return nil , fmt . Errorf ( "Missing \"goTemplate\" key in configuration for %s." , k )
2012-04-22 15:33:22 +00:00
}
ob := map [ string ] interface { } { }
2014-06-14 20:14:34 +00:00
ob [ "handler" ] = "app"
appConfig := map [ string ] interface { } {
"camliRoot" : v . CamliRoot ,
"cacheRoot" : v . CacheRoot ,
"goTemplate" : v . GoTemplate ,
}
2014-06-16 16:05:39 +00:00
if v . HTTPSCert != "" && v . HTTPSKey != "" {
// user can specify these directly in the publish section
appConfig [ "httpsCert" ] = v . HTTPSCert
appConfig [ "httpsKey" ] = v . HTTPSKey
} else {
// default to Camlistore parameters, if any
if tlsO != nil {
appConfig [ "httpsCert" ] = tlsO . httpsCert
appConfig [ "httpsKey" ] = tlsO . httpsKey
}
}
2014-06-14 20:14:34 +00:00
2012-04-22 15:33:22 +00:00
handlerArgs := map [ string ] interface { } {
2014-06-14 20:14:34 +00:00
"program" : v . Program ,
"appConfig" : appConfig ,
2013-12-19 16:33:24 +00:00
}
2014-06-14 20:14:34 +00:00
if v . BaseURL != "" {
handlerArgs [ "baseURL" ] = v . BaseURL
2013-12-19 16:33:24 +00:00
}
2014-06-14 20:14:34 +00:00
program := "publisher"
if v . Program != "" {
program = v . Program
}
handlerArgs [ "program" ] = program
2012-04-22 15:33:22 +00:00
ob [ "handlerArgs" ] = handlerArgs
2012-12-22 00:13:36 +00:00
prefixes [ k ] = ob
2012-04-22 15:33:22 +00:00
pubPrefixes = append ( pubPrefixes , k )
}
2014-03-14 19:08:43 +00:00
sort . Strings ( pubPrefixes )
2012-04-22 15:33:22 +00:00
return pubPrefixes , nil
}
2013-12-14 17:37:56 +00:00
func addUIConfig ( params * configPrefixesParams ,
prefixes jsonconfig . Obj ,
2013-06-20 21:33:00 +00:00
uiPrefix string ,
sourceRoot string ) {
2013-12-14 17:37:56 +00:00
args := map [ string ] interface { } {
2012-03-15 12:31:06 +00:00
"jsonSignRoot" : "/sighelper/" ,
"cache" : "/cache/" ,
}
2013-06-20 21:33:00 +00:00
if sourceRoot != "" {
2013-12-14 17:37:56 +00:00
args [ "sourceRoot" ] = sourceRoot
}
if params . blobPath != "" {
args [ "scaledImage" ] = map [ string ] interface { } {
"type" : "kv" ,
"file" : filepath . Join ( params . blobPath , "thumbmeta.kv" ) ,
}
}
prefixes [ uiPrefix ] = map [ string ] interface { } {
"handler" : "ui" ,
"handlerArgs" : args ,
2013-06-20 21:33:00 +00:00
}
2012-03-15 12:31:06 +00:00
}
2012-12-22 00:13:36 +00:00
func addMongoConfig ( prefixes jsonconfig . Obj , dbname string , dbinfo string ) {
2012-10-23 19:45:46 +00:00
fields := strings . Split ( dbinfo , "@" )
if len ( fields ) != 2 {
exitFailure ( "Malformed mongo config string. Got \"%v\", want: \"user:password@host\"" , dbinfo )
}
host := fields [ 1 ]
fields = strings . Split ( fields [ 0 ] , ":" )
if len ( fields ) != 2 {
exitFailure ( "Malformed mongo config string. Got \"%v\", want: \"user:password\"" , fields [ 0 ] )
}
2012-03-15 12:31:06 +00:00
ob := map [ string ] interface { } { }
ob [ "enabled" ] = true
2014-04-01 16:03:20 +00:00
ob [ "handler" ] = "storage-index"
2012-03-15 12:31:06 +00:00
ob [ "handlerArgs" ] = map [ string ] interface { } {
"blobSource" : "/bs/" ,
2014-04-01 16:03:20 +00:00
"storage" : map [ string ] interface { } {
"type" : "mongo" ,
"host" : host ,
"user" : fields [ 0 ] ,
"password" : fields [ 1 ] ,
"database" : dbname ,
} ,
2012-03-15 12:31:06 +00:00
}
2014-04-01 16:03:20 +00:00
prefixes [ "/index/" ] = ob
2012-03-15 12:31:06 +00:00
}
2012-12-22 00:13:36 +00:00
func addSQLConfig ( rdbms string , prefixes jsonconfig . Obj , dbname string , dbinfo string ) {
2012-03-15 12:31:06 +00:00
fields := strings . Split ( dbinfo , "@" )
if len ( fields ) != 2 {
2012-11-03 18:58:50 +00:00
exitFailure ( "Malformed " + rdbms + " config string. Want: \"user@host:password\"" )
2012-03-15 12:31:06 +00:00
}
user := fields [ 0 ]
fields = strings . Split ( fields [ 1 ] , ":" )
if len ( fields ) != 2 {
2012-11-03 18:58:50 +00:00
exitFailure ( "Malformed " + rdbms + " config string. Want: \"user@host:password\"" )
2012-03-15 12:31:06 +00:00
}
ob := map [ string ] interface { } { }
ob [ "enabled" ] = true
2014-04-01 16:03:20 +00:00
ob [ "handler" ] = "storage-index"
2012-03-15 12:31:06 +00:00
ob [ "handlerArgs" ] = map [ string ] interface { } {
"blobSource" : "/bs/" ,
2014-04-01 16:03:20 +00:00
"storage" : map [ string ] interface { } {
"type" : rdbms ,
"host" : fields [ 0 ] ,
"user" : user ,
"password" : fields [ 1 ] ,
"database" : dbname ,
} ,
2012-03-15 12:31:06 +00:00
}
2014-04-01 16:03:20 +00:00
prefixes [ "/index/" ] = ob
2012-11-03 18:58:50 +00:00
}
2012-12-22 00:13:36 +00:00
func addPostgresConfig ( prefixes jsonconfig . Obj , dbname string , dbinfo string ) {
2012-11-03 18:58:50 +00:00
addSQLConfig ( "postgres" , prefixes , dbname , dbinfo )
}
2012-12-22 00:13:36 +00:00
func addMySQLConfig ( prefixes jsonconfig . Obj , dbname string , dbinfo string ) {
2012-11-03 18:58:50 +00:00
addSQLConfig ( "mysql" , prefixes , dbname , dbinfo )
2012-03-15 12:31:06 +00:00
}
2013-01-10 23:29:08 +00:00
func addSQLiteConfig ( prefixes jsonconfig . Obj , file string ) {
ob := map [ string ] interface { } { }
2014-04-01 16:03:20 +00:00
ob [ "handler" ] = "storage-index"
2013-01-10 23:29:08 +00:00
ob [ "handlerArgs" ] = map [ string ] interface { } {
"blobSource" : "/bs/" ,
2014-04-01 16:03:20 +00:00
"storage" : map [ string ] interface { } {
"type" : "sqlite" ,
"file" : file ,
} ,
2013-01-10 23:29:08 +00:00
}
2014-04-01 16:03:20 +00:00
prefixes [ "/index/" ] = ob
2013-01-10 23:29:08 +00:00
}
2013-08-25 17:25:30 +00:00
func addKVConfig ( prefixes jsonconfig . Obj , file string ) {
2014-04-01 16:03:20 +00:00
prefixes [ "/index/" ] = map [ string ] interface { } {
"handler" : "storage-index" ,
2013-08-25 17:25:30 +00:00
"handlerArgs" : map [ string ] interface { } {
"blobSource" : "/bs/" ,
2014-04-01 16:03:20 +00:00
"storage" : map [ string ] interface { } {
"type" : "kv" ,
"file" : file ,
} ,
2013-08-25 17:25:30 +00:00
} ,
}
}
2013-11-24 23:12:11 +00:00
func addS3Config ( params * configPrefixesParams , prefixes jsonconfig . Obj , s3 string ) error {
2013-09-01 16:50:35 +00:00
f := strings . SplitN ( s3 , ":" , 4 )
if len ( f ) < 3 {
2012-12-22 00:13:36 +00:00
return errors . New ( ` genconfig: expected "s3" field to be of form "access_key_id:secret_access_key:bucket" ` )
}
accessKey , secret , bucket := f [ 0 ] , f [ 1 ] , f [ 2 ]
2013-09-01 16:50:35 +00:00
var hostname string
if len ( f ) == 4 {
hostname = f [ 3 ]
}
2013-01-08 18:44:59 +00:00
isPrimary := false
if _ , ok := prefixes [ "/bs/" ] ; ! ok {
isPrimary = true
}
s3Prefix := ""
if isPrimary {
s3Prefix = "/bs/"
} else {
s3Prefix = "/sto-s3/"
}
2013-09-01 16:50:35 +00:00
args := map [ string ] interface { } {
"aws_access_key" : accessKey ,
"aws_secret_access_key" : secret ,
"bucket" : bucket ,
}
if hostname != "" {
args [ "hostname" ] = hostname
}
2012-12-22 00:13:36 +00:00
prefixes [ s3Prefix ] = map [ string ] interface { } {
2013-09-01 16:50:35 +00:00
"handler" : "storage-s3" ,
"handlerArgs" : args ,
2012-12-22 00:13:36 +00:00
}
2013-01-08 18:44:59 +00:00
if isPrimary {
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
prefixes [ "/cache/" ] = map [ string ] interface { } {
"handler" : "storage-filesystem" ,
"handlerArgs" : map [ string ] interface { } {
2013-01-11 18:52:22 +00:00
"path" : filepath . Join ( tempDir ( ) , "camli-cache" ) ,
2013-01-08 18:44:59 +00:00
} ,
}
} else {
2013-11-24 23:12:11 +00:00
if params . blobPath == "" {
panic ( "unexpected empty blobpath with sync-to-s3" )
}
2013-01-08 18:44:59 +00:00
prefixes [ "/sync-to-s3/" ] = map [ string ] interface { } {
"handler" : "sync" ,
"handlerArgs" : map [ string ] interface { } {
"from" : "/bs/" ,
"to" : s3Prefix ,
2013-11-24 23:12:11 +00:00
"queue" : map [ string ] interface { } {
"type" : "kv" ,
"file" : filepath . Join ( params . blobPath , "sync-to-s3-queue.kv" ) ,
} ,
2013-01-08 18:44:59 +00:00
} ,
}
2012-12-22 00:13:36 +00:00
}
return nil
2012-03-15 12:31:06 +00:00
}
2014-03-09 09:11:57 +00:00
func addGoogleDriveConfig ( params * configPrefixesParams , prefixes jsonconfig . Obj , highCfg string ) error {
2013-07-06 20:29:17 +00:00
f := strings . SplitN ( highCfg , ":" , 4 )
if len ( f ) != 4 {
2013-08-11 15:07:18 +00:00
return errors . New ( ` genconfig: expected "googledrive" field to be of form "client_id:client_secret:refresh_token:parent_id" ` )
}
clientId , secret , refreshToken , parentId := f [ 0 ] , f [ 1 ] , f [ 2 ] , f [ 3 ]
isPrimary := false
if _ , ok := prefixes [ "/bs/" ] ; ! ok {
isPrimary = true
}
prefix := ""
if isPrimary {
prefix = "/bs/"
} else {
prefix = "/sto-googledrive/"
}
prefixes [ prefix ] = map [ string ] interface { } {
"handler" : "storage-googledrive" ,
"handlerArgs" : map [ string ] interface { } {
"parent_id" : parentId ,
"auth" : map [ string ] interface { } {
"client_id" : clientId ,
"client_secret" : secret ,
"refresh_token" : refreshToken ,
} ,
} ,
}
if isPrimary {
prefixes [ "/cache/" ] = map [ string ] interface { } {
"handler" : "storage-filesystem" ,
"handlerArgs" : map [ string ] interface { } {
"path" : filepath . Join ( tempDir ( ) , "camli-cache" ) ,
} ,
}
} else {
prefixes [ "/sync-to-googledrive/" ] = map [ string ] interface { } {
"handler" : "sync" ,
"handlerArgs" : map [ string ] interface { } {
"from" : "/bs/" ,
"to" : prefix ,
2014-03-09 09:11:57 +00:00
"queue" : map [ string ] interface { } {
"type" : "kv" ,
"file" : filepath . Join ( params . blobPath ,
"sync-to-googledrive-queue.kv" ) ,
} ,
2013-08-11 15:07:18 +00:00
} ,
}
}
return nil
}
2014-08-03 02:29:57 +00:00
var errGCSUsage = errors . New ( ` genconfig: expected "googlecloudstorage" field to be of form "client_id:client_secret:refresh_token:bucket" or ":bucketname" ` )
2014-03-09 09:11:57 +00:00
func addGoogleCloudStorageConfig ( params * configPrefixesParams , prefixes jsonconfig . Obj , highCfg string ) error {
2014-08-03 02:29:57 +00:00
var clientID , secret , refreshToken , bucket string
2013-08-11 15:07:18 +00:00
f := strings . SplitN ( highCfg , ":" , 4 )
2014-08-03 02:29:57 +00:00
switch len ( f ) {
default :
return errGCSUsage
case 4 :
clientID , secret , refreshToken , bucket = f [ 0 ] , f [ 1 ] , f [ 2 ] , f [ 3 ]
case 2 :
if f [ 0 ] != "" {
return errGCSUsage
}
bucket = f [ 1 ]
clientID = "auto"
2013-07-06 20:29:17 +00:00
}
isPrimary := false
if _ , ok := prefixes [ "/bs/" ] ; ! ok {
isPrimary = true
}
gsPrefix := ""
if isPrimary {
gsPrefix = "/bs/"
} else {
2013-08-11 15:07:18 +00:00
gsPrefix = "/sto-googlecloudstorage/"
2013-07-06 20:29:17 +00:00
}
prefixes [ gsPrefix ] = map [ string ] interface { } {
2013-08-11 15:07:18 +00:00
"handler" : "storage-googlecloudstorage" ,
2013-07-06 20:29:17 +00:00
"handlerArgs" : map [ string ] interface { } {
"bucket" : bucket ,
"auth" : map [ string ] interface { } {
2014-08-03 02:29:57 +00:00
"client_id" : clientID ,
2013-07-06 20:29:17 +00:00
"client_secret" : secret ,
"refresh_token" : refreshToken ,
// If high-level config is for the common user then fullSyncOnStart = true
// Then the default just works.
//"fullSyncOnStart": true,
//"blockingFullSyncOnStart": false
} ,
} ,
}
if isPrimary {
// TODO: cacheBucket like s3CacheBucket?
prefixes [ "/cache/" ] = map [ string ] interface { } {
"handler" : "storage-filesystem" ,
"handlerArgs" : map [ string ] interface { } {
"path" : filepath . Join ( tempDir ( ) , "camli-cache" ) ,
} ,
}
} else {
2013-08-11 15:07:18 +00:00
prefixes [ "/sync-to-googlecloudstorage/" ] = map [ string ] interface { } {
2013-07-06 20:29:17 +00:00
"handler" : "sync" ,
"handlerArgs" : map [ string ] interface { } {
"from" : "/bs/" ,
"to" : gsPrefix ,
2014-03-09 09:11:57 +00:00
"queue" : map [ string ] interface { } {
"type" : "kv" ,
"file" : filepath . Join ( params . blobPath ,
"sync-to-googlecloud-queue.kv" ) ,
} ,
2013-07-06 20:29:17 +00:00
} ,
}
}
return nil
}
2013-06-26 19:55:47 +00:00
func genLowLevelPrefixes ( params * configPrefixesParams , ownerName string ) ( m jsonconfig . Obj ) {
2012-12-21 23:47:08 +00:00
m = make ( jsonconfig . Obj )
2012-03-15 12:31:06 +00:00
2014-04-01 16:03:20 +00:00
haveIndex := params . haveIndex
2013-01-10 23:29:08 +00:00
root := "/bs/"
pubKeyDest := root
if haveIndex {
root = "/bs-and-maybe-also-index/"
pubKeyDest = "/bs-and-index/"
}
2013-06-26 19:55:47 +00:00
rootArgs := map [ string ] interface { } {
"stealth" : false ,
"blobRoot" : root ,
"statusRoot" : "/status/" ,
}
if ownerName != "" {
rootArgs [ "ownerName" ] = ownerName
}
2012-12-21 23:47:08 +00:00
m [ "/" ] = map [ string ] interface { } {
2013-06-26 19:55:47 +00:00
"handler" : "root" ,
"handlerArgs" : rootArgs ,
2012-11-08 14:27:17 +00:00
}
2013-01-10 23:29:08 +00:00
if haveIndex {
setMap ( m , "/" , "handlerArgs" , "searchRoot" , "/my-search/" )
}
2012-03-15 12:31:06 +00:00
2012-12-21 23:47:08 +00:00
m [ "/setup/" ] = map [ string ] interface { } {
"handler" : "setup" ,
}
2012-04-03 21:39:49 +00:00
2013-06-05 17:18:27 +00:00
m [ "/status/" ] = map [ string ] interface { } {
"handler" : "status" ,
}
2014-04-17 17:23:53 +00:00
importerArgs := map [ string ] interface { } { }
2014-07-04 22:25:28 +00:00
if haveIndex {
m [ "/importer/" ] = map [ string ] interface { } {
"handler" : "importer" ,
"handlerArgs" : importerArgs ,
}
2014-04-17 17:23:53 +00:00
}
2013-06-05 17:18:27 +00:00
2013-07-16 15:55:20 +00:00
if params . shareHandlerPath != "" {
m [ params . shareHandlerPath ] = map [ string ] interface { } {
2013-02-28 23:30:16 +00:00
"handler" : "share" ,
"handlerArgs" : map [ string ] interface { } {
"blobRoot" : "/bs/" ,
} ,
}
}
2012-12-21 23:47:08 +00:00
m [ "/sighelper/" ] = map [ string ] interface { } {
"handler" : "jsonsign" ,
"handlerArgs" : map [ string ] interface { } {
"secretRing" : params . secretRing ,
"keyId" : params . keyId ,
2013-01-10 23:29:08 +00:00
"publicKeyDest" : pubKeyDest ,
2012-03-15 12:31:06 +00:00
} ,
}
2013-11-25 10:23:50 +00:00
storageType := "filesystem"
if params . packBlobs {
storageType = "diskpacked"
}
2013-01-08 18:44:59 +00:00
if params . blobPath != "" {
m [ "/bs/" ] = map [ string ] interface { } {
2013-11-25 10:23:50 +00:00
"handler" : "storage-" + storageType ,
2013-01-08 18:44:59 +00:00
"handlerArgs" : map [ string ] interface { } {
"path" : params . blobPath ,
} ,
}
2012-03-15 12:31:06 +00:00
2013-01-08 18:44:59 +00:00
m [ "/cache/" ] = map [ string ] interface { } {
2013-11-28 04:50:16 +00:00
"handler" : "storage-" + storageType ,
2013-01-08 18:44:59 +00:00
"handlerArgs" : map [ string ] interface { } {
"path" : filepath . Join ( params . blobPath , "/cache" ) ,
} ,
}
2012-03-15 12:31:06 +00:00
}
2013-11-19 04:53:46 +00:00
if params . flickr != "" {
2014-04-17 17:23:53 +00:00
importerArgs [ "flickr" ] = map [ string ] interface { } {
"clientSecret" : params . flickr ,
2013-11-17 21:52:45 +00:00
}
}
2014-03-04 21:25:10 +00:00
if params . picasa != "" {
2014-04-17 17:23:53 +00:00
importerArgs [ "picasa" ] = map [ string ] interface { } {
"clientSecret" : params . picasa ,
2014-03-04 21:25:10 +00:00
}
}
2013-11-17 21:52:45 +00:00
2013-01-10 23:29:08 +00:00
if haveIndex {
2013-08-20 18:11:37 +00:00
syncArgs := map [ string ] interface { } {
"from" : "/bs/" ,
2014-04-01 16:03:20 +00:00
"to" : "/index/" ,
2013-08-20 18:11:37 +00:00
}
2014-03-17 15:42:19 +00:00
// TODO: currently when using s3, the index must be
// sqlite or kvfile, since only through one of those
// can we get a directory.
if params . blobPath == "" && params . indexFileDir == "" {
// We don't actually have a working sync handler, but we keep a stub registered
// so it can be referred to from other places.
// See http://camlistore.org/issue/201
2013-08-20 18:11:37 +00:00
syncArgs [ "idle" ] = true
2013-11-24 23:12:11 +00:00
} else {
2014-03-17 15:42:19 +00:00
dir := params . blobPath
if dir == "" {
dir = params . indexFileDir
}
typ := "kv"
2014-04-01 16:03:20 +00:00
if params . haveSQLite {
2014-03-17 15:42:19 +00:00
typ = "sqlite"
}
2013-11-24 23:12:11 +00:00
syncArgs [ "queue" ] = map [ string ] interface { } {
2014-03-17 15:42:19 +00:00
"type" : typ ,
"file" : filepath . Join ( dir , "sync-to-index-queue." + typ ) ,
2013-11-24 23:12:11 +00:00
}
2013-08-20 18:11:37 +00:00
}
2013-01-10 23:29:08 +00:00
m [ "/sync/" ] = map [ string ] interface { } {
2013-08-20 18:11:37 +00:00
"handler" : "sync" ,
"handlerArgs" : syncArgs ,
2013-01-10 23:29:08 +00:00
}
m [ "/bs-and-index/" ] = map [ string ] interface { } {
"handler" : "storage-replica" ,
"handlerArgs" : map [ string ] interface { } {
2014-04-01 16:03:20 +00:00
"backends" : [ ] interface { } { "/bs/" , "/index/" } ,
2013-01-10 23:29:08 +00:00
} ,
}
m [ "/bs-and-maybe-also-index/" ] = map [ string ] interface { } {
"handler" : "storage-cond" ,
"handlerArgs" : map [ string ] interface { } {
"write" : map [ string ] interface { } {
"if" : "isSchema" ,
"then" : "/bs-and-index/" ,
"else" : "/bs/" ,
} ,
"read" : "/bs/" ,
} ,
}
2013-11-28 19:09:16 +00:00
searchArgs := map [ string ] interface { } {
2014-04-01 16:03:20 +00:00
"index" : "/index/" ,
2013-12-11 08:20:22 +00:00
"owner" : params . searchOwner . String ( ) ,
2013-11-28 19:09:16 +00:00
}
if params . memoryIndex {
searchArgs [ "slurpToMemory" ] = true
}
m [ "/my-search/" ] = map [ string ] interface { } {
2013-12-11 08:20:22 +00:00
"handler" : "search" ,
2013-11-28 19:09:16 +00:00
"handlerArgs" : searchArgs ,
2013-01-10 23:29:08 +00:00
}
2012-03-15 12:31:06 +00:00
}
2012-12-21 23:47:08 +00:00
return
2012-03-15 12:31:06 +00:00
}
2012-11-07 16:51:42 +00:00
// genLowLevelConfig returns a low-level config from a high-level config.
2014-01-23 22:40:12 +00:00
func genLowLevelConfig ( conf * serverconfig . Config ) ( lowLevelConf * Config , err error ) {
2012-03-19 20:09:00 +00:00
obj := jsonconfig . Obj { }
2014-01-23 22:40:12 +00:00
if conf . HTTPS {
if ( conf . HTTPSCert != "" ) != ( conf . HTTPSKey != "" ) {
return nil , errors . New ( "Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)" )
2012-08-04 01:12:39 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . HTTPSCert != "" {
obj [ "httpsCert" ] = conf . HTTPSCert
obj [ "httpsKey" ] = conf . HTTPSKey
2012-08-04 01:12:39 +00:00
} else {
2014-01-23 22:40:12 +00:00
obj [ "httpsCert" ] = osutil . DefaultTLSCert ( )
obj [ "httpsKey" ] = osutil . DefaultTLSKey ( )
2012-08-04 01:12:39 +00:00
}
2012-03-15 12:31:06 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . BaseURL != "" {
u , err := url . Parse ( conf . BaseURL )
2013-08-09 00:50:50 +00:00
if err != nil {
2014-01-23 22:40:12 +00:00
return nil , fmt . Errorf ( "Error parsing baseURL %q as a URL: %v" , conf . BaseURL , err )
2013-08-09 00:50:50 +00:00
}
2013-09-08 05:32:39 +00:00
if u . Path != "" && u . Path != "/" {
return nil , fmt . Errorf ( "baseURL can't have a path, only a scheme, host, and optional port." )
2012-09-16 22:20:49 +00:00
}
2013-09-08 05:32:39 +00:00
u . Path = ""
obj [ "baseURL" ] = u . String ( )
2012-09-16 22:20:49 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . Listen != "" {
obj [ "listen" ] = conf . Listen
2012-08-04 11:42:10 +00:00
}
2014-01-23 22:40:12 +00:00
obj [ "https" ] = conf . HTTPS
obj [ "auth" ] = conf . Auth
2012-08-04 11:42:10 +00:00
2013-08-28 13:53:58 +00:00
username := ""
2014-01-23 22:40:12 +00:00
if conf . DBName == "" {
2013-08-28 13:53:58 +00:00
username = osutil . Username ( )
2012-03-15 12:31:06 +00:00
if username == "" {
2013-08-28 13:53:58 +00:00
return nil , fmt . Errorf ( "USER (USERNAME on windows) env var not set; needed to define dbname" )
2012-03-15 12:31:06 +00:00
}
2014-01-23 22:40:12 +00:00
conf . DBName = "camli" + username
2012-03-15 12:31:06 +00:00
}
2014-04-01 16:03:20 +00:00
var haveSQLite bool
2014-03-17 15:42:19 +00:00
var indexFileDir string // filesystem directory of sqlite, kv, or similar
2014-01-23 22:40:12 +00:00
numIndexers := numSet ( conf . Mongo , conf . MySQL , conf . PostgreSQL , conf . SQLite , conf . KVFile )
runIndex := conf . RunIndex . Get ( )
2014-04-01 16:03:20 +00:00
2012-04-12 02:22:02 +00:00
switch {
2013-01-10 23:29:08 +00:00
case runIndex && numIndexers == 0 :
2013-12-11 08:20:22 +00:00
return nil , fmt . Errorf ( "Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite)." )
2013-01-10 23:29:08 +00:00
case runIndex && numIndexers != 1 :
2013-12-11 08:20:22 +00:00
return nil , fmt . Errorf ( "With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite)." )
2013-01-10 23:29:08 +00:00
case ! runIndex && numIndexers != 0 :
2013-12-11 08:20:22 +00:00
return nil , fmt . Errorf ( "With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite." )
2014-01-23 22:40:12 +00:00
case conf . SQLite != "" :
2014-04-01 16:03:20 +00:00
haveSQLite = true
2014-03-17 15:42:19 +00:00
indexFileDir = filepath . Dir ( conf . SQLite )
2014-01-23 22:40:12 +00:00
case conf . KVFile != "" :
2014-03-17 15:42:19 +00:00
indexFileDir = filepath . Dir ( conf . KVFile )
2012-04-12 02:22:02 +00:00
}
2012-03-15 12:31:06 +00:00
2014-08-02 23:53:58 +00:00
if conf . Identity == "" {
return nil , errors . New ( "no 'identity' in server config" )
}
2014-01-23 22:40:12 +00:00
entity , err := jsonsign . EntityFromSecring ( conf . Identity , conf . IdentitySecretRing )
2012-04-12 18:39:53 +00:00
if err != nil {
return nil , err
}
armoredPublicKey , err := jsonsign . ArmoredPublicKey ( entity )
if err != nil {
return nil , err
}
2014-01-23 22:40:12 +00:00
nolocaldisk := conf . BlobPath == ""
2013-08-20 18:11:37 +00:00
if nolocaldisk {
2014-01-23 22:40:12 +00:00
if conf . S3 == "" && conf . GoogleCloudStorage == "" {
2013-08-20 18:11:37 +00:00
return nil , errors . New ( "You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver." )
}
2014-01-23 22:40:12 +00:00
if conf . S3 != "" && conf . GoogleCloudStorage != "" {
2013-08-20 18:11:37 +00:00
return nil , errors . New ( "Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now." )
}
2013-01-08 18:44:59 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . ShareHandler && conf . ShareHandlerPath == "" {
conf . ShareHandlerPath = "/share/"
2013-07-16 15:55:20 +00:00
}
2012-03-15 12:31:06 +00:00
prefixesParams := & configPrefixesParams {
2014-01-23 22:40:12 +00:00
secretRing : conf . IdentitySecretRing ,
keyId : conf . Identity ,
2014-04-01 16:03:20 +00:00
haveIndex : runIndex ,
haveSQLite : haveSQLite ,
2014-01-23 22:40:12 +00:00
blobPath : conf . BlobPath ,
packBlobs : conf . PackBlobs ,
2013-08-04 02:54:30 +00:00
searchOwner : blob . SHA1FromString ( armoredPublicKey ) ,
2014-01-23 22:40:12 +00:00
shareHandlerPath : conf . ShareHandlerPath ,
flickr : conf . Flickr ,
2014-03-04 21:25:10 +00:00
picasa : conf . Picasa ,
2014-01-23 22:40:12 +00:00
memoryIndex : conf . MemoryIndex . Get ( ) ,
2014-03-17 15:42:19 +00:00
indexFileDir : indexFileDir ,
2012-03-15 12:31:06 +00:00
}
2014-01-23 22:40:12 +00:00
prefixes := genLowLevelPrefixes ( prefixesParams , conf . OwnerName )
2013-01-08 18:44:59 +00:00
var cacheDir string
if nolocaldisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
2013-01-11 18:52:22 +00:00
cacheDir = filepath . Join ( tempDir ( ) , "camli-cache" )
2013-01-08 18:44:59 +00:00
} else {
2014-01-23 22:40:12 +00:00
cacheDir = filepath . Join ( conf . BlobPath , "cache" )
2013-01-08 18:44:59 +00:00
}
2013-08-25 17:25:30 +00:00
if ! noMkdir {
if err := os . MkdirAll ( cacheDir , 0700 ) ; err != nil {
return nil , fmt . Errorf ( "Could not create blobs cache dir %s: %v" , cacheDir , err )
}
2012-03-15 12:31:06 +00:00
}
2014-01-23 22:40:12 +00:00
if len ( conf . Publish ) > 0 {
2013-01-10 23:29:08 +00:00
if ! runIndex {
return nil , fmt . Errorf ( "publishing requires an index" )
}
2014-06-16 16:05:39 +00:00
var tlsO * tlsOpts
httpsCert , ok1 := obj [ "httpsCert" ] . ( string )
httpsKey , ok2 := obj [ "httpsKey" ] . ( string )
if ok1 && ok2 {
tlsO = & tlsOpts {
httpsCert : httpsCert ,
httpsKey : httpsKey ,
}
}
_ , err = addPublishedConfig ( prefixes , conf . Publish , conf . SourceRoot , tlsO )
2012-04-22 15:33:22 +00:00
if err != nil {
return nil , fmt . Errorf ( "Could not generate config for published: %v" , err )
}
}
2013-01-10 23:29:08 +00:00
if runIndex {
2014-06-14 20:14:34 +00:00
addUIConfig ( prefixesParams , prefixes , "/ui/" , conf . SourceRoot )
2013-01-10 23:29:08 +00:00
}
2012-03-15 12:31:06 +00:00
2014-01-23 22:40:12 +00:00
if conf . MySQL != "" {
addMySQLConfig ( prefixes , conf . DBName , conf . MySQL )
2012-04-12 02:22:02 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . PostgreSQL != "" {
addPostgresConfig ( prefixes , conf . DBName , conf . PostgreSQL )
2012-11-03 18:58:50 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . Mongo != "" {
addMongoConfig ( prefixes , conf . DBName , conf . Mongo )
2012-12-22 00:13:36 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . SQLite != "" {
addSQLiteConfig ( prefixes , conf . SQLite )
2013-01-10 23:29:08 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . KVFile != "" {
addKVConfig ( prefixes , conf . KVFile )
2013-08-25 17:25:30 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . S3 != "" {
if err := addS3Config ( prefixesParams , prefixes , conf . S3 ) ; err != nil {
2012-12-22 00:13:36 +00:00
return nil , err
}
2012-04-12 02:22:02 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . GoogleDrive != "" {
2014-03-09 09:11:57 +00:00
if err := addGoogleDriveConfig ( prefixesParams , prefixes , conf . GoogleDrive ) ; err != nil {
2013-08-11 15:07:18 +00:00
return nil , err
}
}
2014-01-23 22:40:12 +00:00
if conf . GoogleCloudStorage != "" {
2014-03-09 09:11:57 +00:00
if err := addGoogleCloudStorageConfig ( prefixesParams , prefixes , conf . GoogleCloudStorage ) ; err != nil {
2013-07-06 20:29:17 +00:00
return nil , err
}
}
2012-03-15 12:31:06 +00:00
obj [ "prefixes" ] = ( map [ string ] interface { } ) ( prefixes )
lowLevelConf = & Config {
2014-01-23 22:40:12 +00:00
Obj : obj ,
2012-03-15 12:31:06 +00:00
}
return lowLevelConf , nil
}
2013-01-10 23:29:08 +00:00
func numSet ( vv ... interface { } ) ( num int ) {
for _ , vi := range vv {
switch v := vi . ( type ) {
case string :
if v != "" {
num ++
}
case bool :
if v {
num ++
}
default :
panic ( "unknown type" )
}
}
return
}
func setMap ( m map [ string ] interface { } , v ... interface { } ) {
if len ( v ) < 2 {
panic ( "too few args" )
}
if len ( v ) == 2 {
m [ v [ 0 ] . ( string ) ] = v [ 1 ]
return
}
setMap ( m [ v [ 0 ] . ( string ) ] . ( map [ string ] interface { } ) , v [ 1 : ] ... )
}
2014-04-01 16:03:20 +00:00
// WriteDefaultConfigFile generates a new default high-level server configuration
// file at filePath. If useSQLite, the default indexer will use SQLite, otherwise
// kv. If filePath already exists, it is overwritten.
func WriteDefaultConfigFile ( filePath string , useSQLite bool ) error {
conf := serverconfig . Config {
Listen : ":3179" ,
HTTPS : false ,
Auth : "localhost" ,
ReplicateTo : make ( [ ] interface { } , 0 ) ,
}
blobDir := osutil . CamliBlobRoot ( )
if err := os . MkdirAll ( blobDir , 0700 ) ; err != nil {
return fmt . Errorf ( "Could not create default blobs directory: %v" , err )
}
conf . BlobPath = blobDir
if useSQLite {
conf . SQLite = filepath . Join ( osutil . CamliVarDir ( ) , "camli-index.db" )
} else {
conf . KVFile = filepath . Join ( osutil . CamliVarDir ( ) , "camli-index.kvdb" )
}
var keyId string
secRing := osutil . SecretRingFile ( )
_ , err := os . Stat ( secRing )
switch {
case err == nil :
keyId , err = jsonsign . KeyIdFromRing ( secRing )
if err != nil {
return fmt . Errorf ( "Could not find any keyId in file %q: %v" , secRing , err )
}
log . Printf ( "Re-using identity with keyId %q found in file %s" , keyId , secRing )
case os . IsNotExist ( err ) :
keyId , err = jsonsign . GenerateNewSecRing ( secRing )
if err != nil {
return fmt . Errorf ( "Could not generate new secRing at file %q: %v" , secRing , err )
}
log . Printf ( "Generated new identity with keyId %q in file %s" , keyId , secRing )
}
if err != nil {
return fmt . Errorf ( "Could not stat secret ring %q: %v" , secRing , err )
}
conf . Identity = keyId
conf . IdentitySecretRing = secRing
confData , err := json . MarshalIndent ( conf , "" , " " )
if err != nil {
return fmt . Errorf ( "Could not json encode config file : %v" , err )
}
if err := ioutil . WriteFile ( filePath , confData , 0600 ) ; err != nil {
return fmt . Errorf ( "Could not create or write default server config: %v" , err )
}
return nil
}