2012-03-15 12:31:06 +00:00
/ *
Copyright 2012 Google Inc .
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
2014-01-23 16:18:22 +00:00
package serverinit
2012-03-15 12:31:06 +00:00
import (
2014-04-01 16:03:20 +00:00
"encoding/json"
2012-08-04 01:12:39 +00:00
"errors"
2012-03-15 12:31:06 +00:00
"fmt"
2014-04-01 16:03:20 +00:00
"log"
2013-08-09 00:50:50 +00:00
"net/url"
2012-03-15 12:31:06 +00:00
"os"
"path/filepath"
2014-08-08 01:47:42 +00:00
"runtime"
2014-08-06 22:07:43 +00:00
"strconv"
2012-03-15 12:31:06 +00:00
"strings"
2013-08-04 02:54:30 +00:00
"camlistore.org/pkg/blob"
2012-03-15 12:31:06 +00:00
"camlistore.org/pkg/jsonconfig"
2012-04-12 18:39:53 +00:00
"camlistore.org/pkg/jsonsign"
2013-09-01 16:50:35 +00:00
"camlistore.org/pkg/osutil"
2014-01-23 22:40:12 +00:00
"camlistore.org/pkg/types/serverconfig"
2014-08-05 04:27:07 +00:00
"camlistore.org/pkg/wkfs"
2012-03-15 12:31:06 +00:00
)
2013-08-25 17:25:30 +00:00
var (
tempDir = os . TempDir
noMkdir bool // for tests to not call os.Mkdir
)
2013-01-11 18:52:22 +00:00
2014-06-16 16:05:39 +00:00
type tlsOpts struct {
httpsCert string
httpsKey string
}
2014-08-08 01:47:42 +00:00
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig ( conf * serverconfig . Config ) ( lowLevelConf * Config , err error ) {
b := & lowBuilder {
high : conf ,
low : jsonconfig . Obj {
"prefixes" : make ( map [ string ] interface { } ) ,
} ,
}
return b . build ( )
}
// A lowBuilder builds a low-level config from a high-level config.
type lowBuilder struct {
high * serverconfig . Config // high-level config (input)
low jsonconfig . Obj // low-level handler config (output)
}
// args is an alias for map[string]interface{} just to cut down on
// noise below. But we take care to convert it back to
// map[string]interface{} in the one place where we accept it.
type args map [ string ] interface { }
func ( b * lowBuilder ) addPrefix ( at , handler string , a args ) {
v := map [ string ] interface { } {
"handler" : handler ,
}
if a != nil {
v [ "handlerArgs" ] = ( map [ string ] interface { } ) ( a )
}
b . low [ "prefixes" ] . ( map [ string ] interface { } ) [ at ] = v
}
func ( b * lowBuilder ) hasPrefix ( p string ) bool {
_ , ok := b . low [ "prefixes" ] . ( map [ string ] interface { } ) [ p ]
return ok
}
func ( b * lowBuilder ) runIndex ( ) bool { return b . high . RunIndex . Get ( ) }
func ( b * lowBuilder ) memoryIndex ( ) bool { return b . high . MemoryIndex . Get ( ) }
// dbName returns which database to use for the provided user ("of").
// The user key be a key as describe in pkg/types/serverconfig/config.go's
// description of DBNames: "index", "queue-sync-to-index", etc.
func ( b * lowBuilder ) dbName ( of string ) string {
if v , ok := b . high . DBNames [ of ] ; ok && v != "" {
return v
}
if of == "index" {
if b . high . DBName != "" {
return b . high . DBName
}
username := osutil . Username ( )
if username == "" {
envVar := "USER"
if runtime . GOOS == "windows" {
envVar += "NAME"
}
2014-08-19 06:17:17 +00:00
return "camlistore_index"
2014-08-08 01:47:42 +00:00
}
return "camli" + username
}
2014-08-23 16:46:23 +00:00
return ""
2014-08-08 01:47:42 +00:00
}
var errNoOwner = errors . New ( "no owner" )
// Error is errNoOwner if no identity configured
func ( b * lowBuilder ) searchOwner ( ) ( br blob . Ref , err error ) {
if b . high . Identity == "" {
return br , errNoOwner
}
entity , err := jsonsign . EntityFromSecring ( b . high . Identity , b . high . IdentitySecretRing )
if err != nil {
return br , err
}
armoredPublicKey , err := jsonsign . ArmoredPublicKey ( entity )
if err != nil {
return br , err
}
return blob . SHA1FromString ( armoredPublicKey ) , nil
}
func ( b * lowBuilder ) addPublishedConfig ( tlsO * tlsOpts ) error {
published := b . high . Publish
2012-04-22 15:33:22 +00:00
for k , v := range published {
2014-06-14 20:14:34 +00:00
if v . CamliRoot == "" {
2014-08-08 01:47:42 +00:00
return fmt . Errorf ( "Missing \"camliRoot\" key in configuration for %s." , k )
2012-04-22 15:33:22 +00:00
}
2014-01-23 22:40:12 +00:00
if v . GoTemplate == "" {
2014-08-08 01:47:42 +00:00
return fmt . Errorf ( "Missing \"goTemplate\" key in configuration for %s." , k )
2012-04-22 15:33:22 +00:00
}
2014-06-14 20:14:34 +00:00
appConfig := map [ string ] interface { } {
"camliRoot" : v . CamliRoot ,
"cacheRoot" : v . CacheRoot ,
"goTemplate" : v . GoTemplate ,
}
2014-06-16 16:05:39 +00:00
if v . HTTPSCert != "" && v . HTTPSKey != "" {
// user can specify these directly in the publish section
appConfig [ "httpsCert" ] = v . HTTPSCert
appConfig [ "httpsKey" ] = v . HTTPSKey
} else {
// default to Camlistore parameters, if any
if tlsO != nil {
appConfig [ "httpsCert" ] = tlsO . httpsCert
appConfig [ "httpsKey" ] = tlsO . httpsKey
}
}
2014-08-08 01:47:42 +00:00
a := args {
2014-06-14 20:14:34 +00:00
"program" : v . Program ,
"appConfig" : appConfig ,
2013-12-19 16:33:24 +00:00
}
2014-06-14 20:14:34 +00:00
if v . BaseURL != "" {
2014-08-08 01:47:42 +00:00
a [ "baseURL" ] = v . BaseURL
2013-12-19 16:33:24 +00:00
}
2014-06-14 20:14:34 +00:00
program := "publisher"
if v . Program != "" {
program = v . Program
}
2014-08-08 01:47:42 +00:00
a [ "program" ] = program
b . addPrefix ( k , "app" , a )
2012-04-22 15:33:22 +00:00
}
2014-08-08 01:47:42 +00:00
return nil
2012-04-22 15:33:22 +00:00
}
2014-08-08 01:47:42 +00:00
func ( b * lowBuilder ) addUIConfig ( ) {
2013-12-14 17:37:56 +00:00
args := map [ string ] interface { } {
2012-03-15 12:31:06 +00:00
"jsonSignRoot" : "/sighelper/" ,
"cache" : "/cache/" ,
}
2014-08-08 01:47:42 +00:00
if b . high . SourceRoot != "" {
args [ "sourceRoot" ] = b . high . SourceRoot
2013-12-14 17:37:56 +00:00
}
2014-08-24 02:54:32 +00:00
var thumbCache map [ string ] interface { }
2014-08-08 01:47:42 +00:00
if b . high . BlobPath != "" {
2014-08-24 02:54:32 +00:00
thumbCache = map [ string ] interface { } {
2013-12-14 17:37:56 +00:00
"type" : "kv" ,
2014-08-08 01:47:42 +00:00
"file" : filepath . Join ( b . high . BlobPath , "thumbmeta.kv" ) ,
2013-12-14 17:37:56 +00:00
}
}
2014-08-24 02:54:32 +00:00
if thumbCache == nil {
sorted , err := b . sortedStorage ( "ui_thumbcache" )
if err == nil {
thumbCache = sorted
}
}
if thumbCache != nil {
args [ "scaledImage" ] = thumbCache
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/ui/" , "ui" , args )
2012-03-15 12:31:06 +00:00
}
2014-08-23 16:46:23 +00:00
func ( b * lowBuilder ) mongoIndexStorage ( confStr , sortedType string ) ( map [ string ] interface { } , error ) {
2014-08-24 02:54:32 +00:00
dbName := b . dbName ( sortedType )
if dbName == "" {
return nil , fmt . Errorf ( "no database name configured for sorted store %q" , sortedType )
}
2014-08-08 01:47:42 +00:00
fields := strings . Split ( confStr , "@" )
if len ( fields ) == 2 {
host := fields [ 1 ]
fields = strings . Split ( fields [ 0 ] , ":" )
if len ( fields ) == 2 {
user , pass := fields [ 0 ] , fields [ 1 ]
return map [ string ] interface { } {
"type" : "mongo" ,
"host" : host ,
"user" : user ,
"password" : pass ,
2014-08-24 02:54:32 +00:00
"database" : dbName ,
2014-08-08 01:47:42 +00:00
} , nil
}
}
return nil , errors . New ( "Malformed mongo config string; want form: \"user:password@host\"" )
2012-03-15 12:31:06 +00:00
}
2014-08-06 22:07:43 +00:00
// parses "user@host:password", which you think would be easy, but we
// documented this format without thinking about port numbers, so this
// uses heuristics to guess what extra colons mean.
func parseUserHostPass ( v string ) ( user , host , password string , ok bool ) {
f := strings . SplitN ( v , "@" , 2 )
if len ( f ) != 2 {
return
2012-03-15 12:31:06 +00:00
}
2014-08-06 22:07:43 +00:00
user = f [ 0 ]
f = strings . Split ( f [ 1 ] , ":" )
if len ( f ) < 2 {
return "" , "" , "" , false
}
host = f [ 0 ]
f = f [ 1 : ]
if len ( f ) >= 2 {
if _ , err := strconv . ParseUint ( f [ 0 ] , 10 , 16 ) ; err == nil {
host = host + ":" + f [ 0 ]
f = f [ 1 : ]
}
}
password = strings . Join ( f , ":" )
ok = true
return
}
2014-08-24 02:54:32 +00:00
func ( b * lowBuilder ) dbIndexStorage ( rdbms string , confStr string , sortedType string ) ( map [ string ] interface { } , error ) {
dbName := b . dbName ( sortedType )
if dbName == "" {
return nil , fmt . Errorf ( "no database name configured for sorted store %q" , sortedType )
}
2014-08-08 01:47:42 +00:00
user , host , password , ok := parseUserHostPass ( confStr )
2014-08-06 22:07:43 +00:00
if ! ok {
2014-08-08 01:47:42 +00:00
return nil , fmt . Errorf ( "Malformed %s config string. Want: \"user@host:password\"" , rdbms )
}
return map [ string ] interface { } {
"type" : rdbms ,
"host" : host ,
"user" : user ,
"password" : password ,
2014-08-24 02:54:32 +00:00
"database" : b . dbName ( sortedType ) ,
2014-08-08 01:47:42 +00:00
} , nil
2012-03-15 12:31:06 +00:00
}
2014-08-23 16:46:23 +00:00
func ( b * lowBuilder ) sortedStorage ( sortedType string ) ( map [ string ] interface { } , error ) {
2014-08-08 01:47:42 +00:00
if b . high . MySQL != "" {
2014-08-23 16:46:23 +00:00
return b . dbIndexStorage ( "mysql" , b . high . MySQL , sortedType )
2014-08-08 01:47:42 +00:00
}
if b . high . PostgreSQL != "" {
2014-08-23 16:46:23 +00:00
return b . dbIndexStorage ( "postgres" , b . high . PostgreSQL , sortedType )
2014-08-08 01:47:42 +00:00
}
if b . high . Mongo != "" {
2014-08-23 16:46:23 +00:00
return b . mongoIndexStorage ( b . high . Mongo , sortedType )
}
if sortedType != "index" {
return nil , fmt . Errorf ( "TODO: finish SQLite & KVFile for non-index queues" )
2014-08-08 01:47:42 +00:00
}
if b . high . SQLite != "" {
return map [ string ] interface { } {
2014-04-01 16:03:20 +00:00
"type" : "sqlite" ,
2014-08-08 01:47:42 +00:00
"file" : b . high . SQLite ,
} , nil
2013-01-10 23:29:08 +00:00
}
2014-08-08 01:47:42 +00:00
if b . high . KVFile != "" {
return map [ string ] interface { } {
"type" : "kv" ,
"file" : b . high . KVFile ,
} , nil
2013-08-25 17:25:30 +00:00
}
2014-08-08 01:47:42 +00:00
panic ( "indexArgs called when not in index mode" )
2013-08-25 17:25:30 +00:00
}
2014-08-08 01:47:42 +00:00
func ( b * lowBuilder ) addS3Config ( s3 string ) error {
2013-09-01 16:50:35 +00:00
f := strings . SplitN ( s3 , ":" , 4 )
if len ( f ) < 3 {
2012-12-22 00:13:36 +00:00
return errors . New ( ` genconfig: expected "s3" field to be of form "access_key_id:secret_access_key:bucket" ` )
}
accessKey , secret , bucket := f [ 0 ] , f [ 1 ] , f [ 2 ]
2013-09-01 16:50:35 +00:00
var hostname string
if len ( f ) == 4 {
hostname = f [ 3 ]
}
2014-08-08 01:47:42 +00:00
isPrimary := ! b . hasPrefix ( "/bs/" )
2013-01-08 18:44:59 +00:00
s3Prefix := ""
if isPrimary {
s3Prefix = "/bs/"
} else {
s3Prefix = "/sto-s3/"
}
2014-08-08 01:47:42 +00:00
a := args {
2013-09-01 16:50:35 +00:00
"aws_access_key" : accessKey ,
"aws_secret_access_key" : secret ,
"bucket" : bucket ,
}
if hostname != "" {
2014-08-08 01:47:42 +00:00
a [ "hostname" ] = hostname
2012-12-22 00:13:36 +00:00
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( s3Prefix , "storage-s3" , a )
2013-01-08 18:44:59 +00:00
if isPrimary {
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/cache/" , "storage-filesystem" , args {
"path" : filepath . Join ( tempDir ( ) , "camli-cache" ) ,
} )
2013-01-08 18:44:59 +00:00
} else {
2014-08-08 01:47:42 +00:00
if b . high . BlobPath == "" {
2013-11-24 23:12:11 +00:00
panic ( "unexpected empty blobpath with sync-to-s3" )
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/sync-to-s3/" , "sync" , args {
"from" : "/bs/" ,
"to" : s3Prefix ,
"queue" : map [ string ] interface { } {
"type" : "kv" ,
"file" : filepath . Join ( b . high . BlobPath , "sync-to-s3-queue.kv" ) ,
2013-01-08 18:44:59 +00:00
} ,
2014-08-08 01:47:42 +00:00
} )
2012-12-22 00:13:36 +00:00
}
return nil
2012-03-15 12:31:06 +00:00
}
2014-08-08 01:47:42 +00:00
func ( b * lowBuilder ) addGoogleDriveConfig ( v string ) error {
f := strings . SplitN ( v , ":" , 4 )
2013-07-06 20:29:17 +00:00
if len ( f ) != 4 {
2013-08-11 15:07:18 +00:00
return errors . New ( ` genconfig: expected "googledrive" field to be of form "client_id:client_secret:refresh_token:parent_id" ` )
}
clientId , secret , refreshToken , parentId := f [ 0 ] , f [ 1 ] , f [ 2 ] , f [ 3 ]
2014-08-08 01:47:42 +00:00
isPrimary := ! b . hasPrefix ( "/bs/" )
2013-08-11 15:07:18 +00:00
prefix := ""
if isPrimary {
prefix = "/bs/"
} else {
prefix = "/sto-googledrive/"
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( prefix , "storage-googledrive" , args {
"parent_id" : parentId ,
"auth" : map [ string ] interface { } {
"client_id" : clientId ,
"client_secret" : secret ,
"refresh_token" : refreshToken ,
2013-08-11 15:07:18 +00:00
} ,
2014-08-08 01:47:42 +00:00
} )
2013-08-11 15:07:18 +00:00
if isPrimary {
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/cache/" , "storage-filesystem" , args {
"path" : filepath . Join ( tempDir ( ) , "camli-cache" ) ,
} )
2013-08-11 15:07:18 +00:00
} else {
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/sync-to-googledrive/" , "sync" , args {
"from" : "/bs/" ,
"to" : prefix ,
"queue" : map [ string ] interface { } {
"type" : "kv" ,
"file" : filepath . Join ( b . high . BlobPath ,
"sync-to-googledrive-queue.kv" ) ,
2013-08-11 15:07:18 +00:00
} ,
2014-08-08 01:47:42 +00:00
} )
2013-08-11 15:07:18 +00:00
}
return nil
}
2014-08-03 02:29:57 +00:00
var errGCSUsage = errors . New ( ` genconfig: expected "googlecloudstorage" field to be of form "client_id:client_secret:refresh_token:bucket" or ":bucketname" ` )
2014-08-08 01:47:42 +00:00
func ( b * lowBuilder ) addGoogleCloudStorageConfig ( v string ) error {
2014-08-03 02:29:57 +00:00
var clientID , secret , refreshToken , bucket string
2014-08-08 01:47:42 +00:00
f := strings . SplitN ( v , ":" , 4 )
2014-08-03 02:29:57 +00:00
switch len ( f ) {
default :
return errGCSUsage
case 4 :
clientID , secret , refreshToken , bucket = f [ 0 ] , f [ 1 ] , f [ 2 ] , f [ 3 ]
case 2 :
if f [ 0 ] != "" {
return errGCSUsage
}
bucket = f [ 1 ]
clientID = "auto"
2013-07-06 20:29:17 +00:00
}
2014-08-08 01:47:42 +00:00
isPrimary := ! b . hasPrefix ( "/bs/" )
2013-07-06 20:29:17 +00:00
gsPrefix := ""
if isPrimary {
gsPrefix = "/bs/"
} else {
2013-08-11 15:07:18 +00:00
gsPrefix = "/sto-googlecloudstorage/"
2013-07-06 20:29:17 +00:00
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( gsPrefix , "storage-googlecloudstorage" , args {
"bucket" : bucket ,
"auth" : map [ string ] interface { } {
"client_id" : clientID ,
"client_secret" : secret ,
"refresh_token" : refreshToken ,
2013-07-06 20:29:17 +00:00
} ,
2014-08-08 01:47:42 +00:00
} )
2013-07-06 20:29:17 +00:00
if isPrimary {
// TODO: cacheBucket like s3CacheBucket?
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/cache/" , "storage-filesystem" , args {
"path" : filepath . Join ( tempDir ( ) , "camli-cache" ) ,
} )
2013-07-06 20:29:17 +00:00
} else {
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/sync-to-googlecloudstorage/" , "sync" , args {
"from" : "/bs/" ,
"to" : gsPrefix ,
"queue" : map [ string ] interface { } {
"type" : "kv" ,
"file" : filepath . Join ( b . high . BlobPath ,
"sync-to-googlecloud-queue.kv" ) ,
2013-07-06 20:29:17 +00:00
} ,
2014-08-08 01:47:42 +00:00
} )
2013-07-06 20:29:17 +00:00
}
return nil
}
2014-08-08 01:47:42 +00:00
// indexFileDir returns the directory of the sqlite or kv file, or the
// empty string.
func ( b * lowBuilder ) indexFileDir ( ) string {
switch {
case b . high . SQLite != "" :
return filepath . Dir ( b . high . SQLite )
case b . high . KVFile != "" :
return filepath . Dir ( b . high . KVFile )
}
return ""
}
2012-03-15 12:31:06 +00:00
2014-08-23 16:46:23 +00:00
func ( b * lowBuilder ) syncToIndexArgs ( ) ( map [ string ] interface { } , error ) {
a := map [ string ] interface { } {
"from" : "/bs/" ,
"to" : "/index/" ,
}
const sortedType = "queue-sync-to-index"
if dbName := b . dbName ( sortedType ) ; dbName != "" {
qj , err := b . sortedStorage ( sortedType )
if err != nil {
return nil , err
}
a [ "queue" ] = qj
return a , nil
}
// TODO: currently when using s3, the index must be
// sqlite or kvfile, since only through one of those
// can we get a directory.
if b . high . BlobPath == "" && b . indexFileDir ( ) == "" {
// We don't actually have a working sync handler, but we keep a stub registered
// so it can be referred to from other places.
// See http://camlistore.org/issue/201
a [ "idle" ] = true
return a , nil
}
dir := b . high . BlobPath
if dir == "" {
dir = b . indexFileDir ( )
}
typ := "kv"
if b . high . SQLite != "" {
typ = "sqlite"
}
a [ "queue" ] = map [ string ] interface { } {
"type" : typ ,
"file" : filepath . Join ( dir , "sync-to-index-queue." + typ ) ,
}
return a , nil
}
2014-08-08 01:47:42 +00:00
func ( b * lowBuilder ) genLowLevelPrefixes ( ) error {
2013-01-10 23:29:08 +00:00
root := "/bs/"
pubKeyDest := root
2014-08-08 01:47:42 +00:00
if b . runIndex ( ) {
2013-01-10 23:29:08 +00:00
root = "/bs-and-maybe-also-index/"
pubKeyDest = "/bs-and-index/"
}
2013-06-26 19:55:47 +00:00
rootArgs := map [ string ] interface { } {
"stealth" : false ,
"blobRoot" : root ,
"statusRoot" : "/status/" ,
}
2014-08-08 01:47:42 +00:00
if b . high . OwnerName != "" {
rootArgs [ "ownerName" ] = b . high . OwnerName
2012-11-08 14:27:17 +00:00
}
2014-08-08 01:47:42 +00:00
if b . runIndex ( ) {
rootArgs [ "searchRoot" ] = "/my-search/"
2012-12-21 23:47:08 +00:00
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/" , "root" , rootArgs )
b . addPrefix ( "/setup/" , "setup" , nil )
b . addPrefix ( "/status/" , "status" , nil )
2012-04-03 21:39:49 +00:00
2014-08-08 01:47:42 +00:00
importerArgs := args { }
if b . high . Flickr != "" {
importerArgs [ "flickr" ] = map [ string ] interface { } {
"clientSecret" : b . high . Flickr ,
2014-07-04 22:25:28 +00:00
}
2014-04-17 17:23:53 +00:00
}
2014-08-08 01:47:42 +00:00
if b . high . Picasa != "" {
importerArgs [ "picasa" ] = map [ string ] interface { } {
"clientSecret" : b . high . Picasa ,
2013-02-28 23:30:16 +00:00
}
}
2014-08-08 01:47:42 +00:00
if b . runIndex ( ) {
b . addPrefix ( "/importer/" , "importer" , importerArgs )
2012-03-15 12:31:06 +00:00
}
2014-08-08 01:47:42 +00:00
if path := b . high . ShareHandlerPath ; path != "" {
b . addPrefix ( path , "share" , args {
"blobRoot" : "/bs/" ,
} )
2013-11-25 10:23:50 +00:00
}
2012-03-15 12:31:06 +00:00
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/sighelper/" , "jsonsign" , args {
"secretRing" : b . high . IdentitySecretRing ,
"keyId" : b . high . Identity ,
"publicKeyDest" : pubKeyDest ,
} )
2012-03-15 12:31:06 +00:00
2014-08-08 01:47:42 +00:00
storageType := "filesystem"
if b . high . PackBlobs {
storageType = "diskpacked"
2013-11-17 21:52:45 +00:00
}
2014-08-08 01:47:42 +00:00
if b . high . BlobPath != "" {
b . addPrefix ( "/bs/" , "storage-" + storageType , args {
"path" : b . high . BlobPath ,
} )
b . addPrefix ( "/cache/" , "storage-" + storageType , args {
"path" : filepath . Join ( b . high . BlobPath , "/cache" ) ,
} )
2014-03-04 21:25:10 +00:00
}
2013-11-17 21:52:45 +00:00
2014-08-08 01:47:42 +00:00
if b . runIndex ( ) {
2014-08-23 16:46:23 +00:00
syncArgs , err := b . syncToIndexArgs ( )
if err != nil {
return err
2013-08-20 18:11:37 +00:00
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/sync/" , "sync" , syncArgs )
2013-01-10 23:29:08 +00:00
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/bs-and-index/" , "storage-replica" , args {
"backends" : [ ] interface { } { "/bs/" , "/index/" } ,
} )
2013-01-10 23:29:08 +00:00
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/bs-and-maybe-also-index/" , "storage-cond" , args {
"write" : map [ string ] interface { } {
"if" : "isSchema" ,
"then" : "/bs-and-index/" ,
"else" : "/bs/" ,
2013-01-10 23:29:08 +00:00
} ,
2014-08-08 01:47:42 +00:00
"read" : "/bs/" ,
} )
2013-01-10 23:29:08 +00:00
2014-08-08 01:47:42 +00:00
owner , err := b . searchOwner ( )
if err != nil {
return err
}
searchArgs := args {
2014-04-01 16:03:20 +00:00
"index" : "/index/" ,
2014-08-08 01:47:42 +00:00
"owner" : owner . String ( ) ,
2013-11-28 19:09:16 +00:00
}
2014-08-08 01:47:42 +00:00
if b . memoryIndex ( ) {
2013-11-28 19:09:16 +00:00
searchArgs [ "slurpToMemory" ] = true
}
2014-08-08 01:47:42 +00:00
b . addPrefix ( "/my-search/" , "search" , searchArgs )
2012-03-15 12:31:06 +00:00
}
2014-08-08 01:47:42 +00:00
return nil
2012-03-15 12:31:06 +00:00
}
2014-08-08 01:47:42 +00:00
func ( b * lowBuilder ) build ( ) ( * Config , error ) {
conf , low := b . high , b . low
2014-01-23 22:40:12 +00:00
if conf . HTTPS {
if ( conf . HTTPSCert != "" ) != ( conf . HTTPSKey != "" ) {
return nil , errors . New ( "Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)" )
2012-08-04 01:12:39 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . HTTPSCert != "" {
2014-08-08 01:47:42 +00:00
low [ "httpsCert" ] = conf . HTTPSCert
low [ "httpsKey" ] = conf . HTTPSKey
2012-08-04 01:12:39 +00:00
} else {
2014-08-08 01:47:42 +00:00
low [ "httpsCert" ] = osutil . DefaultTLSCert ( )
low [ "httpsKey" ] = osutil . DefaultTLSKey ( )
2012-08-04 01:12:39 +00:00
}
2012-03-15 12:31:06 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . BaseURL != "" {
u , err := url . Parse ( conf . BaseURL )
2013-08-09 00:50:50 +00:00
if err != nil {
2014-01-23 22:40:12 +00:00
return nil , fmt . Errorf ( "Error parsing baseURL %q as a URL: %v" , conf . BaseURL , err )
2013-08-09 00:50:50 +00:00
}
2013-09-08 05:32:39 +00:00
if u . Path != "" && u . Path != "/" {
return nil , fmt . Errorf ( "baseURL can't have a path, only a scheme, host, and optional port." )
2012-09-16 22:20:49 +00:00
}
2013-09-08 05:32:39 +00:00
u . Path = ""
2014-08-08 01:47:42 +00:00
low [ "baseURL" ] = u . String ( )
2012-09-16 22:20:49 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . Listen != "" {
2014-08-08 01:47:42 +00:00
low [ "listen" ] = conf . Listen
2012-08-04 11:42:10 +00:00
}
2014-08-08 01:47:42 +00:00
low [ "https" ] = conf . HTTPS
low [ "auth" ] = conf . Auth
2012-08-04 11:42:10 +00:00
2014-01-23 22:40:12 +00:00
numIndexers := numSet ( conf . Mongo , conf . MySQL , conf . PostgreSQL , conf . SQLite , conf . KVFile )
2014-04-01 16:03:20 +00:00
2012-04-12 02:22:02 +00:00
switch {
2014-08-08 01:47:42 +00:00
case b . runIndex ( ) && numIndexers == 0 :
2013-12-11 08:20:22 +00:00
return nil , fmt . Errorf ( "Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite)." )
2014-08-08 01:47:42 +00:00
case b . runIndex ( ) && numIndexers != 1 :
2013-12-11 08:20:22 +00:00
return nil , fmt . Errorf ( "With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite)." )
2014-08-08 01:47:42 +00:00
case ! b . runIndex ( ) && numIndexers != 0 :
2013-12-11 08:20:22 +00:00
return nil , fmt . Errorf ( "With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite." )
2012-04-12 02:22:02 +00:00
}
2012-03-15 12:31:06 +00:00
2014-08-02 23:53:58 +00:00
if conf . Identity == "" {
return nil , errors . New ( "no 'identity' in server config" )
}
2012-04-12 18:39:53 +00:00
2014-01-23 22:40:12 +00:00
nolocaldisk := conf . BlobPath == ""
2013-08-20 18:11:37 +00:00
if nolocaldisk {
2014-01-23 22:40:12 +00:00
if conf . S3 == "" && conf . GoogleCloudStorage == "" {
2013-08-20 18:11:37 +00:00
return nil , errors . New ( "You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver." )
}
2014-01-23 22:40:12 +00:00
if conf . S3 != "" && conf . GoogleCloudStorage != "" {
2013-08-20 18:11:37 +00:00
return nil , errors . New ( "Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now." )
}
2013-01-08 18:44:59 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . ShareHandler && conf . ShareHandlerPath == "" {
conf . ShareHandlerPath = "/share/"
2013-07-16 15:55:20 +00:00
}
2014-08-08 01:47:42 +00:00
if err := b . genLowLevelPrefixes ( ) ; err != nil {
return nil , err
2012-03-15 12:31:06 +00:00
}
2013-01-08 18:44:59 +00:00
var cacheDir string
if nolocaldisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
2013-01-11 18:52:22 +00:00
cacheDir = filepath . Join ( tempDir ( ) , "camli-cache" )
2013-01-08 18:44:59 +00:00
} else {
2014-01-23 22:40:12 +00:00
cacheDir = filepath . Join ( conf . BlobPath , "cache" )
2013-01-08 18:44:59 +00:00
}
2013-08-25 17:25:30 +00:00
if ! noMkdir {
if err := os . MkdirAll ( cacheDir , 0700 ) ; err != nil {
return nil , fmt . Errorf ( "Could not create blobs cache dir %s: %v" , cacheDir , err )
}
2012-03-15 12:31:06 +00:00
}
2014-01-23 22:40:12 +00:00
if len ( conf . Publish ) > 0 {
2014-08-08 01:47:42 +00:00
if ! b . runIndex ( ) {
2013-01-10 23:29:08 +00:00
return nil , fmt . Errorf ( "publishing requires an index" )
}
2014-06-16 16:05:39 +00:00
var tlsO * tlsOpts
2014-08-08 01:47:42 +00:00
httpsCert , ok1 := low [ "httpsCert" ] . ( string )
httpsKey , ok2 := low [ "httpsKey" ] . ( string )
2014-06-16 16:05:39 +00:00
if ok1 && ok2 {
tlsO = & tlsOpts {
httpsCert : httpsCert ,
httpsKey : httpsKey ,
}
}
2014-08-08 01:47:42 +00:00
if err := b . addPublishedConfig ( tlsO ) ; err != nil {
2012-04-22 15:33:22 +00:00
return nil , fmt . Errorf ( "Could not generate config for published: %v" , err )
}
}
2014-08-08 01:47:42 +00:00
if b . runIndex ( ) {
b . addUIConfig ( )
2014-08-23 16:46:23 +00:00
sto , err := b . sortedStorage ( "index" )
2014-08-08 01:47:42 +00:00
if err != nil {
return nil , err
}
b . addPrefix ( "/index/" , "storage-index" , args {
"blobSource" : "/bs/" ,
"storage" : sto ,
} )
2013-01-10 23:29:08 +00:00
}
2012-03-15 12:31:06 +00:00
2014-01-23 22:40:12 +00:00
if conf . S3 != "" {
2014-08-08 01:47:42 +00:00
if err := b . addS3Config ( conf . S3 ) ; err != nil {
2012-12-22 00:13:36 +00:00
return nil , err
}
2012-04-12 02:22:02 +00:00
}
2014-01-23 22:40:12 +00:00
if conf . GoogleDrive != "" {
2014-08-08 01:47:42 +00:00
if err := b . addGoogleDriveConfig ( conf . GoogleDrive ) ; err != nil {
2013-08-11 15:07:18 +00:00
return nil , err
}
}
2014-01-23 22:40:12 +00:00
if conf . GoogleCloudStorage != "" {
2014-08-08 01:47:42 +00:00
if err := b . addGoogleCloudStorageConfig ( conf . GoogleCloudStorage ) ; err != nil {
2013-07-06 20:29:17 +00:00
return nil , err
}
}
2012-03-15 12:31:06 +00:00
2014-08-08 01:47:42 +00:00
return & Config { Obj : b . low } , nil
2012-03-15 12:31:06 +00:00
}
2013-01-10 23:29:08 +00:00
func numSet ( vv ... interface { } ) ( num int ) {
for _ , vi := range vv {
switch v := vi . ( type ) {
case string :
if v != "" {
num ++
}
case bool :
if v {
num ++
}
default :
panic ( "unknown type" )
}
}
return
}
2014-04-01 16:03:20 +00:00
// WriteDefaultConfigFile generates a new default high-level server configuration
// file at filePath. If useSQLite, the default indexer will use SQLite, otherwise
// kv. If filePath already exists, it is overwritten.
func WriteDefaultConfigFile ( filePath string , useSQLite bool ) error {
conf := serverconfig . Config {
Listen : ":3179" ,
HTTPS : false ,
Auth : "localhost" ,
ReplicateTo : make ( [ ] interface { } , 0 ) ,
}
blobDir := osutil . CamliBlobRoot ( )
2014-08-05 04:27:07 +00:00
if err := wkfs . MkdirAll ( blobDir , 0700 ) ; err != nil {
2014-04-01 16:03:20 +00:00
return fmt . Errorf ( "Could not create default blobs directory: %v" , err )
}
conf . BlobPath = blobDir
if useSQLite {
conf . SQLite = filepath . Join ( osutil . CamliVarDir ( ) , "camli-index.db" )
} else {
conf . KVFile = filepath . Join ( osutil . CamliVarDir ( ) , "camli-index.kvdb" )
}
2014-08-16 03:49:26 +00:00
keyID , secretRing , err := getOrMakeKeyring ( )
2014-04-01 16:03:20 +00:00
if err != nil {
2014-08-16 03:49:26 +00:00
return err
2014-04-01 16:03:20 +00:00
}
2014-08-16 03:49:26 +00:00
conf . Identity = keyID
conf . IdentitySecretRing = secretRing
2014-04-01 16:03:20 +00:00
confData , err := json . MarshalIndent ( conf , "" , " " )
if err != nil {
return fmt . Errorf ( "Could not json encode config file : %v" , err )
}
2014-08-05 04:27:07 +00:00
if err := wkfs . WriteFile ( filePath , confData , 0600 ) ; err != nil {
2014-04-01 16:03:20 +00:00
return fmt . Errorf ( "Could not create or write default server config: %v" , err )
}
return nil
}
2014-08-16 03:49:26 +00:00
func getOrMakeKeyring ( ) ( keyID , secRing string , err error ) {
secRing = osutil . SecretRingFile ( )
_ , err = wkfs . Stat ( secRing )
switch {
case err == nil :
keyID , err = jsonsign . KeyIdFromRing ( secRing )
if err != nil {
err = fmt . Errorf ( "Could not find any keyID in file %q: %v" , secRing , err )
return
}
log . Printf ( "Re-using identity with keyID %q found in file %s" , keyID , secRing )
case os . IsNotExist ( err ) :
keyID , err = jsonsign . GenerateNewSecRing ( secRing )
if err != nil {
err = fmt . Errorf ( "Could not generate new secRing at file %q: %v" , secRing , err )
return
}
log . Printf ( "Generated new identity with keyID %q in file %s" , keyID , secRing )
default :
err = fmt . Errorf ( "Could not stat secret ring %q: %v" , secRing , err )
}
return
}