diff --git a/cmd/camdeploy/camdeploy.go b/cmd/camdeploy/camdeploy.go new file mode 100644 index 000000000..b1636a448 --- /dev/null +++ b/cmd/camdeploy/camdeploy.go @@ -0,0 +1,27 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The camdeploy program deploys Camlistore on cloud computing platforms such as Google +// Compute Engine or Amazon EC2. +package main + +import ( + "camlistore.org/pkg/cmdmain" +) + +func main() { + cmdmain.Main() +} diff --git a/cmd/camdeploy/gce.go b/cmd/camdeploy/gce.go new file mode 100644 index 000000000..9bc654b58 --- /dev/null +++ b/cmd/camdeploy/gce.go @@ -0,0 +1,168 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strings" + + "camlistore.org/pkg/cmdmain" + "camlistore.org/pkg/context" + "camlistore.org/pkg/deploy/gce" + + "camlistore.org/third_party/code.google.com/p/goauth2/oauth" +) + +type gceCmd struct { + project string + zone string + machine string + instName string + hostname string + certFile string + keyFile string + sshPub string + verbose bool +} + +func init() { + cmdmain.RegisterCommand("gce", func(flags *flag.FlagSet) cmdmain.CommandRunner { + cmd := new(gceCmd) + flags.StringVar(&cmd.project, "project", "", "Name of Project.") + flags.StringVar(&cmd.zone, "zone", gce.Zone, "GCE zone.") + flags.StringVar(&cmd.machine, "machine", gce.Machine, "e.g. n1-standard-1, f1-micro, g1-small") + flags.StringVar(&cmd.instName, "instance_name", gce.InstanceName, "Name of VM instance.") + flags.StringVar(&cmd.hostname, "hostname", "", "Hostname for the instance and self-signed certificates. Must be given if generating self-signed certs.") + flags.StringVar(&cmd.certFile, "cert", "", "Certificate file for TLS. A self-signed one will be generated if this flag is omitted.") + flags.StringVar(&cmd.keyFile, "key", "", "Key file for the TLS certificate. Must be given with --cert") + flags.StringVar(&cmd.sshPub, "ssh_public_key", "", "SSH public key file to authorize. Can modify later in Google's web UI anyway.") + flags.BoolVar(&cmd.verbose, "verbose", false, "Be verbose.") + return cmd + }) +} + +const ( + clientIdDat = "client-id.dat" + clientSecretDat = "client-secret.dat" + helpEnableAuth = `Enable authentication: in your project console, navigate to "APIs and auth", "Credentials", click on "Create new Client ID", and pick "Installed application", with type "Other". Copy the CLIENT ID to ` + clientIdDat + `, and the CLIENT SECRET to ` + clientSecretDat +) + +func (c *gceCmd) Describe() string { + return "Deploy Camlistore on Google Compute Engine." +} + +func (c *gceCmd) Usage() { + fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n %s\n\n", + "camdeploy gce --project= --hostname= [options]", + "camdeploy gce --project= --cert= --key= [options]") + flag.PrintDefaults() + fmt.Fprintln(os.Stderr, "\nTo get started:\n") + printHelp() +} + +func printHelp() { + for _, v := range []string{gce.HelpCreateProject, helpEnableAuth, gce.HelpEnableAPIs} { + fmt.Fprintf(os.Stderr, "%v\n", v) + } +} + +func (c *gceCmd) RunCommand(args []string) error { + if c.verbose { + gce.Verbose = true + } + if c.project == "" { + return cmdmain.UsageError("Missing --project flag.") + } + if (c.certFile == "") != (c.keyFile == "") { + return cmdmain.UsageError("--cert and --key must both be given together.") + } + if c.certFile == "" && c.hostname == "" { + return cmdmain.UsageError("Either --hostname, or --cert & --key must provided.") + } + config := gce.NewOAuthConfig(readFile(clientIdDat), readFile(clientSecretDat)) + config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob" + + instConf := &gce.InstanceConf{ + Name: c.instName, + Project: c.project, + Machine: c.machine, + Zone: c.zone, + CertFile: c.certFile, + KeyFile: c.keyFile, + Hostname: c.hostname, + } + if c.sshPub != "" { + instConf.SSHPub = strings.TrimSpace(readFile(c.sshPub)) + } + + depl := &gce.Deployer{ + Cl: &http.Client{Transport: c.transport(config)}, + Conf: instConf, + } + inst, err := depl.Create(context.TODO()) + if err != nil { + return err + } + + log.Printf("Instance is up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP) + return nil +} + +func readFile(v string) string { + slurp, err := ioutil.ReadFile(v) + if err != nil { + if os.IsNotExist(err) { + msg := fmt.Sprintf("%v does not exist.", v) + if v == clientIdDat || v == clientSecretDat { + msg = fmt.Sprintf("%v\n%s", msg, helpEnableAuth) + } + log.Fatal(msg) + } + log.Fatalf("Error reading %s: %v", v, err) + } + return strings.TrimSpace(string(slurp)) +} + +func (c *gceCmd) transport(config *oauth.Config) *oauth.Transport { + tr := &oauth.Transport{ + Config: config, + } + tokenCache := oauth.CacheFile(c.project + "-token.dat") + token, err := tokenCache.Token() + if err != nil { + log.Printf("Error getting token from %s: %v", string(tokenCache), err) + log.Printf("Get auth code from %v", config.AuthCodeURL("my-state")) + io.WriteString(os.Stdout, "\nEnter auth code:") + sc := bufio.NewScanner(os.Stdin) + sc.Scan() + authCode := strings.TrimSpace(sc.Text()) + token, err = tr.Exchange(authCode) + if err != nil { + log.Fatalf("Error exchanging auth code for a token: %v", err) + } + tokenCache.PutToken(token) + } + tr.Token = token + return tr +} diff --git a/misc/gce/.gitignore b/misc/gce/.gitignore deleted file mode 100644 index 2d5ad2248..000000000 --- a/misc/gce/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -client-id.dat -client-secret.dat -token.dat -*-token.dat -*.crt -*.key diff --git a/misc/gce/create.go b/misc/gce/create.go deleted file mode 100644 index 310d9e025..000000000 --- a/misc/gce/create.go +++ /dev/null @@ -1,458 +0,0 @@ -package main - -import ( - "bufio" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "camlistore.org/pkg/httputil" - "camlistore.org/pkg/osutil" - - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" - compute "camlistore.org/third_party/code.google.com/p/google-api-go-client/compute/v1" - storage "camlistore.org/third_party/code.google.com/p/google-api-go-client/storage/v1" -) - -var ( - proj = flag.String("project", "", "Name of Project.") - zone = flag.String("zone", "us-central1-a", "GCE zone.") - mach = flag.String("machinetype", "g1-small", "e.g. n1-standard-1, f1-micro, g1-small") - instName = flag.String("instance_name", "camlistore-server", "Name of VM instance.") - hostname = flag.String("hostname", "", "Hostname for the instance and self-signed certificates. Must be given if generating self-signed certs.") - certFile = flag.String("cert", "", "Certificate file for TLS. A self-signed one will be generated if this flag is omitted.") - keyFile = flag.String("key", "", "Key file for the TLS certificate. Must be given with --cert") - sshPub = flag.String("ssh_public_key", "", "SSH public key file to authorize. Can modify later in Google's web UI anyway.") - verbose = flag.Bool("verbose", false, "Be verbose.") -) - -const ( - clientIdDat = "client-id.dat" - clientSecretDat = "client-secret.dat" - helpCreateProject = "Create new project: go to https://console.developers.google.com to create a new Project." - helpEnableAuth = `Enable authentication: in your project console, navigate to "APIs and auth", "Credentials", click on "Create new Client ID"ΒΈ and pick "Installed application", with type "Other". Copy the CLIENT ID to ` + clientIdDat + `, and the CLIENT SECRET to ` + clientSecretDat - helpEnableAPIs = `Enable the project APIs: in your project console, navigate to "APIs and auth", "APIs". In the list, enable "Google Cloud Storage", "Google Cloud Storage JSON API", and "Google Compute Engine".` -) - -func readFile(v string) string { - slurp, err := ioutil.ReadFile(v) - if err != nil { - if os.IsNotExist(err) { - log.Fatalf("%v does not exist.\n%s", v, helpEnableAuth) - } - log.Fatalf("Error reading %s: %v", v, err) - } - return strings.TrimSpace(string(slurp)) -} - -func printHelp() { - for _, v := range []string{helpCreateProject, helpEnableAuth, helpEnableAPIs} { - fmt.Fprintf(os.Stderr, "%v\n", v) - } -} - -func usage() { - fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n %s\n\n", - "go run create.go --project= --hostname= [options]", - "go run create.go --project= --cert= --key= [options]") - flag.PrintDefaults() - fmt.Fprintln(os.Stderr, "\nTo get started with this script:\n") - printHelp() -} - -func main() { - flag.Usage = usage - flag.Parse() - if *proj == "" { - log.Print("Missing --project flag.") - usage() - return - } - if (*certFile == "") != (*keyFile == "") { - log.Print("--cert and --key must both be given together.") - return - } - if *certFile == "" && *hostname == "" { - log.Print("Either --hostname, or --cert & --key must provided.") - return - } - prefix := "https://www.googleapis.com/compute/v1/projects/" + *proj - imageURL := "https://www.googleapis.com/compute/v1/projects/coreos-cloud/global/images/coreos-alpha-402-2-0-v20140807" - machType := prefix + "/zones/" + *zone + "/machineTypes/" + *mach - - config := &oauth.Config{ - // The client-id and secret should be for an "Installed Application" when using - // the CLI. Later we'll use a web application with a callback. - ClientId: readFile(clientIdDat), - ClientSecret: readFile(clientSecretDat), - Scope: strings.Join([]string{ - compute.DevstorageFull_controlScope, - compute.ComputeScope, - "https://www.googleapis.com/auth/sqlservice", - "https://www.googleapis.com/auth/sqlservice.admin", - }, " "), - AuthURL: "https://accounts.google.com/o/oauth2/auth", - TokenURL: "https://accounts.google.com/o/oauth2/token", - RedirectURL: "urn:ietf:wg:oauth:2.0:oob", - } - - tr := &oauth.Transport{ - Config: config, - } - - tokenCache := oauth.CacheFile(*proj + "-token.dat") - token, err := tokenCache.Token() - if err != nil { - log.Printf("Error getting token from %s: %v", string(tokenCache), err) - log.Printf("Get auth code from %v", config.AuthCodeURL("my-state")) - os.Stdout.Write([]byte("\nEnter auth code: ")) - sc := bufio.NewScanner(os.Stdin) - sc.Scan() - authCode := strings.TrimSpace(sc.Text()) - token, err = tr.Exchange(authCode) - if err != nil { - log.Fatalf("Error exchanging auth code for a token: %v", err) - } - tokenCache.PutToken(token) - } - - tr.Token = token - oauthClient := &http.Client{Transport: tr} - computeService, _ := compute.New(oauthClient) - storageService, _ := storage.New(oauthClient) - - cloudConfig := `#cloud-config -write_files: - - path: /var/lib/camlistore/tmp/README - permissions: 0644 - content: | - This is the Camlistore /tmp directory. - - path: /var/lib/camlistore/mysql/README - permissions: 0644 - content: | - This is the Camlistore MySQL data directory. -coreos: - units: - - name: cam-journal-gatewayd.service - content: | - [Unit] - Description=Journal Gateway Service - Requires=cam-journal-gatewayd.socket - - [Service] - ExecStart=/usr/lib/systemd/systemd-journal-gatewayd - User=systemd-journal-gateway - Group=systemd-journal-gateway - SupplementaryGroups=systemd-journal - PrivateTmp=yes - PrivateDevices=yes - PrivateNetwork=yes - ProtectSystem=full - ProtectHome=yes - - [Install] - Also=cam-journal-gatewayd.socket - - name: cam-journal-gatewayd.socket - command: start - content: | - [Unit] - Description=Journal Gateway Service Socket - - [Socket] - ListenStream=/run/camjournald.sock - - [Install] - WantedBy=sockets.target - - name: mysql.service - command: start - content: | - [Unit] - Description=MySQL - After=docker.service - Requires=docker.service - - [Service] - ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker - ExecStart=/opt/bin/systemd-docker run --rm --name %n -v /var/lib/camlistore/mysql:/mysql -e INNODB_BUFFER_POOL_SIZE=NNN camlistore/mysql - RestartSec=1s - Restart=always - Type=notify - NotifyAccess=all - - [Install] - WantedBy=multi-user.target - - name: camlistored.service - command: start - content: | - [Unit] - Description=Camlistore - After=docker.service - Requires=docker.service mysql.service - - [Service] - ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker - ExecStart=/opt/bin/systemd-docker run --rm -p 80:80 -p 443:443 --name %n -v /run/camjournald.sock:/run/camjournald.sock -v /var/lib/camlistore/tmp:/tmp --link=mysql.service:mysqldb camlistore/camlistored - RestartSec=1s - Restart=always - Type=notify - NotifyAccess=all - - [Install] - WantedBy=multi-user.target -` - cloudConfig = strings.Replace(cloudConfig, "INNODB_BUFFER_POOL_SIZE=NNN", "INNODB_BUFFER_POOL_SIZE="+strconv.Itoa(innodbBufferPoolSize(*mach)), -1) - - const maxCloudConfig = 32 << 10 // per compute API docs - if len(cloudConfig) > maxCloudConfig { - log.Fatalf("cloud config length of %d bytes is over %d byte limit", len(cloudConfig), maxCloudConfig) - } - if *sshPub != "" { - key := strings.TrimSpace(readFile(*sshPub)) - cloudConfig += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", key) - } - - blobBucket := *proj + "-camlistore-blobs" - configBucket := *proj + "-camlistore-config" - needBucket := map[string]bool{ - blobBucket: true, - configBucket: true, - } - - buckets, err := storageService.Buckets.List(*proj).Do() - if err != nil { - log.Fatalf("Error listing buckets: %v", err) - } - for _, it := range buckets.Items { - delete(needBucket, it.Name) - } - if len(needBucket) > 0 { - log.Printf("Need to create buckets: %v", needBucket) - var waitBucket sync.WaitGroup - for name := range needBucket { - name := name - waitBucket.Add(1) - go func() { - defer waitBucket.Done() - log.Printf("Creating bucket %s", name) - b, err := storageService.Buckets.Insert(*proj, &storage.Bucket{ - Id: name, - Name: name, - }).Do() - if err != nil { - log.Fatalf("Error creating bucket %s: %v", name, err) - } - log.Printf("Created bucket %s: %+v", name, b) - }() - } - waitBucket.Wait() - } - - if *certFile == "" { - // A bit paranoid since these are illigal GCE project name characters anyway but it doesn't hurt. - r := strings.NewReplacer(".", "_", "/", "_", "\\", "_") - *certFile = r.Replace(*proj) + ".crt" - *keyFile = r.Replace(*proj) + ".key" - - _, errc := os.Stat(*certFile) - _, errk := os.Stat(*keyFile) - switch { - case os.IsNotExist(errc) && os.IsNotExist(errk): - log.Printf("Generating self-signed certificate for %v ...", *hostname) - err, sig := httputil.GenSelfTLS(*hostname, *certFile, *keyFile) - if err != nil { - log.Fatalf("Error generating certificates: %v", err) - } - log.Printf("Wrote key to %s, and certificate to %s with fingerprint %s", *keyFile, *certFile, sig) - case errc != nil: - log.Fatalf("Couldn't stat cert: %v", errc) - case errk != nil: - log.Fatalf("Couldn't stat key: %v", errk) - default: - log.Printf("Using certificate %s and key %s", *certFile, *keyFile) - } - } - - log.Print("Uploading certificate and key...") - err = uploadFile(storageService, *certFile, configBucket, filepath.Base(osutil.DefaultTLSCert())) - if err != nil { - log.Fatalf("Cert upload failed: %v", err) - } - err = uploadFile(storageService, *keyFile, configBucket, filepath.Base(osutil.DefaultTLSKey())) - if err != nil { - log.Fatalf("Key upload failed: %v", err) - } - - instance := &compute.Instance{ - Name: *instName, - Description: "Camlistore server", - MachineType: machType, - Disks: []*compute.AttachedDisk{ - { - AutoDelete: true, - Boot: true, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: *instName + "-coreos-stateless-pd", - SourceImage: imageURL, - }, - }, - }, - Tags: &compute.Tags{ - Items: []string{"http-server", "https-server"}, - }, - Metadata: &compute.Metadata{ - Items: []*compute.MetadataItems{ - { - Key: "camlistore-username", - Value: "test", - }, - { - Key: "camlistore-password", - Value: "insecure", // TODO: this won't be cleartext later - }, - { - Key: "camlistore-blob-bucket", - Value: "gs://" + blobBucket, - }, - { - Key: "camlistore-config-bucket", - Value: "gs://" + configBucket, - }, - { - Key: "user-data", - Value: cloudConfig, - }, - }, - }, - NetworkInterfaces: []*compute.NetworkInterface{ - &compute.NetworkInterface{ - AccessConfigs: []*compute.AccessConfig{ - &compute.AccessConfig{ - Type: "ONE_TO_ONE_NAT", - Name: "External NAT", - }, - }, - Network: prefix + "/global/networks/default", - }, - }, - ServiceAccounts: []*compute.ServiceAccount{ - { - Email: "default", - Scopes: []string{ - compute.DevstorageFull_controlScope, - compute.ComputeScope, - "https://www.googleapis.com/auth/sqlservice", - "https://www.googleapis.com/auth/sqlservice.admin", - }, - }, - }, - } - if *hostname != "" { - instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{ - Key: "camlistore-hostname", - Value: *hostname, - }) - } - const localMySQL = false // later - if localMySQL { - instance.Disks = append(instance.Disks, &compute.AttachedDisk{ - AutoDelete: false, - Boot: false, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: "camlistore-mysql-index-pd", - DiskSizeGb: 4, - }, - }) - } - - log.Print("Creating instance...") - op, err := computeService.Instances.Insert(*proj, *zone, instance).Do() - if err != nil { - log.Fatalf("Failed to create instance: %v", err) - } - opName := op.Name - if *verbose { - log.Printf("Created. Waiting on operation %v", opName) - } -OpLoop: - for { - time.Sleep(2 * time.Second) - op, err := computeService.ZoneOperations.Get(*proj, *zone, opName).Do() - if err != nil { - log.Fatalf("Failed to get op %s: %v", opName, err) - } - switch op.Status { - case "PENDING", "RUNNING": - if *verbose { - log.Printf("Waiting on operation %v", opName) - } - continue - case "DONE": - if op.Error != nil { - for _, operr := range op.Error.Errors { - log.Printf("Error: %+v", operr) - } - log.Fatalf("Failed to start.") - } - if *verbose { - log.Printf("Success. %+v", op) - } - break OpLoop - default: - log.Fatalf("Unknown status %q: %+v", op.Status, op) - } - } - - inst, err := computeService.Instances.Get(*proj, *zone, *instName).Do() - if err != nil { - log.Fatalf("Error getting instance after creation: %v", err) - } - - if *verbose { - ij, _ := json.MarshalIndent(inst, "", " ") - log.Printf("Instance: %s", ij) - } - - addr := inst.NetworkInterfaces[0].AccessConfigs[0].NatIP - log.Printf("Instance is up at %s", addr) -} - -func uploadFile(service *storage.Service, localFilename, bucketName, objectName string) error { - file, err := os.Open(localFilename) - defer file.Close() - if err != nil { - return fmt.Errorf("Error opening %v: %v", localFilename, err) - } - _, err = service.Objects.Insert(bucketName, &storage.Object{Name: objectName}).Media(file).Do() - if err != nil { - return fmt.Errorf("Objects.Insert for %v failed: %v", localFilename, err) - } - return nil -} - -// returns the MySQL InnoDB buffer pool size (in bytes) as a function -// of the GCE machine type. -func innodbBufferPoolSize(machType string) int { - // Totally arbitrary. We don't need much here because - // camlistored slurps this all into its RAM on start-up - // anyway. So this is all prety overkill and more than the - // 8MB default. - switch machType { - case "f1-micro": - return 32 << 20 - case "g1-small": - return 64 << 20 - default: - return 128 << 20 - } -} diff --git a/misc/gce/cloud-config.yaml b/pkg/deploy/gce/cloud-config.yaml similarity index 100% rename from misc/gce/cloud-config.yaml rename to pkg/deploy/gce/cloud-config.yaml diff --git a/pkg/deploy/gce/deploy.go b/pkg/deploy/gce/deploy.go new file mode 100644 index 000000000..16f847b36 --- /dev/null +++ b/pkg/deploy/gce/deploy.go @@ -0,0 +1,497 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package gce provides tools to deploy Camlistore on Google Compute Engine. +package gce + +// TODO: we want to host our own docker images under camlistore.org, so we should make a +// list. For the purposes of this package, we should add to the list camlistore/camlistored +// and camlistore/mysql. + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "camlistore.org/pkg/context" + "camlistore.org/pkg/httputil" + "camlistore.org/pkg/osutil" + + "camlistore.org/third_party/code.google.com/p/goauth2/oauth" + compute "camlistore.org/third_party/code.google.com/p/google-api-go-client/compute/v1" + storage "camlistore.org/third_party/code.google.com/p/google-api-go-client/storage/v1" +) + +const ( + projectsAPIURL = "https://www.googleapis.com/compute/v1/projects/" + coreosImgURL = "https://www.googleapis.com/compute/v1/projects/coreos-cloud/global/images/coreos-stable-444-5-0-v20141016" + + // default instance configuration values. + InstanceName = "camlistore-server" + Machine = "g1-small" + Zone = "us-central1-a" + + HelpCreateProject = "Create new project: go to https://console.developers.google.com to create a new Project." + HelpEnableAPIs = `Enable the project APIs: in your project console, navigate to "APIs and auth", "APIs". In the list, enable "Google Cloud Storage", "Google Cloud Storage JSON API", and "Google Compute Engine".` + HelpDeleteInstance = `Delete an existing Compute Engine instance: in your project console, navigate to "Compute", "Compute Engine", and "VM instances". Select your instance and click "Delete".` +) + +// Verbose enables more info to be printed. +var Verbose bool + +// NewOAuthConfig returns an OAuth configuration template. +func NewOAuthConfig(clientId, clientSecret string) *oauth.Config { + return &oauth.Config{ + Scope: strings.Join([]string{ + compute.DevstorageFull_controlScope, + compute.ComputeScope, + "https://www.googleapis.com/auth/sqlservice", + "https://www.googleapis.com/auth/sqlservice.admin", + }, " "), + AuthURL: "https://accounts.google.com/o/oauth2/auth", + TokenURL: "https://accounts.google.com/o/oauth2/token", + ClientId: clientId, + ClientSecret: clientSecret, + } +} + +// InstanceConf is the configuration for the Google Compute Engine instance that will be deployed. +type InstanceConf struct { + Name string // Name given to the virtual machine instance. + Project string // Google project ID where the instance is created. + Machine string // Machine type. + Zone string // Geographic zone. + SSHPub string // SSH public key. + CertFile string // HTTPS certificate file. + KeyFile string // HTTPS key file. + Hostname string // Fully qualified domain name. + + configBucket string // Project + "-camlistore-config" + blobBucket string // Project + "-camlistore-blobs" +} + +// Deployer creates and starts an instance such as defined in Conf. +type Deployer struct { + Cl *http.Client + Conf *InstanceConf +} + +// Get returns the Instance corresponding to the Project, Zone, and Name defined in the +// Deployer's Conf. +func (d *Deployer) Get(ctx *context.Context) (*compute.Instance, error) { + computeService, err := compute.New(d.Cl) + if err != nil { + return nil, err + } + return computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() +} + +// Create sets up and starts a Google Compute Engine instance as defined in d.Conf. It +// creates the necessary Google Storage buckets beforehand. +func (d *Deployer) Create(ctx *context.Context) (*compute.Instance, error) { + computeService, _ := compute.New(d.Cl) + storageService, _ := storage.New(d.Cl) + + config := cloudConfig(d.Conf) + const maxCloudConfig = 32 << 10 // per compute API docs + if len(config) > maxCloudConfig { + return nil, fmt.Errorf("cloud config length of %d bytes is over %d byte limit", len(config), maxCloudConfig) + } + + if err := d.setBuckets(storageService, ctx); err != nil { + return nil, fmt.Errorf("could not create buckets: %v", err) + } + + if err := d.setupHTTPS(storageService); err != nil { + return nil, fmt.Errorf("could not setup HTTPS: %v", err) + } + + if err := d.createInstance(storageService, computeService, ctx); err != nil { + return nil, fmt.Errorf("could not create compute instance: %v", err) + } + + inst, err := computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() + if err != nil { + return nil, fmt.Errorf("error getting instance after creation: %v", err) + } + if Verbose { + ij, _ := json.MarshalIndent(inst, "", " ") + log.Printf("Instance: %s", ij) + } + return inst, nil +} + +// createInstance starts the creation of the Compute Engine instance and waits for the +// result of the creation operation. It should be called after setBuckets and setupHTTPS. +func (d *Deployer) createInstance(storageService *storage.Service, computeService *compute.Service, ctx *context.Context) error { + prefix := projectsAPIURL + d.Conf.Project + machType := prefix + "/zones/" + d.Conf.Zone + "/machineTypes/" + d.Conf.Machine + config := cloudConfig(d.Conf) + instance := &compute.Instance{ + Name: d.Conf.Name, + Description: "Camlistore server", + MachineType: machType, + Disks: []*compute.AttachedDisk{ + { + AutoDelete: true, + Boot: true, + Type: "PERSISTENT", + InitializeParams: &compute.AttachedDiskInitializeParams{ + DiskName: d.Conf.Name + "-coreos-stateless-pd", + SourceImage: coreosImgURL, + }, + }, + }, + Tags: &compute.Tags{ + Items: []string{"http-server", "https-server"}, + }, + Metadata: &compute.Metadata{ + Items: []*compute.MetadataItems{ + { + Key: "camlistore-username", + Value: "test", + }, + { + Key: "camlistore-password", + Value: "insecure", // TODO: this won't be cleartext later + }, + { + Key: "camlistore-blob-bucket", + Value: "gs://" + d.Conf.blobBucket, + }, + { + Key: "camlistore-config-bucket", + Value: "gs://" + d.Conf.configBucket, + }, + { + Key: "user-data", + Value: config, + }, + }, + }, + NetworkInterfaces: []*compute.NetworkInterface{ + &compute.NetworkInterface{ + AccessConfigs: []*compute.AccessConfig{ + &compute.AccessConfig{ + Type: "ONE_TO_ONE_NAT", + Name: "External NAT", + }, + }, + Network: prefix + "/global/networks/default", + }, + }, + ServiceAccounts: []*compute.ServiceAccount{ + { + Email: "default", + Scopes: []string{ + compute.DevstorageFull_controlScope, + compute.ComputeScope, + "https://www.googleapis.com/auth/sqlservice", + "https://www.googleapis.com/auth/sqlservice.admin", + }, + }, + }, + } + if d.Conf.Hostname != "" { + instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{ + Key: "camlistore-hostname", + Value: d.Conf.Hostname, + }) + } + const localMySQL = false // later + if localMySQL { + instance.Disks = append(instance.Disks, &compute.AttachedDisk{ + AutoDelete: false, + Boot: false, + Type: "PERSISTENT", + InitializeParams: &compute.AttachedDiskInitializeParams{ + DiskName: "camlistore-mysql-index-pd", + DiskSizeGb: 4, + }, + }) + } + + if Verbose { + log.Print("Creating instance...") + } + op, err := computeService.Instances.Insert(d.Conf.Project, d.Conf.Zone, instance).Do() + if err != nil { + return fmt.Errorf("failed to create instance: %v", err) + } + opName := op.Name + if Verbose { + log.Printf("Created. Waiting on operation %v", opName) + } +OpLoop: + for { + if ctx.IsCanceled() { + // TODO(mpl): should we destroy any created instance (and attached + // resources) at this point? So that a subsequent run does not fail + // with something like: + // 2014/11/25 17:08:58 Error: &{Code:RESOURCE_ALREADY_EXISTS Location: Message:The resource 'projects/camli-mpl/zones/europe-west1-b/disks/camlistore-server-coreos-stateless-pd' already exists} + return context.ErrCanceled + } + time.Sleep(2 * time.Second) + op, err := computeService.ZoneOperations.Get(d.Conf.Project, d.Conf.Zone, opName).Do() + if err != nil { + return fmt.Errorf("failed to get op %s: %v", opName, err) + } + switch op.Status { + case "PENDING", "RUNNING": + if Verbose { + log.Printf("Waiting on operation %v", opName) + } + continue + case "DONE": + if op.Error != nil { + for _, operr := range op.Error.Errors { + log.Printf("Error: %+v", operr) + } + return fmt.Errorf("failed to start.") + } + if Verbose { + log.Printf("Success. %+v", op) + } + break OpLoop + default: + return fmt.Errorf("unknown status %q: %+v", op.Status, op) + } + } + return nil +} + +func cloudConfig(conf *InstanceConf) string { + config := strings.Replace(baseInstanceConfig, "INNODB_BUFFER_POOL_SIZE=NNN", "INNODB_BUFFER_POOL_SIZE="+strconv.Itoa(innodbBufferPoolSize(conf.Machine)), -1) + if conf.SSHPub != "" { + config += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", conf.SSHPub) + } + return config +} + +// setBuckets defines the buckets needed by the instance and creates them. +func (d *Deployer) setBuckets(storageService *storage.Service, ctx *context.Context) error { + blobBucket := d.Conf.Project + "-camlistore-blobs" + configBucket := d.Conf.Project + "-camlistore-config" + needBucket := map[string]bool{ + blobBucket: true, + configBucket: true, + } + + buckets, err := storageService.Buckets.List(d.Conf.Project).Do() + if err != nil { + return fmt.Errorf("error listing buckets: %v", err) + } + for _, it := range buckets.Items { + delete(needBucket, it.Name) + } + if len(needBucket) > 0 { + if Verbose { + log.Printf("Need to create buckets: %v", needBucket) + } + var waitBucket sync.WaitGroup + var bucketErr error + for name := range needBucket { + if ctx.IsCanceled() { + return context.ErrCanceled + } + name := name + waitBucket.Add(1) + go func() { + defer waitBucket.Done() + if Verbose { + log.Printf("Creating bucket %s", name) + } + b, err := storageService.Buckets.Insert(d.Conf.Project, &storage.Bucket{ + Id: name, + Name: name, + }).Do() + if err != nil && bucketErr == nil { + bucketErr = fmt.Errorf("error creating bucket %s: %v", name, err) + return + } + if Verbose { + log.Printf("Created bucket %s: %+v", name, b) + } + }() + } + waitBucket.Wait() + if bucketErr != nil { + return bucketErr + } + } + d.Conf.configBucket = configBucket + d.Conf.blobBucket = blobBucket + return nil +} + +// setupHTTPS uploads to the configuration bucket the certificate and key used by the +// instance for HTTPS. It generates them if d.Conf.CertFile or d.Conf.KeyFile is not defined. +// It should be called after setBuckets. +func (d *Deployer) setupHTTPS(storageService *storage.Service) error { + var cert, key io.ReadCloser + var err error + if d.Conf.CertFile != "" && d.Conf.KeyFile != "" { + cert, err = os.Open(d.Conf.CertFile) + if err != nil { + return err + } + defer cert.Close() + key, err = os.Open(d.Conf.KeyFile) + if err != nil { + return err + } + defer key.Close() + } else { + if Verbose { + log.Printf("Generating self-signed certificate for %v ...", d.Conf.Hostname) + } + certBytes, keyBytes, err := httputil.GenSelfTLS(d.Conf.Hostname) + if err != nil { + return fmt.Errorf("error generating certificates: %v", err) + } + if Verbose { + sig, err := httputil.CertFingerprint(certBytes) + if err != nil { + return fmt.Errorf("could not get sha256 fingerprint of certificate: %v", err) + } + log.Printf("Wrote certificate with fingerprint %s", sig) + } + cert = ioutil.NopCloser(bytes.NewReader(certBytes)) + key = ioutil.NopCloser(bytes.NewReader(keyBytes)) + } + + if Verbose { + log.Print("Uploading certificate and key...") + } + _, err = storageService.Objects.Insert(d.Conf.configBucket, + &storage.Object{Name: filepath.Base(osutil.DefaultTLSCert())}).Media(cert).Do() + if err != nil { + return fmt.Errorf("cert upload failed: %v", err) + } + _, err = storageService.Objects.Insert(d.Conf.configBucket, + &storage.Object{Name: filepath.Base(osutil.DefaultTLSKey())}).Media(key).Do() + if err != nil { + return fmt.Errorf("key upload failed: %v", err) + } + return nil +} + +// returns the MySQL InnoDB buffer pool size (in bytes) as a function +// of the GCE machine type. +func innodbBufferPoolSize(machine string) int { + // Totally arbitrary. We don't need much here because + // camlistored slurps this all into its RAM on start-up + // anyway. So this is all prety overkill and more than the + // 8MB default. + switch machine { + case "f1-micro": + return 32 << 20 + case "g1-small": + return 64 << 20 + default: + return 128 << 20 + } +} + +const baseInstanceConfig = `#cloud-config +write_files: + - path: /var/lib/camlistore/tmp/README + permissions: 0644 + content: | + This is the Camlistore /tmp directory. + - path: /var/lib/camlistore/mysql/README + permissions: 0644 + content: | + This is the Camlistore MySQL data directory. +coreos: + units: + - name: cam-journal-gatewayd.service + content: | + [Unit] + Description=Journal Gateway Service + Requires=cam-journal-gatewayd.socket + + [Service] + ExecStart=/usr/lib/systemd/systemd-journal-gatewayd + User=systemd-journal-gateway + Group=systemd-journal-gateway + SupplementaryGroups=systemd-journal + PrivateTmp=yes + PrivateDevices=yes + PrivateNetwork=yes + ProtectSystem=full + ProtectHome=yes + + [Install] + Also=cam-journal-gatewayd.socket + - name: cam-journal-gatewayd.socket + command: start + content: | + [Unit] + Description=Journal Gateway Service Socket + + [Socket] + ListenStream=/run/camjournald.sock + + [Install] + WantedBy=sockets.target + - name: mysql.service + command: start + content: | + [Unit] + Description=MySQL + After=docker.service + Requires=docker.service + + [Service] + ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker + ExecStart=/opt/bin/systemd-docker run --rm --name %n -v /var/lib/camlistore/mysql:/mysql -e INNODB_BUFFER_POOL_SIZE=NNN camlistore/mysql + RestartSec=1s + Restart=always + Type=notify + NotifyAccess=all + + [Install] + WantedBy=multi-user.target + - name: camlistored.service + command: start + content: | + [Unit] + Description=Camlistore + After=docker.service + Requires=docker.service mysql.service + + [Service] + ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker + ExecStart=/opt/bin/systemd-docker run --rm -p 80:80 -p 443:443 --name %n -v /run/camjournald.sock:/run/camjournald.sock -v /var/lib/camlistore/tmp:/tmp --link=mysql.service:mysqldb camlistore/camlistored + RestartSec=1s + Restart=always + Type=notify + NotifyAccess=all + + [Install] + WantedBy=multi-user.target +` diff --git a/misc/gce/notes.txt b/pkg/deploy/gce/notes.txt similarity index 100% rename from misc/gce/notes.txt rename to pkg/deploy/gce/notes.txt diff --git a/pkg/httputil/certs.go b/pkg/httputil/certs.go index a14309573..f3297bea0 100644 --- a/pkg/httputil/certs.go +++ b/pkg/httputil/certs.go @@ -17,16 +17,17 @@ limitations under the License. package httputil import ( + "bytes" "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "math/big" "net/http" - "os" "runtime" "sync" "time" @@ -46,21 +47,15 @@ var ( sysRootsGood bool ) -// GenSelfTLS generates a self-signed certificate and key for hostname, -// and writes them to the given paths. If it succeeds it also returns -// the SHA256 prefix of the new cert. -func GenSelfTLS(hostname, certPath, keyPath string) (error, string) { +// GenSelfTLS generates a self-signed certificate and key for hostname. +func GenSelfTLS(hostname string) (certPEM, keyPEM []byte, err error) { priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { - return fmt.Errorf("failed to generate private key: %s", err), "" + return certPEM, keyPEM, fmt.Errorf("failed to generate private key: %s", err) } now := time.Now() - // TODO(mpl): if no host is specified in the listening address - // (e.g ":3179") we'll end up in this case, and the self-signed - // will have "localhost" as a CommonName. But I don't think - // there's anything we can do about it. Maybe warn... if hostname == "" { hostname = "localhost" } @@ -80,37 +75,54 @@ func GenSelfTLS(hostname, certPath, keyPath string) (error, string) { derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) if err != nil { - return fmt.Errorf("Failed to create certificate: %s", err), "" + return certPEM, keyPEM, fmt.Errorf("failed to create certificate: %s", err) } + var buf bytes.Buffer + if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return certPEM, keyPEM, fmt.Errorf("error writing self-signed HTTPS cert: %v", err) + } + certPEM = []byte(string(buf.Bytes())) - certOut, err := wkfs.Create(certPath) - if err != nil { - return fmt.Errorf("failed to open %s for writing: %s", certPath, err), "" - } - if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { - return err, "" - } - if err := certOut.Close(); err != nil { - return fmt.Errorf("Writing writing self-signed HTTPS cert: %v", err), "" + buf.Reset() + if err := pem.Encode(&buf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + return certPEM, keyPEM, fmt.Errorf("error writing self-signed HTTPS private key: %v", err) } + keyPEM = buf.Bytes() + return certPEM, keyPEM, nil +} - cert, err := x509.ParseCertificate(derBytes) - if err != nil { - return fmt.Errorf("Failed to parse certificate: %v", err), "" +// CertFingerprint returns the SHA-256 prefix of the x509 certificate encoded in certPEM. +func CertFingerprint(certPEM []byte) (string, error) { + p, _ := pem.Decode(certPEM) + if p == nil { + return "", errors.New("no valid PEM data found") } - sig := hashutil.SHA256Prefix(cert.Raw) + cert, err := x509.ParseCertificate(p.Bytes) + if err != nil { + return "", fmt.Errorf("failed to parse certificate: %v", err) + } + return hashutil.SHA256Prefix(cert.Raw), nil +} - keyOut, err := wkfs.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) +// GenSelfTLSFiles generates a self-signed certificate and key for hostname, +// and writes them to the given paths. If it succeeds it also returns +// the SHA256 prefix of the new cert. +func GenSelfTLSFiles(hostname, certPath, keyPath string) (fingerprint string, err error) { + cert, key, err := GenSelfTLS(hostname) if err != nil { - return fmt.Errorf("failed to open %s for writing: %v", keyPath, err), "" + return "", err } - if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { - return fmt.Errorf("Error writing self-signed HTTPS private key: %v", err), "" + sig, err := CertFingerprint(cert) + if err != nil { + return "", fmt.Errorf("could not get SHA-256 fingerprint of certificate: %v", err) } - if err := keyOut.Close(); err != nil { - return fmt.Errorf("Error writing self-signed HTTPS private key: %v", err), "" + if err := wkfs.WriteFile(certPath, cert, 0666); err != nil { + return "", fmt.Errorf("failed to write self-signed TLS cert: %v", err) } - return nil, sig + if err := wkfs.WriteFile(keyPath, key, 0600); err != nil { + return "", fmt.Errorf("failed to write self-signed TLS key: %v", err) + } + return sig, nil } // InstallCerts adds Mozilla's Certificate Authority root set to diff --git a/server/camlistored/camlistored.go b/server/camlistored/camlistored.go index ead32b686..38e055002 100644 --- a/server/camlistored/camlistored.go +++ b/server/camlistored/camlistored.go @@ -18,8 +18,6 @@ limitations under the License. package main import ( - "crypto/x509" - "encoding/pem" "flag" "fmt" "io" @@ -37,7 +35,6 @@ import ( "time" "camlistore.org/pkg/buildinfo" - "camlistore.org/pkg/hashutil" "camlistore.org/pkg/httputil" "camlistore.org/pkg/legal/legalprint" "camlistore.org/pkg/netutil" @@ -197,7 +194,7 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string) _, err2 := wkfs.Stat(key) if err1 != nil || err2 != nil { if os.IsNotExist(err1) || os.IsNotExist(err2) { - err, sig := httputil.GenSelfTLS(hostname, defCert, defKey) + sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey) if err != nil { exitf("Could not generate self-signed TLS cert: %q", err) } @@ -209,7 +206,7 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string) } // Always generate new certificates if the config's httpsCert and httpsKey are empty. if cert == "" && key == "" { - err, sig := httputil.GenSelfTLS(hostname, defCert, defKey) + sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey) if err != nil { exitf("Could not generate self signed creds: %q", err) } @@ -221,15 +218,10 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string) if err != nil { exitf("Failed to read pem certificate: %s", err) } - block, _ := pem.Decode(data) - if block == nil { - exitf("Failed to decode pem certificate") - } - certif, err := x509.ParseCertificate(block.Bytes) + sig, err := httputil.CertFingerprint(data) if err != nil { - exitf("Failed to parse certificate: %v", err) + exitf("certificate error: %v", err) } - sig := hashutil.SHA256Prefix(certif.Raw) log.Printf("TLS enabled, with SHA-256 certificate fingerprint: %v", sig) ws.SetTLS(cert, key) }