mirror of https://github.com/perkeep/perkeep.git
pkg/deploy/gce: lib + tool to deploy on Google Cloud
http://camlistore.org/issue/531 Change-Id: I69ffe0544341d380bb844aef1dcca8a1ae441ea7
This commit is contained in:
mpl
parent
5020c59dcb
commit
99fe925efc
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
Copyright 2014 The Camlistore Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// The camdeploy program deploys Camlistore on cloud computing platforms such as Google
|
||||
// Compute Engine or Amazon EC2.
|
||||
package main
|
||||
|
||||
import (
|
||||
"camlistore.org/pkg/cmdmain"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cmdmain.Main()
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
/*
|
||||
Copyright 2014 The Camlistore Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"camlistore.org/pkg/cmdmain"
|
||||
"camlistore.org/pkg/context"
|
||||
"camlistore.org/pkg/deploy/gce"
|
||||
|
||||
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
|
||||
)
|
||||
|
||||
type gceCmd struct {
|
||||
project string
|
||||
zone string
|
||||
machine string
|
||||
instName string
|
||||
hostname string
|
||||
certFile string
|
||||
keyFile string
|
||||
sshPub string
|
||||
verbose bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdmain.RegisterCommand("gce", func(flags *flag.FlagSet) cmdmain.CommandRunner {
|
||||
cmd := new(gceCmd)
|
||||
flags.StringVar(&cmd.project, "project", "", "Name of Project.")
|
||||
flags.StringVar(&cmd.zone, "zone", gce.Zone, "GCE zone.")
|
||||
flags.StringVar(&cmd.machine, "machine", gce.Machine, "e.g. n1-standard-1, f1-micro, g1-small")
|
||||
flags.StringVar(&cmd.instName, "instance_name", gce.InstanceName, "Name of VM instance.")
|
||||
flags.StringVar(&cmd.hostname, "hostname", "", "Hostname for the instance and self-signed certificates. Must be given if generating self-signed certs.")
|
||||
flags.StringVar(&cmd.certFile, "cert", "", "Certificate file for TLS. A self-signed one will be generated if this flag is omitted.")
|
||||
flags.StringVar(&cmd.keyFile, "key", "", "Key file for the TLS certificate. Must be given with --cert")
|
||||
flags.StringVar(&cmd.sshPub, "ssh_public_key", "", "SSH public key file to authorize. Can modify later in Google's web UI anyway.")
|
||||
flags.BoolVar(&cmd.verbose, "verbose", false, "Be verbose.")
|
||||
return cmd
|
||||
})
|
||||
}
|
||||
|
||||
const (
|
||||
clientIdDat = "client-id.dat"
|
||||
clientSecretDat = "client-secret.dat"
|
||||
helpEnableAuth = `Enable authentication: in your project console, navigate to "APIs and auth", "Credentials", click on "Create new Client ID", and pick "Installed application", with type "Other". Copy the CLIENT ID to ` + clientIdDat + `, and the CLIENT SECRET to ` + clientSecretDat
|
||||
)
|
||||
|
||||
func (c *gceCmd) Describe() string {
|
||||
return "Deploy Camlistore on Google Compute Engine."
|
||||
}
|
||||
|
||||
func (c *gceCmd) Usage() {
|
||||
fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n %s\n\n",
|
||||
"camdeploy gce --project=<project> --hostname=<hostname> [options]",
|
||||
"camdeploy gce --project=<project> --cert=<cert file> --key=<key file> [options]")
|
||||
flag.PrintDefaults()
|
||||
fmt.Fprintln(os.Stderr, "\nTo get started:\n")
|
||||
printHelp()
|
||||
}
|
||||
|
||||
func printHelp() {
|
||||
for _, v := range []string{gce.HelpCreateProject, helpEnableAuth, gce.HelpEnableAPIs} {
|
||||
fmt.Fprintf(os.Stderr, "%v\n", v)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *gceCmd) RunCommand(args []string) error {
|
||||
if c.verbose {
|
||||
gce.Verbose = true
|
||||
}
|
||||
if c.project == "" {
|
||||
return cmdmain.UsageError("Missing --project flag.")
|
||||
}
|
||||
if (c.certFile == "") != (c.keyFile == "") {
|
||||
return cmdmain.UsageError("--cert and --key must both be given together.")
|
||||
}
|
||||
if c.certFile == "" && c.hostname == "" {
|
||||
return cmdmain.UsageError("Either --hostname, or --cert & --key must provided.")
|
||||
}
|
||||
config := gce.NewOAuthConfig(readFile(clientIdDat), readFile(clientSecretDat))
|
||||
config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
|
||||
|
||||
instConf := &gce.InstanceConf{
|
||||
Name: c.instName,
|
||||
Project: c.project,
|
||||
Machine: c.machine,
|
||||
Zone: c.zone,
|
||||
CertFile: c.certFile,
|
||||
KeyFile: c.keyFile,
|
||||
Hostname: c.hostname,
|
||||
}
|
||||
if c.sshPub != "" {
|
||||
instConf.SSHPub = strings.TrimSpace(readFile(c.sshPub))
|
||||
}
|
||||
|
||||
depl := &gce.Deployer{
|
||||
Cl: &http.Client{Transport: c.transport(config)},
|
||||
Conf: instConf,
|
||||
}
|
||||
inst, err := depl.Create(context.TODO())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Instance is up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP)
|
||||
return nil
|
||||
}
|
||||
|
||||
func readFile(v string) string {
|
||||
slurp, err := ioutil.ReadFile(v)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
msg := fmt.Sprintf("%v does not exist.", v)
|
||||
if v == clientIdDat || v == clientSecretDat {
|
||||
msg = fmt.Sprintf("%v\n%s", msg, helpEnableAuth)
|
||||
}
|
||||
log.Fatal(msg)
|
||||
}
|
||||
log.Fatalf("Error reading %s: %v", v, err)
|
||||
}
|
||||
return strings.TrimSpace(string(slurp))
|
||||
}
|
||||
|
||||
func (c *gceCmd) transport(config *oauth.Config) *oauth.Transport {
|
||||
tr := &oauth.Transport{
|
||||
Config: config,
|
||||
}
|
||||
tokenCache := oauth.CacheFile(c.project + "-token.dat")
|
||||
token, err := tokenCache.Token()
|
||||
if err != nil {
|
||||
log.Printf("Error getting token from %s: %v", string(tokenCache), err)
|
||||
log.Printf("Get auth code from %v", config.AuthCodeURL("my-state"))
|
||||
io.WriteString(os.Stdout, "\nEnter auth code:")
|
||||
sc := bufio.NewScanner(os.Stdin)
|
||||
sc.Scan()
|
||||
authCode := strings.TrimSpace(sc.Text())
|
||||
token, err = tr.Exchange(authCode)
|
||||
if err != nil {
|
||||
log.Fatalf("Error exchanging auth code for a token: %v", err)
|
||||
}
|
||||
tokenCache.PutToken(token)
|
||||
}
|
||||
tr.Token = token
|
||||
return tr
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
client-id.dat
|
||||
client-secret.dat
|
||||
token.dat
|
||||
*-token.dat
|
||||
*.crt
|
||||
*.key
|
||||
|
|
@ -1,458 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"camlistore.org/pkg/httputil"
|
||||
"camlistore.org/pkg/osutil"
|
||||
|
||||
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
|
||||
compute "camlistore.org/third_party/code.google.com/p/google-api-go-client/compute/v1"
|
||||
storage "camlistore.org/third_party/code.google.com/p/google-api-go-client/storage/v1"
|
||||
)
|
||||
|
||||
var (
|
||||
proj = flag.String("project", "", "Name of Project.")
|
||||
zone = flag.String("zone", "us-central1-a", "GCE zone.")
|
||||
mach = flag.String("machinetype", "g1-small", "e.g. n1-standard-1, f1-micro, g1-small")
|
||||
instName = flag.String("instance_name", "camlistore-server", "Name of VM instance.")
|
||||
hostname = flag.String("hostname", "", "Hostname for the instance and self-signed certificates. Must be given if generating self-signed certs.")
|
||||
certFile = flag.String("cert", "", "Certificate file for TLS. A self-signed one will be generated if this flag is omitted.")
|
||||
keyFile = flag.String("key", "", "Key file for the TLS certificate. Must be given with --cert")
|
||||
sshPub = flag.String("ssh_public_key", "", "SSH public key file to authorize. Can modify later in Google's web UI anyway.")
|
||||
verbose = flag.Bool("verbose", false, "Be verbose.")
|
||||
)
|
||||
|
||||
const (
|
||||
clientIdDat = "client-id.dat"
|
||||
clientSecretDat = "client-secret.dat"
|
||||
helpCreateProject = "Create new project: go to https://console.developers.google.com to create a new Project."
|
||||
helpEnableAuth = `Enable authentication: in your project console, navigate to "APIs and auth", "Credentials", click on "Create new Client ID"¸ and pick "Installed application", with type "Other". Copy the CLIENT ID to ` + clientIdDat + `, and the CLIENT SECRET to ` + clientSecretDat
|
||||
helpEnableAPIs = `Enable the project APIs: in your project console, navigate to "APIs and auth", "APIs". In the list, enable "Google Cloud Storage", "Google Cloud Storage JSON API", and "Google Compute Engine".`
|
||||
)
|
||||
|
||||
func readFile(v string) string {
|
||||
slurp, err := ioutil.ReadFile(v)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
log.Fatalf("%v does not exist.\n%s", v, helpEnableAuth)
|
||||
}
|
||||
log.Fatalf("Error reading %s: %v", v, err)
|
||||
}
|
||||
return strings.TrimSpace(string(slurp))
|
||||
}
|
||||
|
||||
func printHelp() {
|
||||
for _, v := range []string{helpCreateProject, helpEnableAuth, helpEnableAPIs} {
|
||||
fmt.Fprintf(os.Stderr, "%v\n", v)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n %s\n\n",
|
||||
"go run create.go --project=<project> --hostname=<hostname> [options]",
|
||||
"go run create.go --project=<project> --cert=<cert file> --key=<key file> [options]")
|
||||
flag.PrintDefaults()
|
||||
fmt.Fprintln(os.Stderr, "\nTo get started with this script:\n")
|
||||
printHelp()
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Usage = usage
|
||||
flag.Parse()
|
||||
if *proj == "" {
|
||||
log.Print("Missing --project flag.")
|
||||
usage()
|
||||
return
|
||||
}
|
||||
if (*certFile == "") != (*keyFile == "") {
|
||||
log.Print("--cert and --key must both be given together.")
|
||||
return
|
||||
}
|
||||
if *certFile == "" && *hostname == "" {
|
||||
log.Print("Either --hostname, or --cert & --key must provided.")
|
||||
return
|
||||
}
|
||||
prefix := "https://www.googleapis.com/compute/v1/projects/" + *proj
|
||||
imageURL := "https://www.googleapis.com/compute/v1/projects/coreos-cloud/global/images/coreos-alpha-402-2-0-v20140807"
|
||||
machType := prefix + "/zones/" + *zone + "/machineTypes/" + *mach
|
||||
|
||||
config := &oauth.Config{
|
||||
// The client-id and secret should be for an "Installed Application" when using
|
||||
// the CLI. Later we'll use a web application with a callback.
|
||||
ClientId: readFile(clientIdDat),
|
||||
ClientSecret: readFile(clientSecretDat),
|
||||
Scope: strings.Join([]string{
|
||||
compute.DevstorageFull_controlScope,
|
||||
compute.ComputeScope,
|
||||
"https://www.googleapis.com/auth/sqlservice",
|
||||
"https://www.googleapis.com/auth/sqlservice.admin",
|
||||
}, " "),
|
||||
AuthURL: "https://accounts.google.com/o/oauth2/auth",
|
||||
TokenURL: "https://accounts.google.com/o/oauth2/token",
|
||||
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
|
||||
}
|
||||
|
||||
tr := &oauth.Transport{
|
||||
Config: config,
|
||||
}
|
||||
|
||||
tokenCache := oauth.CacheFile(*proj + "-token.dat")
|
||||
token, err := tokenCache.Token()
|
||||
if err != nil {
|
||||
log.Printf("Error getting token from %s: %v", string(tokenCache), err)
|
||||
log.Printf("Get auth code from %v", config.AuthCodeURL("my-state"))
|
||||
os.Stdout.Write([]byte("\nEnter auth code: "))
|
||||
sc := bufio.NewScanner(os.Stdin)
|
||||
sc.Scan()
|
||||
authCode := strings.TrimSpace(sc.Text())
|
||||
token, err = tr.Exchange(authCode)
|
||||
if err != nil {
|
||||
log.Fatalf("Error exchanging auth code for a token: %v", err)
|
||||
}
|
||||
tokenCache.PutToken(token)
|
||||
}
|
||||
|
||||
tr.Token = token
|
||||
oauthClient := &http.Client{Transport: tr}
|
||||
computeService, _ := compute.New(oauthClient)
|
||||
storageService, _ := storage.New(oauthClient)
|
||||
|
||||
cloudConfig := `#cloud-config
|
||||
write_files:
|
||||
- path: /var/lib/camlistore/tmp/README
|
||||
permissions: 0644
|
||||
content: |
|
||||
This is the Camlistore /tmp directory.
|
||||
- path: /var/lib/camlistore/mysql/README
|
||||
permissions: 0644
|
||||
content: |
|
||||
This is the Camlistore MySQL data directory.
|
||||
coreos:
|
||||
units:
|
||||
- name: cam-journal-gatewayd.service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Journal Gateway Service
|
||||
Requires=cam-journal-gatewayd.socket
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/lib/systemd/systemd-journal-gatewayd
|
||||
User=systemd-journal-gateway
|
||||
Group=systemd-journal-gateway
|
||||
SupplementaryGroups=systemd-journal
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
PrivateNetwork=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
|
||||
[Install]
|
||||
Also=cam-journal-gatewayd.socket
|
||||
- name: cam-journal-gatewayd.socket
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Journal Gateway Service Socket
|
||||
|
||||
[Socket]
|
||||
ListenStream=/run/camjournald.sock
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
- name: mysql.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=MySQL
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker
|
||||
ExecStart=/opt/bin/systemd-docker run --rm --name %n -v /var/lib/camlistore/mysql:/mysql -e INNODB_BUFFER_POOL_SIZE=NNN camlistore/mysql
|
||||
RestartSec=1s
|
||||
Restart=always
|
||||
Type=notify
|
||||
NotifyAccess=all
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: camlistored.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Camlistore
|
||||
After=docker.service
|
||||
Requires=docker.service mysql.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker
|
||||
ExecStart=/opt/bin/systemd-docker run --rm -p 80:80 -p 443:443 --name %n -v /run/camjournald.sock:/run/camjournald.sock -v /var/lib/camlistore/tmp:/tmp --link=mysql.service:mysqldb camlistore/camlistored
|
||||
RestartSec=1s
|
||||
Restart=always
|
||||
Type=notify
|
||||
NotifyAccess=all
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
cloudConfig = strings.Replace(cloudConfig, "INNODB_BUFFER_POOL_SIZE=NNN", "INNODB_BUFFER_POOL_SIZE="+strconv.Itoa(innodbBufferPoolSize(*mach)), -1)
|
||||
|
||||
const maxCloudConfig = 32 << 10 // per compute API docs
|
||||
if len(cloudConfig) > maxCloudConfig {
|
||||
log.Fatalf("cloud config length of %d bytes is over %d byte limit", len(cloudConfig), maxCloudConfig)
|
||||
}
|
||||
if *sshPub != "" {
|
||||
key := strings.TrimSpace(readFile(*sshPub))
|
||||
cloudConfig += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", key)
|
||||
}
|
||||
|
||||
blobBucket := *proj + "-camlistore-blobs"
|
||||
configBucket := *proj + "-camlistore-config"
|
||||
needBucket := map[string]bool{
|
||||
blobBucket: true,
|
||||
configBucket: true,
|
||||
}
|
||||
|
||||
buckets, err := storageService.Buckets.List(*proj).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Error listing buckets: %v", err)
|
||||
}
|
||||
for _, it := range buckets.Items {
|
||||
delete(needBucket, it.Name)
|
||||
}
|
||||
if len(needBucket) > 0 {
|
||||
log.Printf("Need to create buckets: %v", needBucket)
|
||||
var waitBucket sync.WaitGroup
|
||||
for name := range needBucket {
|
||||
name := name
|
||||
waitBucket.Add(1)
|
||||
go func() {
|
||||
defer waitBucket.Done()
|
||||
log.Printf("Creating bucket %s", name)
|
||||
b, err := storageService.Buckets.Insert(*proj, &storage.Bucket{
|
||||
Id: name,
|
||||
Name: name,
|
||||
}).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating bucket %s: %v", name, err)
|
||||
}
|
||||
log.Printf("Created bucket %s: %+v", name, b)
|
||||
}()
|
||||
}
|
||||
waitBucket.Wait()
|
||||
}
|
||||
|
||||
if *certFile == "" {
|
||||
// A bit paranoid since these are illigal GCE project name characters anyway but it doesn't hurt.
|
||||
r := strings.NewReplacer(".", "_", "/", "_", "\\", "_")
|
||||
*certFile = r.Replace(*proj) + ".crt"
|
||||
*keyFile = r.Replace(*proj) + ".key"
|
||||
|
||||
_, errc := os.Stat(*certFile)
|
||||
_, errk := os.Stat(*keyFile)
|
||||
switch {
|
||||
case os.IsNotExist(errc) && os.IsNotExist(errk):
|
||||
log.Printf("Generating self-signed certificate for %v ...", *hostname)
|
||||
err, sig := httputil.GenSelfTLS(*hostname, *certFile, *keyFile)
|
||||
if err != nil {
|
||||
log.Fatalf("Error generating certificates: %v", err)
|
||||
}
|
||||
log.Printf("Wrote key to %s, and certificate to %s with fingerprint %s", *keyFile, *certFile, sig)
|
||||
case errc != nil:
|
||||
log.Fatalf("Couldn't stat cert: %v", errc)
|
||||
case errk != nil:
|
||||
log.Fatalf("Couldn't stat key: %v", errk)
|
||||
default:
|
||||
log.Printf("Using certificate %s and key %s", *certFile, *keyFile)
|
||||
}
|
||||
}
|
||||
|
||||
log.Print("Uploading certificate and key...")
|
||||
err = uploadFile(storageService, *certFile, configBucket, filepath.Base(osutil.DefaultTLSCert()))
|
||||
if err != nil {
|
||||
log.Fatalf("Cert upload failed: %v", err)
|
||||
}
|
||||
err = uploadFile(storageService, *keyFile, configBucket, filepath.Base(osutil.DefaultTLSKey()))
|
||||
if err != nil {
|
||||
log.Fatalf("Key upload failed: %v", err)
|
||||
}
|
||||
|
||||
instance := &compute.Instance{
|
||||
Name: *instName,
|
||||
Description: "Camlistore server",
|
||||
MachineType: machType,
|
||||
Disks: []*compute.AttachedDisk{
|
||||
{
|
||||
AutoDelete: true,
|
||||
Boot: true,
|
||||
Type: "PERSISTENT",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
DiskName: *instName + "-coreos-stateless-pd",
|
||||
SourceImage: imageURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
Tags: &compute.Tags{
|
||||
Items: []string{"http-server", "https-server"},
|
||||
},
|
||||
Metadata: &compute.Metadata{
|
||||
Items: []*compute.MetadataItems{
|
||||
{
|
||||
Key: "camlistore-username",
|
||||
Value: "test",
|
||||
},
|
||||
{
|
||||
Key: "camlistore-password",
|
||||
Value: "insecure", // TODO: this won't be cleartext later
|
||||
},
|
||||
{
|
||||
Key: "camlistore-blob-bucket",
|
||||
Value: "gs://" + blobBucket,
|
||||
},
|
||||
{
|
||||
Key: "camlistore-config-bucket",
|
||||
Value: "gs://" + configBucket,
|
||||
},
|
||||
{
|
||||
Key: "user-data",
|
||||
Value: cloudConfig,
|
||||
},
|
||||
},
|
||||
},
|
||||
NetworkInterfaces: []*compute.NetworkInterface{
|
||||
&compute.NetworkInterface{
|
||||
AccessConfigs: []*compute.AccessConfig{
|
||||
&compute.AccessConfig{
|
||||
Type: "ONE_TO_ONE_NAT",
|
||||
Name: "External NAT",
|
||||
},
|
||||
},
|
||||
Network: prefix + "/global/networks/default",
|
||||
},
|
||||
},
|
||||
ServiceAccounts: []*compute.ServiceAccount{
|
||||
{
|
||||
Email: "default",
|
||||
Scopes: []string{
|
||||
compute.DevstorageFull_controlScope,
|
||||
compute.ComputeScope,
|
||||
"https://www.googleapis.com/auth/sqlservice",
|
||||
"https://www.googleapis.com/auth/sqlservice.admin",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if *hostname != "" {
|
||||
instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
|
||||
Key: "camlistore-hostname",
|
||||
Value: *hostname,
|
||||
})
|
||||
}
|
||||
const localMySQL = false // later
|
||||
if localMySQL {
|
||||
instance.Disks = append(instance.Disks, &compute.AttachedDisk{
|
||||
AutoDelete: false,
|
||||
Boot: false,
|
||||
Type: "PERSISTENT",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
DiskName: "camlistore-mysql-index-pd",
|
||||
DiskSizeGb: 4,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
log.Print("Creating instance...")
|
||||
op, err := computeService.Instances.Insert(*proj, *zone, instance).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create instance: %v", err)
|
||||
}
|
||||
opName := op.Name
|
||||
if *verbose {
|
||||
log.Printf("Created. Waiting on operation %v", opName)
|
||||
}
|
||||
OpLoop:
|
||||
for {
|
||||
time.Sleep(2 * time.Second)
|
||||
op, err := computeService.ZoneOperations.Get(*proj, *zone, opName).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get op %s: %v", opName, err)
|
||||
}
|
||||
switch op.Status {
|
||||
case "PENDING", "RUNNING":
|
||||
if *verbose {
|
||||
log.Printf("Waiting on operation %v", opName)
|
||||
}
|
||||
continue
|
||||
case "DONE":
|
||||
if op.Error != nil {
|
||||
for _, operr := range op.Error.Errors {
|
||||
log.Printf("Error: %+v", operr)
|
||||
}
|
||||
log.Fatalf("Failed to start.")
|
||||
}
|
||||
if *verbose {
|
||||
log.Printf("Success. %+v", op)
|
||||
}
|
||||
break OpLoop
|
||||
default:
|
||||
log.Fatalf("Unknown status %q: %+v", op.Status, op)
|
||||
}
|
||||
}
|
||||
|
||||
inst, err := computeService.Instances.Get(*proj, *zone, *instName).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting instance after creation: %v", err)
|
||||
}
|
||||
|
||||
if *verbose {
|
||||
ij, _ := json.MarshalIndent(inst, "", " ")
|
||||
log.Printf("Instance: %s", ij)
|
||||
}
|
||||
|
||||
addr := inst.NetworkInterfaces[0].AccessConfigs[0].NatIP
|
||||
log.Printf("Instance is up at %s", addr)
|
||||
}
|
||||
|
||||
func uploadFile(service *storage.Service, localFilename, bucketName, objectName string) error {
|
||||
file, err := os.Open(localFilename)
|
||||
defer file.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error opening %v: %v", localFilename, err)
|
||||
}
|
||||
_, err = service.Objects.Insert(bucketName, &storage.Object{Name: objectName}).Media(file).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Objects.Insert for %v failed: %v", localFilename, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// returns the MySQL InnoDB buffer pool size (in bytes) as a function
|
||||
// of the GCE machine type.
|
||||
func innodbBufferPoolSize(machType string) int {
|
||||
// Totally arbitrary. We don't need much here because
|
||||
// camlistored slurps this all into its RAM on start-up
|
||||
// anyway. So this is all prety overkill and more than the
|
||||
// 8MB default.
|
||||
switch machType {
|
||||
case "f1-micro":
|
||||
return 32 << 20
|
||||
case "g1-small":
|
||||
return 64 << 20
|
||||
default:
|
||||
return 128 << 20
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,497 @@
|
|||
/*
|
||||
Copyright 2014 The Camlistore Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package gce provides tools to deploy Camlistore on Google Compute Engine.
|
||||
package gce
|
||||
|
||||
// TODO: we want to host our own docker images under camlistore.org, so we should make a
|
||||
// list. For the purposes of this package, we should add to the list camlistore/camlistored
|
||||
// and camlistore/mysql.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"camlistore.org/pkg/context"
|
||||
"camlistore.org/pkg/httputil"
|
||||
"camlistore.org/pkg/osutil"
|
||||
|
||||
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
|
||||
compute "camlistore.org/third_party/code.google.com/p/google-api-go-client/compute/v1"
|
||||
storage "camlistore.org/third_party/code.google.com/p/google-api-go-client/storage/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
projectsAPIURL = "https://www.googleapis.com/compute/v1/projects/"
|
||||
coreosImgURL = "https://www.googleapis.com/compute/v1/projects/coreos-cloud/global/images/coreos-stable-444-5-0-v20141016"
|
||||
|
||||
// default instance configuration values.
|
||||
InstanceName = "camlistore-server"
|
||||
Machine = "g1-small"
|
||||
Zone = "us-central1-a"
|
||||
|
||||
HelpCreateProject = "Create new project: go to https://console.developers.google.com to create a new Project."
|
||||
HelpEnableAPIs = `Enable the project APIs: in your project console, navigate to "APIs and auth", "APIs". In the list, enable "Google Cloud Storage", "Google Cloud Storage JSON API", and "Google Compute Engine".`
|
||||
HelpDeleteInstance = `Delete an existing Compute Engine instance: in your project console, navigate to "Compute", "Compute Engine", and "VM instances". Select your instance and click "Delete".`
|
||||
)
|
||||
|
||||
// Verbose enables more info to be printed.
|
||||
var Verbose bool
|
||||
|
||||
// NewOAuthConfig returns an OAuth configuration template.
|
||||
func NewOAuthConfig(clientId, clientSecret string) *oauth.Config {
|
||||
return &oauth.Config{
|
||||
Scope: strings.Join([]string{
|
||||
compute.DevstorageFull_controlScope,
|
||||
compute.ComputeScope,
|
||||
"https://www.googleapis.com/auth/sqlservice",
|
||||
"https://www.googleapis.com/auth/sqlservice.admin",
|
||||
}, " "),
|
||||
AuthURL: "https://accounts.google.com/o/oauth2/auth",
|
||||
TokenURL: "https://accounts.google.com/o/oauth2/token",
|
||||
ClientId: clientId,
|
||||
ClientSecret: clientSecret,
|
||||
}
|
||||
}
|
||||
|
||||
// InstanceConf is the configuration for the Google Compute Engine instance that will be deployed.
|
||||
type InstanceConf struct {
|
||||
Name string // Name given to the virtual machine instance.
|
||||
Project string // Google project ID where the instance is created.
|
||||
Machine string // Machine type.
|
||||
Zone string // Geographic zone.
|
||||
SSHPub string // SSH public key.
|
||||
CertFile string // HTTPS certificate file.
|
||||
KeyFile string // HTTPS key file.
|
||||
Hostname string // Fully qualified domain name.
|
||||
|
||||
configBucket string // Project + "-camlistore-config"
|
||||
blobBucket string // Project + "-camlistore-blobs"
|
||||
}
|
||||
|
||||
// Deployer creates and starts an instance such as defined in Conf.
|
||||
type Deployer struct {
|
||||
Cl *http.Client
|
||||
Conf *InstanceConf
|
||||
}
|
||||
|
||||
// Get returns the Instance corresponding to the Project, Zone, and Name defined in the
|
||||
// Deployer's Conf.
|
||||
func (d *Deployer) Get(ctx *context.Context) (*compute.Instance, error) {
|
||||
computeService, err := compute.New(d.Cl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do()
|
||||
}
|
||||
|
||||
// Create sets up and starts a Google Compute Engine instance as defined in d.Conf. It
|
||||
// creates the necessary Google Storage buckets beforehand.
|
||||
func (d *Deployer) Create(ctx *context.Context) (*compute.Instance, error) {
|
||||
computeService, _ := compute.New(d.Cl)
|
||||
storageService, _ := storage.New(d.Cl)
|
||||
|
||||
config := cloudConfig(d.Conf)
|
||||
const maxCloudConfig = 32 << 10 // per compute API docs
|
||||
if len(config) > maxCloudConfig {
|
||||
return nil, fmt.Errorf("cloud config length of %d bytes is over %d byte limit", len(config), maxCloudConfig)
|
||||
}
|
||||
|
||||
if err := d.setBuckets(storageService, ctx); err != nil {
|
||||
return nil, fmt.Errorf("could not create buckets: %v", err)
|
||||
}
|
||||
|
||||
if err := d.setupHTTPS(storageService); err != nil {
|
||||
return nil, fmt.Errorf("could not setup HTTPS: %v", err)
|
||||
}
|
||||
|
||||
if err := d.createInstance(storageService, computeService, ctx); err != nil {
|
||||
return nil, fmt.Errorf("could not create compute instance: %v", err)
|
||||
}
|
||||
|
||||
inst, err := computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting instance after creation: %v", err)
|
||||
}
|
||||
if Verbose {
|
||||
ij, _ := json.MarshalIndent(inst, "", " ")
|
||||
log.Printf("Instance: %s", ij)
|
||||
}
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
// createInstance starts the creation of the Compute Engine instance and waits for the
|
||||
// result of the creation operation. It should be called after setBuckets and setupHTTPS.
|
||||
func (d *Deployer) createInstance(storageService *storage.Service, computeService *compute.Service, ctx *context.Context) error {
|
||||
prefix := projectsAPIURL + d.Conf.Project
|
||||
machType := prefix + "/zones/" + d.Conf.Zone + "/machineTypes/" + d.Conf.Machine
|
||||
config := cloudConfig(d.Conf)
|
||||
instance := &compute.Instance{
|
||||
Name: d.Conf.Name,
|
||||
Description: "Camlistore server",
|
||||
MachineType: machType,
|
||||
Disks: []*compute.AttachedDisk{
|
||||
{
|
||||
AutoDelete: true,
|
||||
Boot: true,
|
||||
Type: "PERSISTENT",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
DiskName: d.Conf.Name + "-coreos-stateless-pd",
|
||||
SourceImage: coreosImgURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
Tags: &compute.Tags{
|
||||
Items: []string{"http-server", "https-server"},
|
||||
},
|
||||
Metadata: &compute.Metadata{
|
||||
Items: []*compute.MetadataItems{
|
||||
{
|
||||
Key: "camlistore-username",
|
||||
Value: "test",
|
||||
},
|
||||
{
|
||||
Key: "camlistore-password",
|
||||
Value: "insecure", // TODO: this won't be cleartext later
|
||||
},
|
||||
{
|
||||
Key: "camlistore-blob-bucket",
|
||||
Value: "gs://" + d.Conf.blobBucket,
|
||||
},
|
||||
{
|
||||
Key: "camlistore-config-bucket",
|
||||
Value: "gs://" + d.Conf.configBucket,
|
||||
},
|
||||
{
|
||||
Key: "user-data",
|
||||
Value: config,
|
||||
},
|
||||
},
|
||||
},
|
||||
NetworkInterfaces: []*compute.NetworkInterface{
|
||||
&compute.NetworkInterface{
|
||||
AccessConfigs: []*compute.AccessConfig{
|
||||
&compute.AccessConfig{
|
||||
Type: "ONE_TO_ONE_NAT",
|
||||
Name: "External NAT",
|
||||
},
|
||||
},
|
||||
Network: prefix + "/global/networks/default",
|
||||
},
|
||||
},
|
||||
ServiceAccounts: []*compute.ServiceAccount{
|
||||
{
|
||||
Email: "default",
|
||||
Scopes: []string{
|
||||
compute.DevstorageFull_controlScope,
|
||||
compute.ComputeScope,
|
||||
"https://www.googleapis.com/auth/sqlservice",
|
||||
"https://www.googleapis.com/auth/sqlservice.admin",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if d.Conf.Hostname != "" {
|
||||
instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
|
||||
Key: "camlistore-hostname",
|
||||
Value: d.Conf.Hostname,
|
||||
})
|
||||
}
|
||||
const localMySQL = false // later
|
||||
if localMySQL {
|
||||
instance.Disks = append(instance.Disks, &compute.AttachedDisk{
|
||||
AutoDelete: false,
|
||||
Boot: false,
|
||||
Type: "PERSISTENT",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
DiskName: "camlistore-mysql-index-pd",
|
||||
DiskSizeGb: 4,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if Verbose {
|
||||
log.Print("Creating instance...")
|
||||
}
|
||||
op, err := computeService.Instances.Insert(d.Conf.Project, d.Conf.Zone, instance).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create instance: %v", err)
|
||||
}
|
||||
opName := op.Name
|
||||
if Verbose {
|
||||
log.Printf("Created. Waiting on operation %v", opName)
|
||||
}
|
||||
OpLoop:
|
||||
for {
|
||||
if ctx.IsCanceled() {
|
||||
// TODO(mpl): should we destroy any created instance (and attached
|
||||
// resources) at this point? So that a subsequent run does not fail
|
||||
// with something like:
|
||||
// 2014/11/25 17:08:58 Error: &{Code:RESOURCE_ALREADY_EXISTS Location: Message:The resource 'projects/camli-mpl/zones/europe-west1-b/disks/camlistore-server-coreos-stateless-pd' already exists}
|
||||
return context.ErrCanceled
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
op, err := computeService.ZoneOperations.Get(d.Conf.Project, d.Conf.Zone, opName).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get op %s: %v", opName, err)
|
||||
}
|
||||
switch op.Status {
|
||||
case "PENDING", "RUNNING":
|
||||
if Verbose {
|
||||
log.Printf("Waiting on operation %v", opName)
|
||||
}
|
||||
continue
|
||||
case "DONE":
|
||||
if op.Error != nil {
|
||||
for _, operr := range op.Error.Errors {
|
||||
log.Printf("Error: %+v", operr)
|
||||
}
|
||||
return fmt.Errorf("failed to start.")
|
||||
}
|
||||
if Verbose {
|
||||
log.Printf("Success. %+v", op)
|
||||
}
|
||||
break OpLoop
|
||||
default:
|
||||
return fmt.Errorf("unknown status %q: %+v", op.Status, op)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloudConfig(conf *InstanceConf) string {
|
||||
config := strings.Replace(baseInstanceConfig, "INNODB_BUFFER_POOL_SIZE=NNN", "INNODB_BUFFER_POOL_SIZE="+strconv.Itoa(innodbBufferPoolSize(conf.Machine)), -1)
|
||||
if conf.SSHPub != "" {
|
||||
config += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", conf.SSHPub)
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// setBuckets defines the buckets needed by the instance and creates them.
|
||||
func (d *Deployer) setBuckets(storageService *storage.Service, ctx *context.Context) error {
|
||||
blobBucket := d.Conf.Project + "-camlistore-blobs"
|
||||
configBucket := d.Conf.Project + "-camlistore-config"
|
||||
needBucket := map[string]bool{
|
||||
blobBucket: true,
|
||||
configBucket: true,
|
||||
}
|
||||
|
||||
buckets, err := storageService.Buckets.List(d.Conf.Project).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error listing buckets: %v", err)
|
||||
}
|
||||
for _, it := range buckets.Items {
|
||||
delete(needBucket, it.Name)
|
||||
}
|
||||
if len(needBucket) > 0 {
|
||||
if Verbose {
|
||||
log.Printf("Need to create buckets: %v", needBucket)
|
||||
}
|
||||
var waitBucket sync.WaitGroup
|
||||
var bucketErr error
|
||||
for name := range needBucket {
|
||||
if ctx.IsCanceled() {
|
||||
return context.ErrCanceled
|
||||
}
|
||||
name := name
|
||||
waitBucket.Add(1)
|
||||
go func() {
|
||||
defer waitBucket.Done()
|
||||
if Verbose {
|
||||
log.Printf("Creating bucket %s", name)
|
||||
}
|
||||
b, err := storageService.Buckets.Insert(d.Conf.Project, &storage.Bucket{
|
||||
Id: name,
|
||||
Name: name,
|
||||
}).Do()
|
||||
if err != nil && bucketErr == nil {
|
||||
bucketErr = fmt.Errorf("error creating bucket %s: %v", name, err)
|
||||
return
|
||||
}
|
||||
if Verbose {
|
||||
log.Printf("Created bucket %s: %+v", name, b)
|
||||
}
|
||||
}()
|
||||
}
|
||||
waitBucket.Wait()
|
||||
if bucketErr != nil {
|
||||
return bucketErr
|
||||
}
|
||||
}
|
||||
d.Conf.configBucket = configBucket
|
||||
d.Conf.blobBucket = blobBucket
|
||||
return nil
|
||||
}
|
||||
|
||||
// setupHTTPS uploads to the configuration bucket the certificate and key used by the
|
||||
// instance for HTTPS. It generates them if d.Conf.CertFile or d.Conf.KeyFile is not defined.
|
||||
// It should be called after setBuckets.
|
||||
func (d *Deployer) setupHTTPS(storageService *storage.Service) error {
|
||||
var cert, key io.ReadCloser
|
||||
var err error
|
||||
if d.Conf.CertFile != "" && d.Conf.KeyFile != "" {
|
||||
cert, err = os.Open(d.Conf.CertFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cert.Close()
|
||||
key, err = os.Open(d.Conf.KeyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer key.Close()
|
||||
} else {
|
||||
if Verbose {
|
||||
log.Printf("Generating self-signed certificate for %v ...", d.Conf.Hostname)
|
||||
}
|
||||
certBytes, keyBytes, err := httputil.GenSelfTLS(d.Conf.Hostname)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating certificates: %v", err)
|
||||
}
|
||||
if Verbose {
|
||||
sig, err := httputil.CertFingerprint(certBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get sha256 fingerprint of certificate: %v", err)
|
||||
}
|
||||
log.Printf("Wrote certificate with fingerprint %s", sig)
|
||||
}
|
||||
cert = ioutil.NopCloser(bytes.NewReader(certBytes))
|
||||
key = ioutil.NopCloser(bytes.NewReader(keyBytes))
|
||||
}
|
||||
|
||||
if Verbose {
|
||||
log.Print("Uploading certificate and key...")
|
||||
}
|
||||
_, err = storageService.Objects.Insert(d.Conf.configBucket,
|
||||
&storage.Object{Name: filepath.Base(osutil.DefaultTLSCert())}).Media(cert).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cert upload failed: %v", err)
|
||||
}
|
||||
_, err = storageService.Objects.Insert(d.Conf.configBucket,
|
||||
&storage.Object{Name: filepath.Base(osutil.DefaultTLSKey())}).Media(key).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("key upload failed: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// returns the MySQL InnoDB buffer pool size (in bytes) as a function
|
||||
// of the GCE machine type.
|
||||
func innodbBufferPoolSize(machine string) int {
|
||||
// Totally arbitrary. We don't need much here because
|
||||
// camlistored slurps this all into its RAM on start-up
|
||||
// anyway. So this is all prety overkill and more than the
|
||||
// 8MB default.
|
||||
switch machine {
|
||||
case "f1-micro":
|
||||
return 32 << 20
|
||||
case "g1-small":
|
||||
return 64 << 20
|
||||
default:
|
||||
return 128 << 20
|
||||
}
|
||||
}
|
||||
|
||||
const baseInstanceConfig = `#cloud-config
|
||||
write_files:
|
||||
- path: /var/lib/camlistore/tmp/README
|
||||
permissions: 0644
|
||||
content: |
|
||||
This is the Camlistore /tmp directory.
|
||||
- path: /var/lib/camlistore/mysql/README
|
||||
permissions: 0644
|
||||
content: |
|
||||
This is the Camlistore MySQL data directory.
|
||||
coreos:
|
||||
units:
|
||||
- name: cam-journal-gatewayd.service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Journal Gateway Service
|
||||
Requires=cam-journal-gatewayd.socket
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/lib/systemd/systemd-journal-gatewayd
|
||||
User=systemd-journal-gateway
|
||||
Group=systemd-journal-gateway
|
||||
SupplementaryGroups=systemd-journal
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
PrivateNetwork=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
|
||||
[Install]
|
||||
Also=cam-journal-gatewayd.socket
|
||||
- name: cam-journal-gatewayd.socket
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Journal Gateway Service Socket
|
||||
|
||||
[Socket]
|
||||
ListenStream=/run/camjournald.sock
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
- name: mysql.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=MySQL
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker
|
||||
ExecStart=/opt/bin/systemd-docker run --rm --name %n -v /var/lib/camlistore/mysql:/mysql -e INNODB_BUFFER_POOL_SIZE=NNN camlistore/mysql
|
||||
RestartSec=1s
|
||||
Restart=always
|
||||
Type=notify
|
||||
NotifyAccess=all
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: camlistored.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Camlistore
|
||||
After=docker.service
|
||||
Requires=docker.service mysql.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin ibuildthecloud/systemd-docker
|
||||
ExecStart=/opt/bin/systemd-docker run --rm -p 80:80 -p 443:443 --name %n -v /run/camjournald.sock:/run/camjournald.sock -v /var/lib/camlistore/tmp:/tmp --link=mysql.service:mysqldb camlistore/camlistored
|
||||
RestartSec=1s
|
||||
Restart=always
|
||||
Type=notify
|
||||
NotifyAccess=all
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
|
|
@ -17,16 +17,17 @@ limitations under the License.
|
|||
package httputil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -46,21 +47,15 @@ var (
|
|||
sysRootsGood bool
|
||||
)
|
||||
|
||||
// GenSelfTLS generates a self-signed certificate and key for hostname,
|
||||
// and writes them to the given paths. If it succeeds it also returns
|
||||
// the SHA256 prefix of the new cert.
|
||||
func GenSelfTLS(hostname, certPath, keyPath string) (error, string) {
|
||||
// GenSelfTLS generates a self-signed certificate and key for hostname.
|
||||
func GenSelfTLS(hostname string) (certPEM, keyPEM []byte, err error) {
|
||||
priv, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate private key: %s", err), ""
|
||||
return certPEM, keyPEM, fmt.Errorf("failed to generate private key: %s", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// TODO(mpl): if no host is specified in the listening address
|
||||
// (e.g ":3179") we'll end up in this case, and the self-signed
|
||||
// will have "localhost" as a CommonName. But I don't think
|
||||
// there's anything we can do about it. Maybe warn...
|
||||
if hostname == "" {
|
||||
hostname = "localhost"
|
||||
}
|
||||
|
|
@ -80,37 +75,54 @@ func GenSelfTLS(hostname, certPath, keyPath string) (error, string) {
|
|||
|
||||
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create certificate: %s", err), ""
|
||||
return certPEM, keyPEM, fmt.Errorf("failed to create certificate: %s", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
|
||||
return certPEM, keyPEM, fmt.Errorf("error writing self-signed HTTPS cert: %v", err)
|
||||
}
|
||||
certPEM = []byte(string(buf.Bytes()))
|
||||
|
||||
certOut, err := wkfs.Create(certPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open %s for writing: %s", certPath, err), ""
|
||||
}
|
||||
if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
|
||||
return err, ""
|
||||
}
|
||||
if err := certOut.Close(); err != nil {
|
||||
return fmt.Errorf("Writing writing self-signed HTTPS cert: %v", err), ""
|
||||
buf.Reset()
|
||||
if err := pem.Encode(&buf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
|
||||
return certPEM, keyPEM, fmt.Errorf("error writing self-signed HTTPS private key: %v", err)
|
||||
}
|
||||
keyPEM = buf.Bytes()
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
cert, err := x509.ParseCertificate(derBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to parse certificate: %v", err), ""
|
||||
// CertFingerprint returns the SHA-256 prefix of the x509 certificate encoded in certPEM.
|
||||
func CertFingerprint(certPEM []byte) (string, error) {
|
||||
p, _ := pem.Decode(certPEM)
|
||||
if p == nil {
|
||||
return "", errors.New("no valid PEM data found")
|
||||
}
|
||||
sig := hashutil.SHA256Prefix(cert.Raw)
|
||||
cert, err := x509.ParseCertificate(p.Bytes)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse certificate: %v", err)
|
||||
}
|
||||
return hashutil.SHA256Prefix(cert.Raw), nil
|
||||
}
|
||||
|
||||
keyOut, err := wkfs.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
// GenSelfTLSFiles generates a self-signed certificate and key for hostname,
|
||||
// and writes them to the given paths. If it succeeds it also returns
|
||||
// the SHA256 prefix of the new cert.
|
||||
func GenSelfTLSFiles(hostname, certPath, keyPath string) (fingerprint string, err error) {
|
||||
cert, key, err := GenSelfTLS(hostname)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open %s for writing: %v", keyPath, err), ""
|
||||
return "", err
|
||||
}
|
||||
if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
|
||||
return fmt.Errorf("Error writing self-signed HTTPS private key: %v", err), ""
|
||||
sig, err := CertFingerprint(cert)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not get SHA-256 fingerprint of certificate: %v", err)
|
||||
}
|
||||
if err := keyOut.Close(); err != nil {
|
||||
return fmt.Errorf("Error writing self-signed HTTPS private key: %v", err), ""
|
||||
if err := wkfs.WriteFile(certPath, cert, 0666); err != nil {
|
||||
return "", fmt.Errorf("failed to write self-signed TLS cert: %v", err)
|
||||
}
|
||||
return nil, sig
|
||||
if err := wkfs.WriteFile(keyPath, key, 0600); err != nil {
|
||||
return "", fmt.Errorf("failed to write self-signed TLS key: %v", err)
|
||||
}
|
||||
return sig, nil
|
||||
}
|
||||
|
||||
// InstallCerts adds Mozilla's Certificate Authority root set to
|
||||
|
|
|
|||
|
|
@ -18,8 +18,6 @@ limitations under the License.
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -37,7 +35,6 @@ import (
|
|||
"time"
|
||||
|
||||
"camlistore.org/pkg/buildinfo"
|
||||
"camlistore.org/pkg/hashutil"
|
||||
"camlistore.org/pkg/httputil"
|
||||
"camlistore.org/pkg/legal/legalprint"
|
||||
"camlistore.org/pkg/netutil"
|
||||
|
|
@ -197,7 +194,7 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string)
|
|||
_, err2 := wkfs.Stat(key)
|
||||
if err1 != nil || err2 != nil {
|
||||
if os.IsNotExist(err1) || os.IsNotExist(err2) {
|
||||
err, sig := httputil.GenSelfTLS(hostname, defCert, defKey)
|
||||
sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey)
|
||||
if err != nil {
|
||||
exitf("Could not generate self-signed TLS cert: %q", err)
|
||||
}
|
||||
|
|
@ -209,7 +206,7 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string)
|
|||
}
|
||||
// Always generate new certificates if the config's httpsCert and httpsKey are empty.
|
||||
if cert == "" && key == "" {
|
||||
err, sig := httputil.GenSelfTLS(hostname, defCert, defKey)
|
||||
sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey)
|
||||
if err != nil {
|
||||
exitf("Could not generate self signed creds: %q", err)
|
||||
}
|
||||
|
|
@ -221,15 +218,10 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string)
|
|||
if err != nil {
|
||||
exitf("Failed to read pem certificate: %s", err)
|
||||
}
|
||||
block, _ := pem.Decode(data)
|
||||
if block == nil {
|
||||
exitf("Failed to decode pem certificate")
|
||||
}
|
||||
certif, err := x509.ParseCertificate(block.Bytes)
|
||||
sig, err := httputil.CertFingerprint(data)
|
||||
if err != nil {
|
||||
exitf("Failed to parse certificate: %v", err)
|
||||
exitf("certificate error: %v", err)
|
||||
}
|
||||
sig := hashutil.SHA256Prefix(certif.Raw)
|
||||
log.Printf("TLS enabled, with SHA-256 certificate fingerprint: %v", sig)
|
||||
ws.SetTLS(cert, key)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue