mirror of https://github.com/perkeep/perkeep.git
pkg/deploy/gce: remove GCE deploy support
We used to have a web-based Perkeep launcher at perkeep.org/launch that created a GCE-based Perkeep instance for users, where they pay Google for compute time. (One of those "one click deploy" template things) Unfortunately, Google broke their APIs for doing the third party VM creations and we disabled it some years ago. But the code remains. And now, updating it again, we find that they've broken it again: Error: pkg/deploy/gce/deploy.go:358:4: servicemanagement.NewServicesService(s).Enable undefined (type *servicemanagement.ServicesService has no field or method Enable) It's not worth fighting Google's API breakages. Just remove the GCE launcher support as it's been unused for years. We can always resurrect this code from git if really needed. But a Digital Ocean or Fly launcher would probably be much easier. Signed-off-by: Brad Fitzpatrick <brad@danga.com>
This commit is contained in:
Brad Fitzpatrick
parent
199456cf97
commit
b5823a65b9
|
|
@ -1,4 +1,9 @@
|
|||
on: [push, pull_request]
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "main"
|
||||
pull_request:
|
||||
# all PRs on all branches
|
||||
name: "tests/lint"
|
||||
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
on: [push, pull_request]
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "main"
|
||||
pull_request:
|
||||
# all PRs on all branches
|
||||
name: "tests/linux"
|
||||
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
on: [push, pull_request]
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "main"
|
||||
pull_request:
|
||||
# all PRs on all branches
|
||||
name: "tests/macos"
|
||||
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
on: [push, pull_request]
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "main"
|
||||
pull_request:
|
||||
# all PRs on all branches
|
||||
name: "tests/tidy"
|
||||
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
on: [push, pull_request]
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "main"
|
||||
pull_request:
|
||||
# all PRs on all branches
|
||||
name: "tests/windows"
|
||||
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
/*
|
||||
Copyright 2014 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// The pk-deploy program deploys Perkeep on cloud computing platforms
|
||||
// such as Google Compute Engine or Amazon EC2.
|
||||
package main // import "perkeep.org/cmd/pk-deploy"
|
||||
|
||||
import (
|
||||
"perkeep.org/pkg/cmdmain"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cmdmain.Main()
|
||||
}
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
/*
|
||||
Copyright 2014 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"perkeep.org/pkg/cmdmain"
|
||||
"perkeep.org/pkg/deploy/gce"
|
||||
|
||||
"go4.org/oauthutil"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
type gceCmd struct {
|
||||
project string
|
||||
zone string
|
||||
machine string
|
||||
instName string
|
||||
verbose bool
|
||||
hostname string
|
||||
wip bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdmain.RegisterMode("gce", func(flags *flag.FlagSet) cmdmain.CommandRunner {
|
||||
cmd := new(gceCmd)
|
||||
flags.StringVar(&cmd.project, "project", "", "Name of Project.")
|
||||
flags.StringVar(&cmd.zone, "zone", gce.DefaultRegion, "GCE zone or region. If a region is given, a random zone in that region is selected. See https://cloud.google.com/compute/docs/zones")
|
||||
flags.StringVar(&cmd.machine, "machine", gce.DefaultMachineType, "GCE machine type (e.g. n1-standard-1, f1-micro, g1-small); see https://cloud.google.com/compute/docs/machine-types")
|
||||
flags.StringVar(&cmd.instName, "instance_name", gce.DefaultInstanceName, "Name of VM instance.")
|
||||
flags.StringVar(&cmd.hostname, "hostname", "", "Optional hostname for the instance. If set, the custom metadata variable \"camlistore-hostname\" on the instance takes that value. Otherwise, perkeepd sets that variable to the hostname we get from the camliNet DNS.")
|
||||
flags.BoolVar(&cmd.wip, "wip", false, "Developer option. Deploy the WORKINPROGRESS perkeepd tarball.")
|
||||
flags.BoolVar(&cmd.verbose, "verbose", false, "Be verbose.")
|
||||
return cmd
|
||||
})
|
||||
}
|
||||
|
||||
const (
|
||||
clientIdDat = "client-id.dat"
|
||||
clientSecretDat = "client-secret.dat"
|
||||
helpEnableAuth = `Enable authentication: in your project console, navigate to "APIs and auth", "Credentials", click on "Create new Client ID", and pick "Installed application", with type "Other". Copy the CLIENT ID to ` + clientIdDat + `, and the CLIENT SECRET to ` + clientSecretDat
|
||||
helpCreateProject = "Go to " + gce.ConsoleURL + " to create a new Google Cloud project"
|
||||
helpEnableAPIs = `Enable the project APIs: in your project console, navigate to "APIs and auth", "APIs". In the list, enable "Google Cloud Storage", "Google Cloud Storage", "Google Cloud Storage JSON API", and "Google Compute Engine".`
|
||||
)
|
||||
|
||||
func (c *gceCmd) Describe() string {
|
||||
return "Deploy Perkeep on Google Compute Engine."
|
||||
}
|
||||
|
||||
func (c *gceCmd) Usage() {
|
||||
fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n %s\n\n",
|
||||
"pk-deploy gce --project=<project> --hostname=<hostname> [options]",
|
||||
"pk-deploy gce --project=<project> --cert=<cert file> --key=<key file> [options]")
|
||||
flag.PrintDefaults()
|
||||
fmt.Fprintln(os.Stderr, "\nTo get started:")
|
||||
printHelp()
|
||||
}
|
||||
|
||||
func printHelp() {
|
||||
for _, v := range []string{helpCreateProject, helpEnableAuth, helpEnableAPIs} {
|
||||
fmt.Fprintf(os.Stderr, "%v\n", v)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *gceCmd) RunCommand(args []string) error {
|
||||
if c.verbose {
|
||||
gce.Verbose = true
|
||||
}
|
||||
if c.project == "" {
|
||||
return cmdmain.UsageError("Missing --project flag.")
|
||||
}
|
||||
|
||||
// We embed the client ID and client secret, per
|
||||
// https://developers.google.com/identity/protocols/OAuth2InstalledApp
|
||||
// Notably: "The client ID and client secret obtained from the
|
||||
// Developers Console are embedded in the source code of your
|
||||
// application. In this context, the client secret is
|
||||
// obviously not treated as a secret."
|
||||
//
|
||||
// These were created at:
|
||||
// https://console.developers.google.com/apis/credentials?project=camlistore-website
|
||||
// (Notes for Brad and Mathieu)
|
||||
const (
|
||||
clientID = "574004351801-9qqoggh6b5v3jqt722v43ikmgmtv60h3.apps.googleusercontent.com"
|
||||
clientSecret = "Gf1zwaOcbJnRTE5zD4feKaTI" // NOT a secret, despite name
|
||||
)
|
||||
config := gce.NewOAuthConfig(clientID, clientSecret)
|
||||
config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
|
||||
|
||||
hc := oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{
|
||||
Config: config,
|
||||
CacheFile: c.project + "-token.json",
|
||||
AuthCode: func() string {
|
||||
fmt.Println("Get auth code from:")
|
||||
fmt.Printf("%v\n\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
|
||||
fmt.Print("Enter auth code: ")
|
||||
sc := bufio.NewScanner(os.Stdin)
|
||||
sc.Scan()
|
||||
return strings.TrimSpace(sc.Text())
|
||||
},
|
||||
}))
|
||||
|
||||
zone := c.zone
|
||||
if gce.LooksLikeRegion(zone) {
|
||||
region := zone
|
||||
zones, err := gce.ZonesOfRegion(hc, c.project, region)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(zones) == 0 {
|
||||
return fmt.Errorf("no zones found in region %q; invalid region?", region)
|
||||
}
|
||||
zone = zones[rand.Intn(len(zones))]
|
||||
}
|
||||
|
||||
instConf := &gce.InstanceConf{
|
||||
Name: c.instName,
|
||||
Project: c.project,
|
||||
Machine: c.machine,
|
||||
Zone: zone,
|
||||
Hostname: c.hostname,
|
||||
WIP: c.wip,
|
||||
}
|
||||
|
||||
log.Printf("Creating instance %s (in project %s) in zone %s ...", c.instName, c.project, zone)
|
||||
depl := &gce.Deployer{
|
||||
Client: hc,
|
||||
Conf: instConf,
|
||||
Logger: log.New(cmdmain.Stderr, "", log.Flags()),
|
||||
}
|
||||
inst, err := depl.Create(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Instance created; starting up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -70,44 +70,6 @@ or
|
|||
`CAMLI_FORCE_OSARCH` (bool)
|
||||
Used by make.go to force building an unrecommended OS/ARCH pair.
|
||||
|
||||
`CAMLI_GCE_\*`
|
||||
: Variables prefixed with `CAMLI_GCE_` concern the Google Compute Engine deploy
|
||||
handler in [pkg/deploy/gce](/pkg/deploy/gce), which is only used by camweb to
|
||||
launch Perkeep on Google Compute Engine. They do not affect Perkeep's
|
||||
behaviour.
|
||||
|
||||
`CAMLI_GCE_CLIENTID` (string)
|
||||
: See `CAMLI_GCE_*` first. This string is used by gce.DeployHandler as the
|
||||
application's OAuth Client ID. If blank, camweb does not enable the Google
|
||||
Compute Engine launcher.
|
||||
|
||||
`CAMLI_GCE_CLIENTSECRET` (string)
|
||||
: See `CAMLI_GCE_*` first. Used by gce.DeployHandler as the application's OAuth
|
||||
Client Secret. If blank, gce.NewDeployHandler returns an error, and camweb
|
||||
fails to start if the Google Compute Engine launcher was enabled.
|
||||
|
||||
`CAMLI_GCE_DATA` (string)
|
||||
: See `CAMLI_GCE_*` first. Path to the directory where gce.DeployHandler stores
|
||||
the instances configuration and state. If blank, the "camli-gce-data" default
|
||||
is used instead.
|
||||
|
||||
`CAMLI_GCE_PROJECT` (string)
|
||||
: See `CAMLI_GCE_*` first. ID of the Google Project that provides the above
|
||||
client ID and secret. It is used when we query for the list of all the
|
||||
existing zones, since such a query requires a project ID. If blank, a
|
||||
hard-coded list of zones is used instead.
|
||||
|
||||
`CAMLI_GCE_SERVICE_ACCOUNT` (string)
|
||||
: See `CAMLI_GCE_*` first. Path to a Google service account JSON file. This
|
||||
account should have at least compute.readonly permissions on the Google
|
||||
Project with ID CAMLI_GCE_PROJECT. It is used to authenticate when querying
|
||||
for the list of all the existing zones. If blank, a hard-coded list of zones
|
||||
is used instead.
|
||||
|
||||
`CAMLI_GCE_XSRFKEY` (string)
|
||||
: See `CAMLI_GCE_*` first. Used by gce.DeployHandler as the XSRF protection key.
|
||||
If blank, gce.NewDeployHandler generates a new random key instead.
|
||||
|
||||
`CAMLI_GOPHERJS_GOROOT` (string)
|
||||
: As gopherjs does not build with go tip, when make.go is run with go devel,
|
||||
CAMLI_GOPHERJS_GOROOT should be set to a Go 1.10 root so that gopherjs can be
|
||||
|
|
|
|||
1
make.go
1
make.go
|
|
@ -104,7 +104,6 @@ func main() {
|
|||
"perkeep.org/cmd/pk-get",
|
||||
"perkeep.org/cmd/pk-put",
|
||||
"perkeep.org/cmd/pk",
|
||||
"perkeep.org/cmd/pk-deploy",
|
||||
"perkeep.org/server/perkeepd",
|
||||
"perkeep.org/app/hello",
|
||||
"perkeep.org/app/publisher",
|
||||
|
|
|
|||
|
|
@ -1,19 +0,0 @@
|
|||
#cloud-config
|
||||
|
||||
coreos:
|
||||
units:
|
||||
- name: hello-docker.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Camlistore
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/bin/docker run -p 80:80 camlistore/hello
|
||||
RestartSec=500ms
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,917 +0,0 @@
|
|||
/*
|
||||
Copyright 2014 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package gce provides tools to deploy Perkeep on Google Compute Engine.
|
||||
package gce // import "perkeep.org/pkg/deploy/gce"
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"perkeep.org/internal/osutil"
|
||||
|
||||
"cloud.google.com/go/logging"
|
||||
"go4.org/cloud/google/gceutil"
|
||||
"go4.org/syncutil"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
|
||||
compute "google.golang.org/api/compute/v1"
|
||||
"google.golang.org/api/googleapi"
|
||||
servicemanagement "google.golang.org/api/servicemanagement/v1"
|
||||
storage "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultInstanceName = "camlistore-server"
|
||||
DefaultMachineType = "g1-small"
|
||||
DefaultRegion = "us-central1"
|
||||
|
||||
projectsAPIURL = "https://www.googleapis.com/compute/v1/projects/"
|
||||
|
||||
fallbackZone = "us-central1-a"
|
||||
|
||||
camliUsername = "camlistore" // directly set in compute metadata, so not user settable.
|
||||
|
||||
configDir = "config"
|
||||
|
||||
ConsoleURL = "https://console.developers.google.com"
|
||||
helpDeleteInstance = `To delete an existing Compute Engine instance: in your project console, navigate to "Compute", "Compute Engine", and "VM instances". Select your instance and click "Delete".`
|
||||
)
|
||||
|
||||
var (
|
||||
// Verbose enables more info to be printed.
|
||||
Verbose bool
|
||||
)
|
||||
|
||||
// certFilename returns the HTTPS certificate file name
|
||||
func certFilename() string {
|
||||
return filepath.Base(osutil.DefaultTLSCert())
|
||||
}
|
||||
|
||||
// keyFilename returns the HTTPS key name
|
||||
func keyFilename() string {
|
||||
return filepath.Base(osutil.DefaultTLSKey())
|
||||
}
|
||||
|
||||
// NewOAuthConfig returns an OAuth configuration template.
|
||||
func NewOAuthConfig(clientID, clientSecret string) *oauth2.Config {
|
||||
return &oauth2.Config{
|
||||
Scopes: []string{
|
||||
logging.WriteScope,
|
||||
compute.DevstorageFullControlScope,
|
||||
compute.ComputeScope,
|
||||
cloudresourcemanager.CloudPlatformScope,
|
||||
servicemanagement.CloudPlatformScope,
|
||||
"https://www.googleapis.com/auth/sqlservice",
|
||||
"https://www.googleapis.com/auth/sqlservice.admin",
|
||||
},
|
||||
Endpoint: google.Endpoint,
|
||||
ClientID: clientID,
|
||||
ClientSecret: clientSecret,
|
||||
}
|
||||
}
|
||||
|
||||
// InstanceConf is the configuration for the Google Compute Engine instance that will be deployed.
|
||||
type InstanceConf struct {
|
||||
Name string // Name given to the virtual machine instance.
|
||||
Project string // Google project ID where the instance is created.
|
||||
CreateProject bool // CreateProject defines whether to first create project.
|
||||
Machine string // Machine type.
|
||||
Zone string // GCE zone; see https://cloud.google.com/compute/docs/zones
|
||||
Hostname string // Fully qualified domain name.
|
||||
|
||||
configDir string // bucketBase() + "/config"
|
||||
blobDir string // bucketBase() + "/blobs"
|
||||
|
||||
Ctime time.Time // Timestamp for this configuration.
|
||||
|
||||
WIP bool // Whether to use the perkeepd-WORKINPROGRESS.tar.gz tarball instead of the "production" one
|
||||
}
|
||||
|
||||
// Deployer creates and starts an instance such as defined in Conf.
|
||||
type Deployer struct {
|
||||
// Client is an OAuth2 client, authenticated for working with
|
||||
// the user's Google Cloud resources.
|
||||
Client *http.Client
|
||||
|
||||
Conf *InstanceConf
|
||||
|
||||
*log.Logger // Cannot be nil.
|
||||
}
|
||||
|
||||
// Get returns the Instance corresponding to the Project, Zone, and Name defined in the
|
||||
// Deployer's Conf.
|
||||
func (d *Deployer) Get() (*compute.Instance, error) {
|
||||
computeService, err := compute.New(d.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do()
|
||||
}
|
||||
|
||||
type instanceExistsError struct {
|
||||
project string
|
||||
zone string
|
||||
name string
|
||||
}
|
||||
|
||||
func (e instanceExistsError) Error() string {
|
||||
if e.project == "" {
|
||||
panic("instanceExistsErr has no project")
|
||||
}
|
||||
msg := "some instance(s) already exist as (" + e.project
|
||||
if e.zone != "" {
|
||||
msg += ", " + e.zone
|
||||
}
|
||||
if e.name != "" {
|
||||
msg += ", " + e.name
|
||||
}
|
||||
msg += "), you need to delete them first."
|
||||
return msg
|
||||
}
|
||||
|
||||
// projectHasInstance checks for all the possible zones if there's already an instance for the project.
|
||||
// It returns the name of the zone at the first instance it finds, if any.
|
||||
func (d *Deployer) projectHasInstance() (zone string, err error) {
|
||||
s, err := compute.New(d.Client)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// TODO(mpl): make use of the handler's cached zones.
|
||||
zl, err := compute.NewZonesService(s).List(d.Conf.Project).Do()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not get a list of zones: %v", err)
|
||||
}
|
||||
computeService, _ := compute.New(d.Client)
|
||||
var zoneOnce sync.Once
|
||||
var grp syncutil.Group
|
||||
errc := make(chan error, 1)
|
||||
zonec := make(chan string, 1)
|
||||
timeout := time.NewTimer(30 * time.Second)
|
||||
defer timeout.Stop()
|
||||
for _, z := range zl.Items {
|
||||
z := z
|
||||
grp.Go(func() error {
|
||||
list, err := computeService.Instances.List(d.Conf.Project, z.Name).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not list existing instances: %v", err)
|
||||
}
|
||||
if len(list.Items) > 0 {
|
||||
zoneOnce.Do(func() {
|
||||
zonec <- z.Name
|
||||
})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
go func() {
|
||||
errc <- grp.Err()
|
||||
}()
|
||||
// We block until either an instance was found in a zone, or all the instance
|
||||
// listing is done. Or we timed-out.
|
||||
select {
|
||||
case err = <-errc:
|
||||
return "", err
|
||||
case zone = <-zonec:
|
||||
// We voluntarily ignore any listing error if we found at least one instance
|
||||
// because that's what we primarily want to report about.
|
||||
return zone, nil
|
||||
case <-timeout.C:
|
||||
return "", errors.New("timed out")
|
||||
}
|
||||
}
|
||||
|
||||
type projectIDError struct {
|
||||
id string
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e projectIDError) Error() string {
|
||||
if e.id == "" {
|
||||
panic("projectIDError without an id")
|
||||
}
|
||||
if e.cause != nil {
|
||||
return fmt.Sprintf("project ID error for %v: %v", e.id, e.cause)
|
||||
}
|
||||
return fmt.Sprintf("project ID error for %v", e.id)
|
||||
}
|
||||
|
||||
// CreateProject creates a new Google Cloud Project. It returns the project ID,
|
||||
// which is a random number in (0,1e10), prefixed with "camlistore-launcher-".
|
||||
func (d *Deployer) CreateProject(ctx context.Context) (string, error) {
|
||||
s, err := cloudresourcemanager.New(d.Client)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Allow for a few retries, when we generated an already taken project ID
|
||||
creationTimeout := time.Now().Add(time.Minute)
|
||||
var projectID, projectName string
|
||||
for {
|
||||
if d.Conf.Project != "" {
|
||||
projectID = d.Conf.Project
|
||||
projectName = projectID
|
||||
} else {
|
||||
projectID = genRandomProjectID()
|
||||
projectName = strings.Replace(projectID, "camlistore-launcher-", "Camlistore ", 1)
|
||||
}
|
||||
project := cloudresourcemanager.Project{
|
||||
Name: projectName,
|
||||
ProjectId: projectID,
|
||||
}
|
||||
if time.Now().After(creationTimeout) {
|
||||
return "", errors.New("timeout while trying to create project")
|
||||
}
|
||||
d.Printf("Trying to create project %v", projectID)
|
||||
op, err := cloudresourcemanager.NewProjectsService(s).Create(&project).Do()
|
||||
if err != nil {
|
||||
gerr, ok := err.(*googleapi.Error)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("could not create project: %v", err)
|
||||
}
|
||||
if gerr.Code != 409 {
|
||||
return "", fmt.Errorf("could not create project: %v", gerr.Message)
|
||||
}
|
||||
// it's ok using time.Sleep, and no backoff, as the
|
||||
// timeout is pretty short, and we only retry on a 409.
|
||||
time.Sleep(time.Second)
|
||||
// retry if project ID already exists.
|
||||
d.Printf("Project %v already exists, will retry with a new project ID", project.ProjectId)
|
||||
continue
|
||||
}
|
||||
|
||||
// as per
|
||||
// https://cloud.google.com/resource-manager/reference/rest/v1/projects/create
|
||||
// recommendation
|
||||
timeout := time.Now().Add(30 * time.Second)
|
||||
backoff := time.Second
|
||||
startPolling := 5 * time.Second
|
||||
time.Sleep(startPolling)
|
||||
for {
|
||||
if time.Now().After(timeout) {
|
||||
return "", fmt.Errorf("timeout while trying to check project creation")
|
||||
}
|
||||
if !op.Done {
|
||||
// it's ok to just sleep, as our timeout is pretty short.
|
||||
time.Sleep(backoff)
|
||||
backoff *= 2
|
||||
op, err = cloudresourcemanager.NewOperationsService(s).Get(op.Name).Do()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not check project creation status: %v", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if op.Error != nil {
|
||||
// TODO(mpl): ghetto logging for now. detect at least the quota errors.
|
||||
var details string
|
||||
for _, v := range op.Error.Details {
|
||||
details += string(v)
|
||||
}
|
||||
return "", fmt.Errorf("could not create project: %v, %v", op.Error.Message, details)
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
d.Printf("Success creating project %v", projectID)
|
||||
return projectID, nil
|
||||
}
|
||||
|
||||
func genRandomProjectID() string {
|
||||
// we're allowed up to 30 characters, and we already consume 20 with
|
||||
// "camlistore-launcher-", so we've got 10 chars left of randomness. Should
|
||||
// be plenty enough I think.
|
||||
var n *big.Int
|
||||
var err error
|
||||
zero := big.NewInt(0)
|
||||
for {
|
||||
n, err = rand.Int(rand.Reader, big.NewInt(1e10)) // max is 1e10 - 1
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("rand.Int error: %v", err))
|
||||
}
|
||||
if n.Cmp(zero) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("camlistore-launcher-%d", n)
|
||||
}
|
||||
|
||||
func (d *Deployer) enableAPIs() error {
|
||||
// TODO(mpl): For now we're lucky enough that servicemanagement seems to
|
||||
// work even when the Service Management API hasn't been enabled for the
|
||||
// project. If/when it does not anymore, then we should use serviceuser
|
||||
// instead. http://stackoverflow.com/a/43503392/1775619
|
||||
s, err := servicemanagement.New(d.Client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
list, err := servicemanagement.NewServicesService(s).List().ConsumerId("project:" + d.Conf.Project).Do()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
requiredServices := map[string]string{
|
||||
"storage-component.googleapis.com": "Google Cloud Storage",
|
||||
"storage-api.googleapis.com": "Google Cloud Storage JSON",
|
||||
"logging.googleapis.com": "Stackdriver Logging",
|
||||
"compute.googleapis.com": "Google Compute Engine",
|
||||
"geocoding-backend.googleapis.com": "Google Maps Geocoding",
|
||||
}
|
||||
enabledServices := make(map[string]bool)
|
||||
for _, v := range list.Services {
|
||||
enabledServices[v.ServiceName] = true
|
||||
}
|
||||
errc := make(chan error, len(requiredServices))
|
||||
var wg sync.WaitGroup
|
||||
for k, v := range requiredServices {
|
||||
if _, ok := enabledServices[k]; ok {
|
||||
continue
|
||||
}
|
||||
d.Printf("%v API not enabled; enabling it with Service Management", v)
|
||||
op, err := servicemanagement.NewServicesService(s).
|
||||
Enable(k, &servicemanagement.EnableServiceRequest{ConsumerId: "project:" + d.Conf.Project}).Do()
|
||||
if err != nil {
|
||||
gerr, ok := err.(*googleapi.Error)
|
||||
if !ok {
|
||||
return err
|
||||
}
|
||||
if gerr.Code != http.StatusBadRequest {
|
||||
return err
|
||||
}
|
||||
for _, v := range gerr.Errors {
|
||||
if v.Reason == "failedPrecondition" && strings.Contains(v.Message, "billing-enabled") {
|
||||
return fmt.Errorf("you need to enabling billing for project %v: https://console.cloud.google.com/billing/?project=%v", d.Conf.Project, d.Conf.Project)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(service, opName string) {
|
||||
defer wg.Done()
|
||||
timeout := time.Now().Add(2 * time.Minute)
|
||||
backoff := time.Second
|
||||
startPolling := 5 * time.Second
|
||||
time.Sleep(startPolling)
|
||||
for {
|
||||
if time.Now().After(timeout) {
|
||||
errc <- fmt.Errorf("timeout while trying to enable service: %v", service)
|
||||
return
|
||||
}
|
||||
op, err := servicemanagement.NewOperationsService(s).Get(opName).Do()
|
||||
if err != nil {
|
||||
errc <- fmt.Errorf("could not check service enabling status: %v", err)
|
||||
return
|
||||
}
|
||||
if !op.Done {
|
||||
// it's ok to just sleep, as our timeout is pretty short.
|
||||
time.Sleep(backoff)
|
||||
backoff *= 2
|
||||
continue
|
||||
}
|
||||
if op.Error != nil {
|
||||
errc <- fmt.Errorf("could not enable service %v: %v", service, op.Error.Message)
|
||||
return
|
||||
}
|
||||
d.Printf("%v service successfully enabled", service)
|
||||
return
|
||||
}
|
||||
}(k, op.Name)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errc)
|
||||
for err := range errc {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Deployer) checkProjectID() error {
|
||||
// TODO(mpl): cache the computeService in Deployer, instead of recreating a new one everytime?
|
||||
s, err := compute.New(d.Client)
|
||||
if err != nil {
|
||||
return projectIDError{
|
||||
id: d.Conf.Project,
|
||||
cause: err,
|
||||
}
|
||||
}
|
||||
project, err := compute.NewProjectsService(s).Get(d.Conf.Project).Do()
|
||||
if err != nil {
|
||||
return projectIDError{
|
||||
id: d.Conf.Project,
|
||||
cause: err,
|
||||
}
|
||||
}
|
||||
if project.Name != d.Conf.Project {
|
||||
return projectIDError{
|
||||
id: d.Conf.Project,
|
||||
cause: fmt.Errorf("project IDs do not match: got %q, wanted %q", project.Name, d.Conf.Project),
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errAttrNotFound = errors.New("attribute not found")
|
||||
|
||||
// getInstanceAttribute returns the value for attr in the custom metadata of the
|
||||
// instance. It returns errAttrNotFound is such a metadata attributed does not
|
||||
// exist.
|
||||
func (d *Deployer) getInstanceAttribute(attr string) (string, error) {
|
||||
s, err := compute.New(d.Client)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error getting compute service: %v", err)
|
||||
}
|
||||
inst, err := compute.NewInstancesService(s).Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error getting instance: %v", err)
|
||||
}
|
||||
for _, v := range inst.Metadata.Items {
|
||||
if v.Key == attr {
|
||||
return *(v.Value), nil
|
||||
}
|
||||
}
|
||||
return "", errAttrNotFound
|
||||
}
|
||||
|
||||
// Create sets up and starts a Google Compute Engine instance as defined in d.Conf. It
|
||||
// creates the necessary Google Storage buckets beforehand.
|
||||
func (d *Deployer) Create(ctx context.Context) (*compute.Instance, error) {
|
||||
if err := d.enableAPIs(); err != nil {
|
||||
return nil, projectIDError{
|
||||
id: d.Conf.Project,
|
||||
cause: err,
|
||||
}
|
||||
}
|
||||
if err := d.checkProjectID(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
computeService, _ := compute.New(d.Client)
|
||||
storageService, _ := storage.New(d.Client)
|
||||
|
||||
fwc := make(chan error, 1)
|
||||
go func() {
|
||||
fwc <- d.setFirewall(ctx, computeService)
|
||||
}()
|
||||
|
||||
config := cloudConfig(d.Conf)
|
||||
const maxCloudConfig = 32 << 10 // per compute API docs
|
||||
if len(config) > maxCloudConfig {
|
||||
return nil, fmt.Errorf("cloud config length of %d bytes is over %d byte limit", len(config), maxCloudConfig)
|
||||
}
|
||||
|
||||
if zone, err := d.projectHasInstance(); zone != "" {
|
||||
return nil, instanceExistsError{
|
||||
project: d.Conf.Project,
|
||||
zone: zone,
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("could not scan project for existing instances: %v", err)
|
||||
}
|
||||
|
||||
if err := d.setBuckets(ctx, storageService); err != nil {
|
||||
return nil, fmt.Errorf("could not create buckets: %v", err)
|
||||
}
|
||||
|
||||
if err := d.createInstance(ctx, computeService); err != nil {
|
||||
return nil, fmt.Errorf("could not create compute instance: %v", err)
|
||||
}
|
||||
|
||||
inst, err := computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting instance after creation: %v", err)
|
||||
}
|
||||
if Verbose {
|
||||
ij, _ := json.MarshalIndent(inst, "", " ")
|
||||
d.Printf("Instance: %s", ij)
|
||||
}
|
||||
|
||||
if err = <-fwc; err != nil {
|
||||
return nil, fmt.Errorf("could not create firewall rules: %v", err)
|
||||
}
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
func randPassword() string {
|
||||
buf := make([]byte, 5)
|
||||
if n, err := rand.Read(buf); err != nil || n != len(buf) {
|
||||
log.Fatalf("crypto/rand.Read = %v, %v", n, err)
|
||||
}
|
||||
return fmt.Sprintf("%x", buf)
|
||||
}
|
||||
|
||||
// LooksLikeRegion reports whether s looks like a GCE region.
|
||||
func LooksLikeRegion(s string) bool {
|
||||
return strings.Count(s, "-") == 1
|
||||
}
|
||||
|
||||
// createInstance starts the creation of the Compute Engine instance and waits for the
|
||||
// result of the creation operation. It should be called after setBuckets and setupHTTPS.
|
||||
func (d *Deployer) createInstance(ctx context.Context, computeService *compute.Service) error {
|
||||
coreosImgURL, err := gceutil.CoreOSImageURL(d.Client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error looking up latest CoreOS stable image: %v", err)
|
||||
}
|
||||
prefix := projectsAPIURL + d.Conf.Project
|
||||
machType := prefix + "/zones/" + d.Conf.Zone + "/machineTypes/" + d.Conf.Machine
|
||||
config := cloudConfig(d.Conf)
|
||||
instance := &compute.Instance{
|
||||
Name: d.Conf.Name,
|
||||
Description: "Camlistore server",
|
||||
MachineType: machType,
|
||||
Disks: []*compute.AttachedDisk{
|
||||
{
|
||||
AutoDelete: true,
|
||||
Boot: true,
|
||||
Type: "PERSISTENT",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
DiskName: d.Conf.Name + "-coreos-stateless-pd",
|
||||
SourceImage: coreosImgURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
Tags: &compute.Tags{
|
||||
Items: []string{"http-server", "https-server"},
|
||||
},
|
||||
Metadata: &compute.Metadata{
|
||||
Items: []*compute.MetadataItems{
|
||||
{
|
||||
Key: "camlistore-username",
|
||||
Value: googleapi.String(camliUsername),
|
||||
},
|
||||
{
|
||||
Key: "camlistore-password",
|
||||
Value: googleapi.String(randPassword()),
|
||||
},
|
||||
{
|
||||
Key: "camlistore-blob-dir",
|
||||
Value: googleapi.String("gs://" + d.Conf.blobDir),
|
||||
},
|
||||
{
|
||||
Key: "camlistore-config-dir",
|
||||
Value: googleapi.String("gs://" + d.Conf.configDir),
|
||||
},
|
||||
{
|
||||
Key: "perkeep-config-version",
|
||||
// perkeep-config-version being defined requires this launcher to deploy Perkeep
|
||||
// at rev >= 7eda9fd5027fda88166d6c03b6490cffbf2de5fb , so that any newly deployed Perkeep
|
||||
// knows it can use the new configuration without DBNames.
|
||||
// TODO(mpl): but how do we enforce it, or at least make it more obvious/documented?
|
||||
// With a flag defaulting to "1" maybe?
|
||||
Value: googleapi.String("1"),
|
||||
},
|
||||
{
|
||||
Key: "user-data",
|
||||
Value: googleapi.String(config),
|
||||
},
|
||||
},
|
||||
},
|
||||
NetworkInterfaces: []*compute.NetworkInterface{
|
||||
{
|
||||
AccessConfigs: []*compute.AccessConfig{
|
||||
{
|
||||
Type: "ONE_TO_ONE_NAT",
|
||||
Name: "External NAT",
|
||||
},
|
||||
},
|
||||
Network: prefix + "/global/networks/default",
|
||||
},
|
||||
},
|
||||
ServiceAccounts: []*compute.ServiceAccount{
|
||||
{
|
||||
Email: "default",
|
||||
Scopes: []string{
|
||||
logging.WriteScope,
|
||||
compute.DevstorageFullControlScope,
|
||||
compute.ComputeScope,
|
||||
"https://www.googleapis.com/auth/sqlservice",
|
||||
"https://www.googleapis.com/auth/sqlservice.admin",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if d.Conf.Hostname != "" {
|
||||
instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
|
||||
Key: "camlistore-hostname",
|
||||
Value: googleapi.String(d.Conf.Hostname),
|
||||
})
|
||||
}
|
||||
const localMySQL = false // later
|
||||
if localMySQL {
|
||||
instance.Disks = append(instance.Disks, &compute.AttachedDisk{
|
||||
AutoDelete: false,
|
||||
Boot: false,
|
||||
Type: "PERSISTENT",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
DiskName: "camlistore-mysql-index-pd",
|
||||
DiskSizeGb: 4,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if Verbose {
|
||||
d.Print("Creating instance...")
|
||||
}
|
||||
op, err := computeService.Instances.Insert(d.Conf.Project, d.Conf.Zone, instance).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create instance: %v", err)
|
||||
}
|
||||
opName := op.Name
|
||||
if Verbose {
|
||||
d.Printf("Created. Waiting on operation %v", opName)
|
||||
}
|
||||
OpLoop:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
op, err := computeService.ZoneOperations.Get(d.Conf.Project, d.Conf.Zone, opName).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get op %s: %v", opName, err)
|
||||
}
|
||||
switch op.Status {
|
||||
case "PENDING", "RUNNING":
|
||||
if Verbose {
|
||||
d.Printf("Waiting on operation %v", opName)
|
||||
}
|
||||
continue
|
||||
case "DONE":
|
||||
if op.Error != nil {
|
||||
for _, operr := range op.Error.Errors {
|
||||
d.Printf("Error: %+v", operr)
|
||||
}
|
||||
return fmt.Errorf("failed to start")
|
||||
}
|
||||
if Verbose {
|
||||
d.Printf("Success. %+v", op)
|
||||
}
|
||||
break OpLoop
|
||||
default:
|
||||
return fmt.Errorf("unknown status %q: %+v", op.Status, op)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloudConfig(conf *InstanceConf) string {
|
||||
config := strings.Replace(baseInstanceConfig, "INNODB_BUFFER_POOL_SIZE=NNN", "INNODB_BUFFER_POOL_SIZE="+strconv.Itoa(innodbBufferPoolSize(conf.Machine)), -1)
|
||||
perkeepdTarball := "https://storage.googleapis.com/camlistore-release/docker/"
|
||||
if conf.WIP {
|
||||
perkeepdTarball += "perkeepd-WORKINPROGRESS.tar.gz"
|
||||
} else {
|
||||
perkeepdTarball += "perkeepd.tar.gz"
|
||||
}
|
||||
config = strings.Replace(config, "CAMLISTORED_TARBALL", perkeepdTarball, 1)
|
||||
return config
|
||||
}
|
||||
|
||||
// setBuckets defines the buckets needed by the instance and creates them.
|
||||
func (d *Deployer) setBuckets(ctx context.Context, storageService *storage.Service) error {
|
||||
projBucket := d.Conf.Project + "-camlistore"
|
||||
|
||||
needBucket := map[string]bool{
|
||||
projBucket: true,
|
||||
}
|
||||
|
||||
buckets, err := storageService.Buckets.List(d.Conf.Project).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error listing buckets: %v", err)
|
||||
}
|
||||
for _, it := range buckets.Items {
|
||||
delete(needBucket, it.Name)
|
||||
}
|
||||
if len(needBucket) > 0 {
|
||||
if Verbose {
|
||||
d.Printf("Need to create buckets: %v", needBucket)
|
||||
}
|
||||
var waitBucket sync.WaitGroup
|
||||
var bucketErr error
|
||||
for name := range needBucket {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
name := name
|
||||
waitBucket.Add(1)
|
||||
go func() {
|
||||
defer waitBucket.Done()
|
||||
if Verbose {
|
||||
d.Printf("Creating bucket %s", name)
|
||||
}
|
||||
b, err := storageService.Buckets.Insert(d.Conf.Project, &storage.Bucket{
|
||||
Id: name,
|
||||
Name: name,
|
||||
}).Do()
|
||||
if err != nil && bucketErr == nil {
|
||||
bucketErr = fmt.Errorf("error creating bucket %s: %v", name, err)
|
||||
return
|
||||
}
|
||||
if Verbose {
|
||||
d.Printf("Created bucket %s: %+v", name, b)
|
||||
}
|
||||
}()
|
||||
}
|
||||
waitBucket.Wait()
|
||||
if bucketErr != nil {
|
||||
return bucketErr
|
||||
}
|
||||
}
|
||||
|
||||
d.Conf.configDir = path.Join(projBucket, configDir)
|
||||
d.Conf.blobDir = path.Join(projBucket, "blobs")
|
||||
return nil
|
||||
}
|
||||
|
||||
// setFirewall adds the firewall rules needed for ports 80 & 433 to the default network.
|
||||
func (d *Deployer) setFirewall(ctx context.Context, computeService *compute.Service) error {
|
||||
defaultNet, err := computeService.Networks.Get(d.Conf.Project, "default").Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting default network: %v", err)
|
||||
}
|
||||
|
||||
needRules := map[string]compute.Firewall{
|
||||
"default-allow-http": {
|
||||
Name: "default-allow-http",
|
||||
SourceRanges: []string{"0.0.0.0/0"},
|
||||
SourceTags: []string{"http-server"},
|
||||
Allowed: []*compute.FirewallAllowed{{IPProtocol: "tcp", Ports: []string{"80"}}},
|
||||
Network: defaultNet.SelfLink,
|
||||
},
|
||||
"default-allow-https": {
|
||||
Name: "default-allow-https",
|
||||
SourceRanges: []string{"0.0.0.0/0"},
|
||||
SourceTags: []string{"https-server"},
|
||||
Allowed: []*compute.FirewallAllowed{{IPProtocol: "tcp", Ports: []string{"443"}}},
|
||||
Network: defaultNet.SelfLink,
|
||||
},
|
||||
}
|
||||
|
||||
rules, err := computeService.Firewalls.List(d.Conf.Project).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error listing rules: %v", err)
|
||||
}
|
||||
for _, it := range rules.Items {
|
||||
delete(needRules, it.Name)
|
||||
}
|
||||
if len(needRules) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if Verbose {
|
||||
d.Printf("Need to create rules: %v", needRules)
|
||||
}
|
||||
var wg syncutil.Group
|
||||
for name, rule := range needRules {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
name, rule := name, rule
|
||||
wg.Go(func() error {
|
||||
if Verbose {
|
||||
d.Printf("Creating rule %s", name)
|
||||
}
|
||||
r, err := computeService.Firewalls.Insert(d.Conf.Project, &rule).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating rule %s: %v", name, err)
|
||||
}
|
||||
if Verbose {
|
||||
d.Printf("Created rule %s: %+v", name, r)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return wg.Err()
|
||||
}
|
||||
|
||||
// returns the MySQL InnoDB buffer pool size (in bytes) as a function
|
||||
// of the GCE machine type.
|
||||
func innodbBufferPoolSize(machine string) int {
|
||||
// Totally arbitrary. We don't need much here because
|
||||
// perkeepd slurps this all into its RAM on start-up
|
||||
// anyway. So this is all prety overkill and more than the
|
||||
// 8MB default.
|
||||
switch machine {
|
||||
case "f1-micro":
|
||||
return 32 << 20
|
||||
case "g1-small":
|
||||
return 64 << 20
|
||||
default:
|
||||
return 128 << 20
|
||||
}
|
||||
}
|
||||
|
||||
const baseInstanceConfig = `#cloud-config
|
||||
write_files:
|
||||
- path: /var/lib/camlistore/tmp/README
|
||||
permissions: 0644
|
||||
content: |
|
||||
This is the Camlistore /tmp directory.
|
||||
- path: /var/lib/camlistore/mysql/README
|
||||
permissions: 0644
|
||||
content: |
|
||||
This is the Camlistore MySQL data directory.
|
||||
coreos:
|
||||
units:
|
||||
- name: cam-journal-gatewayd.service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Journal Gateway Service
|
||||
Requires=cam-journal-gatewayd.socket
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/lib/systemd/systemd-journal-gatewayd
|
||||
User=systemd-journal-gateway
|
||||
Group=systemd-journal-gateway
|
||||
SupplementaryGroups=systemd-journal
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
PrivateNetwork=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
|
||||
[Install]
|
||||
Also=cam-journal-gatewayd.socket
|
||||
- name: cam-journal-gatewayd.socket
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Journal Gateway Service Socket
|
||||
|
||||
[Socket]
|
||||
ListenStream=/run/camjournald.sock
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
- name: mysql.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=MySQL
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/bin/bash -c '/usr/bin/curl https://storage.googleapis.com/camlistore-release/docker/systemd-docker.tar.gz | /bin/gunzip -c | /usr/bin/docker load'
|
||||
ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin camlistore/systemd-docker
|
||||
ExecStart=/opt/bin/systemd-docker run --rm --name %n -v /var/lib/camlistore/mysql:/mysql -e INNODB_BUFFER_POOL_SIZE=NNN camlistore/mysql
|
||||
RestartSec=1s
|
||||
Restart=always
|
||||
Type=notify
|
||||
NotifyAccess=all
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: perkeepd.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Camlistore
|
||||
After=docker.service mysql.service
|
||||
Requires=docker.service mysql.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin camlistore/systemd-docker
|
||||
ExecStartPre=/bin/bash -c '/usr/bin/curl CAMLISTORED_TARBALL | /bin/gunzip -c | /usr/bin/docker load'
|
||||
ExecStart=/opt/bin/systemd-docker run --rm -p 80:80 -p 443:443 --name %n -v /run/camjournald.sock:/run/camjournald.sock -v /var/lib/camlistore/tmp:/tmp --link=mysql.service:mysqldb perkeep/server
|
||||
RestartSec=1s
|
||||
Restart=always
|
||||
Type=notify
|
||||
NotifyAccess=all
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,31 +0,0 @@
|
|||
non-core dev:
|
||||
gcutil --service_version="v1" --project="camanaged" addinstance "camlistore" --zone="us-central1-b" --machine_type="n1-standard-1" --network="default" --external_ip_address="107.178.214.163" --metadata="cam-key-1:cam-value-1" --metadata="cam-key-2:cam-value-2" --metadata="sshKeys:bradfitz:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw6Dk3iskKylP2zginCOAzIunMA38vGL9b/i18UG/Iuq+jKczZXB/1dlcZGSOs3+LtGh/C341TXTioydxTw+ux1AbmUk4c6L404skl85XFOys/GLxA4sHxBSb5we0Q57yohSgeZNlQd+Scmu5v7WC0N7I3hOK0lJgtxRNyC2nncGC0UOm+IGPTWcqPJERTauH/OhoAddWQehf1ugxTJYFU9atl3Op/mDXfyGBSLweWAQ84fhVKRZnl4i9Yhk1b357Q8cVKH6UQUADVamo7CQOsenzx99UL0thFRTSbuKALyf9e+SPwJrtIxZaX+skVSR+CzooRbypIamLbNXhfbxNz bradfitz@Bradleys-MacBook-Air.local" --service_account_scopes="https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/compute.readonly,https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/sqlservice,https://www.googleapis.com/auth/sqlservice.admin,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/datastore,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/compute,https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/taskqueue,https://www.googleapis.com/auth/bigquery,https://www.googleapis.com/auth/sqlservice,https://www.googleapis.com/auth/datastore" --tags="tag,tag2,http-server,https-server" --persistent_boot_disk="true" --auto_delete_boot_disk="false" --image=projects/debian-cloud/global/images/backports-debian-7-wheezy-v20140718
|
||||
|
||||
$ curl -H "Metadata-Flavor:Google" http://metadata/computeMetadata/v1/instance/service-accounts/default/scopes
|
||||
https://www.googleapis.com/auth/bigquery
|
||||
https://www.googleapis.com/auth/cloud-platform
|
||||
https://www.googleapis.com/auth/compute
|
||||
https://www.googleapis.com/auth/compute.readonly
|
||||
https://www.googleapis.com/auth/datastore
|
||||
https://www.googleapis.com/auth/devstorage.full_control
|
||||
https://www.googleapis.com/auth/sqlservice
|
||||
https://www.googleapis.com/auth/sqlservice.admin
|
||||
https://www.googleapis.com/auth/taskqueue
|
||||
https://www.googleapis.com/auth/userinfo.email
|
||||
|
||||
gcutil --project=camanaged addinstance \
|
||||
--image=projects/coreos-cloud/global/images/coreos-alpha-394-0-0-v20140801 \
|
||||
--persistent_boot_disk \
|
||||
--zone=us-central1-a --machine_type=n1-standard-1 \
|
||||
--external_ip_address=107.178.208.16 \
|
||||
--auto_delete_boot_disk \
|
||||
--tags=http-server,https-server \
|
||||
--metadata_from_file=user-data:cloud-config.yaml core1
|
||||
|
||||
TODO:
|
||||
- allow config from /gcs/bucket/key; add pkg for os.Stat/os.Open wrappers checking
|
||||
prefix
|
||||
- use that package for:
|
||||
"httpsCert": "/home/bradfitz/keys/camlihouse/ssl.crt",
|
||||
"httpsKey": "/home/bradfitz/keys/camlihouse/ssl.key",
|
||||
"identitySecretRing": "/home/bradfitz/.config/perkeep/identity-secring.gpg",
|
||||
|
|
@ -85,7 +85,7 @@ func DefaultEnvConfig() (*Config, error) {
|
|||
|
||||
externalIP, _ := metadata.ExternalIP()
|
||||
hostName, _ := metadata.InstanceAttributeValue("camlistore-hostname")
|
||||
// If they specified a hostname (probably with pk-deploy), then:
|
||||
// If they specified a hostname (previously common with old pk-deploy), then:
|
||||
// if it looks like an FQDN, perkeepd is going to rely on Let's
|
||||
// Encrypt, else perkeepd is going to generate some self-signed for that
|
||||
// hostname.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
|
|
@ -40,26 +39,19 @@ import (
|
|||
txttemplate "text/template"
|
||||
"time"
|
||||
|
||||
"perkeep.org/internal/netutil"
|
||||
"perkeep.org/internal/osutil"
|
||||
"perkeep.org/pkg/buildinfo"
|
||||
"perkeep.org/pkg/deploy/gce"
|
||||
"perkeep.org/pkg/types/camtypes"
|
||||
|
||||
"cloud.google.com/go/compute/metadata"
|
||||
"cloud.google.com/go/datastore"
|
||||
"cloud.google.com/go/logging"
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/mailgun/mailgun-go"
|
||||
"github.com/russross/blackfriday"
|
||||
"go4.org/cloud/cloudlaunch"
|
||||
"go4.org/writerutil"
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
"golang.org/x/oauth2/google"
|
||||
cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
|
||||
compute "google.golang.org/api/compute/v1"
|
||||
"google.golang.org/api/option"
|
||||
storageapi "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -80,7 +72,6 @@ var (
|
|||
tlsKeyFile = flag.String("tlskey", "", "TLS private key file")
|
||||
alsoRun = flag.String("also_run", "", "[optiona] Path to run as a child process. (Used to run perkeep.org's ./scripts/run-blob-server)")
|
||||
devMode = flag.Bool("dev", false, "in dev mode")
|
||||
flagStaging = flag.Bool("staging", false, "Deploy to a test GCE instance. Requires -cloudlaunch=true")
|
||||
flagVersion = flag.Bool("version", false, "show version")
|
||||
|
||||
gceProjectID = flag.String("gce_project_id", "", "GCE project ID; required if not running on GCE and gce_log_name is specified.")
|
||||
|
|
@ -502,104 +493,6 @@ func runAsChild(res string) {
|
|||
}()
|
||||
}
|
||||
|
||||
func gceDeployHandlerConfig() (*gce.Config, error) {
|
||||
if inProd {
|
||||
return deployerCredsFromGCS()
|
||||
}
|
||||
clientId := os.Getenv("CAMLI_GCE_CLIENTID")
|
||||
if clientId != "" {
|
||||
return &gce.Config{
|
||||
ClientID: clientId,
|
||||
ClientSecret: os.Getenv("CAMLI_GCE_CLIENTSECRET"),
|
||||
Project: os.Getenv("CAMLI_GCE_PROJECT"),
|
||||
ServiceAccount: os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT"),
|
||||
DataDir: os.Getenv("CAMLI_GCE_DATA"),
|
||||
}, nil
|
||||
}
|
||||
configDir, err := osutil.PerkeepConfigDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configFile := filepath.Join(configDir, "launcher-config.json")
|
||||
data, err := os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading launcher-config.json (expected of type https://godoc.org/"+prodDomain+"/pkg/deploy/gce#Config): %v", err)
|
||||
}
|
||||
var config gce.Config
|
||||
if err := json.Unmarshal(data, &config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
var cloudLauncherEnabled = false
|
||||
|
||||
// gceDeployHandler returns an http.Handler for a GCE launcher,
|
||||
// configured to run at /prefix/ (the trailing slash can be omitted).
|
||||
// The launcher is not initialized if:
|
||||
// - in production, the launcher-config.json file is not found in the relevant bucket
|
||||
// - neither CAMLI_GCE_CLIENTID is set, nor launcher-config.json is found in the
|
||||
// camlistore server config dir.
|
||||
func gceDeployHandler(prefix string) (*gce.DeployHandler, error) {
|
||||
if !cloudLauncherEnabled {
|
||||
return nil, errors.New("The Perkeep Cloud Launcher is no longer available.")
|
||||
}
|
||||
var hostPort string
|
||||
var err error
|
||||
scheme := "https"
|
||||
if inProd {
|
||||
if inStaging {
|
||||
hostPort = stagingHostname + ":443"
|
||||
} else {
|
||||
hostPort = prodDomain + ":443"
|
||||
}
|
||||
} else {
|
||||
addr := *httpsAddr
|
||||
if *devMode && *httpsAddr == "" {
|
||||
addr = *httpAddr
|
||||
scheme = "http"
|
||||
}
|
||||
hostPort, err = netutil.ListenHostPort(addr)
|
||||
if err != nil {
|
||||
// the deploy handler needs to know its own
|
||||
// hostname or IP for the oauth2 callback.
|
||||
return nil, fmt.Errorf("invalid -https flag: %v", err)
|
||||
}
|
||||
}
|
||||
config, err := gceDeployHandlerConfig()
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
gceh, err := gce.NewDeployHandlerFromConfig(hostPort, prefix, config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("NewDeployHandlerFromConfig: %v", err)
|
||||
}
|
||||
|
||||
pageBytes, err := os.ReadFile(filepath.Join(*root, "tmpl", "page.html"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := gceh.AddTemplateTheme(string(pageBytes)); err != nil {
|
||||
return nil, fmt.Errorf("AddTemplateTheme: %v", err)
|
||||
}
|
||||
gceh.SetScheme(scheme)
|
||||
log.Printf("Starting Perkeep launcher on %s://%s%s", scheme, hostPort, prefix)
|
||||
return gceh, nil
|
||||
}
|
||||
|
||||
var launchConfig = &cloudlaunch.Config{
|
||||
Name: "camweb",
|
||||
BinaryBucket: prodBucket,
|
||||
GCEProjectID: "camlistore-website",
|
||||
Scopes: []string{
|
||||
storageapi.DevstorageFullControlScope,
|
||||
compute.ComputeScope,
|
||||
logging.WriteScope,
|
||||
datastore.ScopeDatastore,
|
||||
cloudresourcemanager.CloudPlatformScope,
|
||||
},
|
||||
}
|
||||
|
||||
func checkInProduction() bool {
|
||||
if !metadata.OnGCE() {
|
||||
return false
|
||||
|
|
@ -815,29 +708,6 @@ func projectID() string {
|
|||
return projID
|
||||
}
|
||||
|
||||
func initStaging() error {
|
||||
if *flagStaging {
|
||||
launchConfig.Name = stagingInstName
|
||||
return nil
|
||||
}
|
||||
// If we are the instance that has just been deployed, we can't rely on
|
||||
// *flagStaging, since there's no way to pass flags through launchConfig.
|
||||
// And we need to know if we're a staging instance, so we can set
|
||||
// launchConfig.Name properly before we get into restartLoop from
|
||||
// MaybeDeploy. So we use our own instance name as a hint.
|
||||
if !metadata.OnGCE() {
|
||||
return nil
|
||||
}
|
||||
instName, err := metadata.InstanceName()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Instance could not get its Instance Name: %v", err)
|
||||
}
|
||||
if instName == stagingInstName {
|
||||
launchConfig.Name = stagingInstName
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if *flagVersion {
|
||||
|
|
@ -845,10 +715,6 @@ func main() {
|
|||
buildinfo.Summary(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
|
||||
return
|
||||
}
|
||||
if err := initStaging(); err != nil {
|
||||
log.Fatalf("Error setting up staging: %v", err)
|
||||
}
|
||||
launchConfig.MaybeDeploy()
|
||||
setProdFlags()
|
||||
|
||||
if *root == "" {
|
||||
|
|
@ -904,16 +770,9 @@ func main() {
|
|||
})
|
||||
}
|
||||
|
||||
gceLauncher, err := gceDeployHandler("/launch/")
|
||||
if err != nil {
|
||||
log.Printf("Not installing GCE /launch/ handler: %v", err)
|
||||
err := err
|
||||
mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, fmt.Sprintf("GCE launcher disabled: %v", err), 500)
|
||||
})
|
||||
} else {
|
||||
mux.Handle("/launch/", gceLauncher)
|
||||
}
|
||||
mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "GCE launcher no longer supported", 500)
|
||||
})
|
||||
|
||||
var handler http.Handler = &redirectRootHandler{Handler: mux}
|
||||
if *logDir != "" || *logStdout {
|
||||
|
|
@ -939,23 +798,6 @@ func main() {
|
|||
log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
|
||||
}
|
||||
handler = NewLoggingHandler(handler, gceLogger{logc.Logger(*gceLogName)})
|
||||
if gceLauncher != nil {
|
||||
var logc *logging.Client
|
||||
if metadata.OnGCE() {
|
||||
logc, err = logging.NewClient(ctx, projID)
|
||||
} else {
|
||||
logc, err = logging.NewClient(ctx, projID, option.WithHTTPClient(hc))
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
commonLabels := logging.CommonLabels(map[string]string{
|
||||
"from": "camli-gce-launcher",
|
||||
})
|
||||
logger := logc.Logger(*gceLogName, commonLabels).StandardLogger(logging.Default)
|
||||
logger.SetPrefix("launcher: ")
|
||||
gceLauncher.SetLogger(logger)
|
||||
}
|
||||
}
|
||||
|
||||
emailErr := make(chan error)
|
||||
|
|
@ -1092,18 +934,6 @@ func fromGCS(filename string) ([]byte, error) {
|
|||
return slurp(filename)
|
||||
}
|
||||
|
||||
func deployerCredsFromGCS() (*gce.Config, error) {
|
||||
var cfg gce.Config
|
||||
data, err := fromGCS("launcher-config.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("could not JSON decode camli GCE launcher config: %v", err)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var issueNum = regexp.MustCompile(`^/(?:issue|bug)s?(/\d*)?$`)
|
||||
|
||||
// issueRedirect returns whether the request should be redirected to the
|
||||
|
|
|
|||
Loading…
Reference in New Issue