From b5823a65b9587bebaca3a72c81e34b4e4130cbfa Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 30 Dec 2023 12:16:34 -0800 Subject: [PATCH] pkg/deploy/gce: remove GCE deploy support We used to have a web-based Perkeep launcher at perkeep.org/launch that created a GCE-based Perkeep instance for users, where they pay Google for compute time. (One of those "one click deploy" template things) Unfortunately, Google broke their APIs for doing the third party VM creations and we disabled it some years ago. But the code remains. And now, updating it again, we find that they've broken it again: Error: pkg/deploy/gce/deploy.go:358:4: servicemanagement.NewServicesService(s).Enable undefined (type *servicemanagement.ServicesService has no field or method Enable) It's not worth fighting Google's API breakages. Just remove the GCE launcher support as it's been unused for years. We can always resurrect this code from git if really needed. But a Digital Ocean or Fly launcher would probably be much easier. Signed-off-by: Brad Fitzpatrick --- .github/workflows/tests-lint.yaml | 7 +- .github/workflows/tests-linux.yml | 7 +- .github/workflows/tests-macos.yml | 7 +- .github/workflows/tests-tidy.yaml | 7 +- .github/workflows/tests-windows.yml | 7 +- cmd/pk-deploy/camdeploy.go | 27 - cmd/pk-deploy/gce.go | 160 ---- doc/environment-vars.md | 38 - make.go | 1 - pkg/deploy/gce/cloud-config.yaml | 19 - pkg/deploy/gce/deploy.go | 917 ------------------ pkg/deploy/gce/handler.go | 1325 --------------------------- pkg/deploy/gce/notes.txt | 31 - pkg/serverinit/env.go | 2 +- website/pk-web/pkweb.go | 176 +--- 15 files changed, 34 insertions(+), 2697 deletions(-) delete mode 100644 cmd/pk-deploy/camdeploy.go delete mode 100644 cmd/pk-deploy/gce.go delete mode 100644 pkg/deploy/gce/cloud-config.yaml delete mode 100644 pkg/deploy/gce/deploy.go delete mode 100644 pkg/deploy/gce/handler.go delete mode 100644 pkg/deploy/gce/notes.txt diff --git a/.github/workflows/tests-lint.yaml b/.github/workflows/tests-lint.yaml index 9f825095b..2404c2e56 100644 --- a/.github/workflows/tests-lint.yaml +++ b/.github/workflows/tests-lint.yaml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/lint" jobs: diff --git a/.github/workflows/tests-linux.yml b/.github/workflows/tests-linux.yml index 876d3b36c..3d53050af 100644 --- a/.github/workflows/tests-linux.yml +++ b/.github/workflows/tests-linux.yml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/linux" jobs: diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3431fa6ce..35ca82f08 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/macos" jobs: diff --git a/.github/workflows/tests-tidy.yaml b/.github/workflows/tests-tidy.yaml index 5dec275f8..b4a161708 100644 --- a/.github/workflows/tests-tidy.yaml +++ b/.github/workflows/tests-tidy.yaml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/tidy" jobs: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 933fbdcbe..3d8ef8a8e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/windows" jobs: diff --git a/cmd/pk-deploy/camdeploy.go b/cmd/pk-deploy/camdeploy.go deleted file mode 100644 index 6f0fb1429..000000000 --- a/cmd/pk-deploy/camdeploy.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2014 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// The pk-deploy program deploys Perkeep on cloud computing platforms -// such as Google Compute Engine or Amazon EC2. -package main // import "perkeep.org/cmd/pk-deploy" - -import ( - "perkeep.org/pkg/cmdmain" -) - -func main() { - cmdmain.Main() -} diff --git a/cmd/pk-deploy/gce.go b/cmd/pk-deploy/gce.go deleted file mode 100644 index f6613ec9e..000000000 --- a/cmd/pk-deploy/gce.go +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2014 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bufio" - "context" - "flag" - "fmt" - "log" - "math/rand" - "os" - "strings" - - "perkeep.org/pkg/cmdmain" - "perkeep.org/pkg/deploy/gce" - - "go4.org/oauthutil" - "golang.org/x/oauth2" -) - -type gceCmd struct { - project string - zone string - machine string - instName string - verbose bool - hostname string - wip bool -} - -func init() { - cmdmain.RegisterMode("gce", func(flags *flag.FlagSet) cmdmain.CommandRunner { - cmd := new(gceCmd) - flags.StringVar(&cmd.project, "project", "", "Name of Project.") - flags.StringVar(&cmd.zone, "zone", gce.DefaultRegion, "GCE zone or region. If a region is given, a random zone in that region is selected. See https://cloud.google.com/compute/docs/zones") - flags.StringVar(&cmd.machine, "machine", gce.DefaultMachineType, "GCE machine type (e.g. n1-standard-1, f1-micro, g1-small); see https://cloud.google.com/compute/docs/machine-types") - flags.StringVar(&cmd.instName, "instance_name", gce.DefaultInstanceName, "Name of VM instance.") - flags.StringVar(&cmd.hostname, "hostname", "", "Optional hostname for the instance. If set, the custom metadata variable \"camlistore-hostname\" on the instance takes that value. Otherwise, perkeepd sets that variable to the hostname we get from the camliNet DNS.") - flags.BoolVar(&cmd.wip, "wip", false, "Developer option. Deploy the WORKINPROGRESS perkeepd tarball.") - flags.BoolVar(&cmd.verbose, "verbose", false, "Be verbose.") - return cmd - }) -} - -const ( - clientIdDat = "client-id.dat" - clientSecretDat = "client-secret.dat" - helpEnableAuth = `Enable authentication: in your project console, navigate to "APIs and auth", "Credentials", click on "Create new Client ID", and pick "Installed application", with type "Other". Copy the CLIENT ID to ` + clientIdDat + `, and the CLIENT SECRET to ` + clientSecretDat - helpCreateProject = "Go to " + gce.ConsoleURL + " to create a new Google Cloud project" - helpEnableAPIs = `Enable the project APIs: in your project console, navigate to "APIs and auth", "APIs". In the list, enable "Google Cloud Storage", "Google Cloud Storage", "Google Cloud Storage JSON API", and "Google Compute Engine".` -) - -func (c *gceCmd) Describe() string { - return "Deploy Perkeep on Google Compute Engine." -} - -func (c *gceCmd) Usage() { - fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n %s\n\n", - "pk-deploy gce --project= --hostname= [options]", - "pk-deploy gce --project= --cert= --key= [options]") - flag.PrintDefaults() - fmt.Fprintln(os.Stderr, "\nTo get started:") - printHelp() -} - -func printHelp() { - for _, v := range []string{helpCreateProject, helpEnableAuth, helpEnableAPIs} { - fmt.Fprintf(os.Stderr, "%v\n", v) - } -} - -func (c *gceCmd) RunCommand(args []string) error { - if c.verbose { - gce.Verbose = true - } - if c.project == "" { - return cmdmain.UsageError("Missing --project flag.") - } - - // We embed the client ID and client secret, per - // https://developers.google.com/identity/protocols/OAuth2InstalledApp - // Notably: "The client ID and client secret obtained from the - // Developers Console are embedded in the source code of your - // application. In this context, the client secret is - // obviously not treated as a secret." - // - // These were created at: - // https://console.developers.google.com/apis/credentials?project=camlistore-website - // (Notes for Brad and Mathieu) - const ( - clientID = "574004351801-9qqoggh6b5v3jqt722v43ikmgmtv60h3.apps.googleusercontent.com" - clientSecret = "Gf1zwaOcbJnRTE5zD4feKaTI" // NOT a secret, despite name - ) - config := gce.NewOAuthConfig(clientID, clientSecret) - config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob" - - hc := oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{ - Config: config, - CacheFile: c.project + "-token.json", - AuthCode: func() string { - fmt.Println("Get auth code from:") - fmt.Printf("%v\n\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce)) - fmt.Print("Enter auth code: ") - sc := bufio.NewScanner(os.Stdin) - sc.Scan() - return strings.TrimSpace(sc.Text()) - }, - })) - - zone := c.zone - if gce.LooksLikeRegion(zone) { - region := zone - zones, err := gce.ZonesOfRegion(hc, c.project, region) - if err != nil { - return err - } - if len(zones) == 0 { - return fmt.Errorf("no zones found in region %q; invalid region?", region) - } - zone = zones[rand.Intn(len(zones))] - } - - instConf := &gce.InstanceConf{ - Name: c.instName, - Project: c.project, - Machine: c.machine, - Zone: zone, - Hostname: c.hostname, - WIP: c.wip, - } - - log.Printf("Creating instance %s (in project %s) in zone %s ...", c.instName, c.project, zone) - depl := &gce.Deployer{ - Client: hc, - Conf: instConf, - Logger: log.New(cmdmain.Stderr, "", log.Flags()), - } - inst, err := depl.Create(context.Background()) - if err != nil { - return err - } - - log.Printf("Instance created; starting up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP) - return nil -} diff --git a/doc/environment-vars.md b/doc/environment-vars.md index ad1a756b4..81118d1d8 100644 --- a/doc/environment-vars.md +++ b/doc/environment-vars.md @@ -70,44 +70,6 @@ or `CAMLI_FORCE_OSARCH` (bool) Used by make.go to force building an unrecommended OS/ARCH pair. -`CAMLI_GCE_\*` -: Variables prefixed with `CAMLI_GCE_` concern the Google Compute Engine deploy - handler in [pkg/deploy/gce](/pkg/deploy/gce), which is only used by camweb to - launch Perkeep on Google Compute Engine. They do not affect Perkeep's - behaviour. - -`CAMLI_GCE_CLIENTID` (string) -: See `CAMLI_GCE_*` first. This string is used by gce.DeployHandler as the - application's OAuth Client ID. If blank, camweb does not enable the Google - Compute Engine launcher. - -`CAMLI_GCE_CLIENTSECRET` (string) -: See `CAMLI_GCE_*` first. Used by gce.DeployHandler as the application's OAuth - Client Secret. If blank, gce.NewDeployHandler returns an error, and camweb - fails to start if the Google Compute Engine launcher was enabled. - -`CAMLI_GCE_DATA` (string) -: See `CAMLI_GCE_*` first. Path to the directory where gce.DeployHandler stores - the instances configuration and state. If blank, the "camli-gce-data" default - is used instead. - -`CAMLI_GCE_PROJECT` (string) -: See `CAMLI_GCE_*` first. ID of the Google Project that provides the above - client ID and secret. It is used when we query for the list of all the - existing zones, since such a query requires a project ID. If blank, a - hard-coded list of zones is used instead. - -`CAMLI_GCE_SERVICE_ACCOUNT` (string) -: See `CAMLI_GCE_*` first. Path to a Google service account JSON file. This - account should have at least compute.readonly permissions on the Google - Project with ID CAMLI_GCE_PROJECT. It is used to authenticate when querying - for the list of all the existing zones. If blank, a hard-coded list of zones - is used instead. - -`CAMLI_GCE_XSRFKEY` (string) -: See `CAMLI_GCE_*` first. Used by gce.DeployHandler as the XSRF protection key. - If blank, gce.NewDeployHandler generates a new random key instead. - `CAMLI_GOPHERJS_GOROOT` (string) : As gopherjs does not build with go tip, when make.go is run with go devel, CAMLI_GOPHERJS_GOROOT should be set to a Go 1.10 root so that gopherjs can be diff --git a/make.go b/make.go index 9b4f51721..f9708c16a 100644 --- a/make.go +++ b/make.go @@ -104,7 +104,6 @@ func main() { "perkeep.org/cmd/pk-get", "perkeep.org/cmd/pk-put", "perkeep.org/cmd/pk", - "perkeep.org/cmd/pk-deploy", "perkeep.org/server/perkeepd", "perkeep.org/app/hello", "perkeep.org/app/publisher", diff --git a/pkg/deploy/gce/cloud-config.yaml b/pkg/deploy/gce/cloud-config.yaml deleted file mode 100644 index 78c4ebb95..000000000 --- a/pkg/deploy/gce/cloud-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -#cloud-config - -coreos: - units: - - name: hello-docker.service - command: start - content: | - [Unit] - Description=Camlistore - After=docker.service - Requires=docker.service - - [Service] - ExecStart=/usr/bin/docker run -p 80:80 camlistore/hello - RestartSec=500ms - Restart=always - - [Install] - WantedBy=multi-user.target diff --git a/pkg/deploy/gce/deploy.go b/pkg/deploy/gce/deploy.go deleted file mode 100644 index 359f2227f..000000000 --- a/pkg/deploy/gce/deploy.go +++ /dev/null @@ -1,917 +0,0 @@ -/* -Copyright 2014 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package gce provides tools to deploy Perkeep on Google Compute Engine. -package gce // import "perkeep.org/pkg/deploy/gce" - -import ( - "context" - "crypto/rand" - "encoding/json" - "errors" - "fmt" - "log" - "math/big" - "net/http" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "perkeep.org/internal/osutil" - - "cloud.google.com/go/logging" - "go4.org/cloud/google/gceutil" - "go4.org/syncutil" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" - compute "google.golang.org/api/compute/v1" - "google.golang.org/api/googleapi" - servicemanagement "google.golang.org/api/servicemanagement/v1" - storage "google.golang.org/api/storage/v1" -) - -const ( - DefaultInstanceName = "camlistore-server" - DefaultMachineType = "g1-small" - DefaultRegion = "us-central1" - - projectsAPIURL = "https://www.googleapis.com/compute/v1/projects/" - - fallbackZone = "us-central1-a" - - camliUsername = "camlistore" // directly set in compute metadata, so not user settable. - - configDir = "config" - - ConsoleURL = "https://console.developers.google.com" - helpDeleteInstance = `To delete an existing Compute Engine instance: in your project console, navigate to "Compute", "Compute Engine", and "VM instances". Select your instance and click "Delete".` -) - -var ( - // Verbose enables more info to be printed. - Verbose bool -) - -// certFilename returns the HTTPS certificate file name -func certFilename() string { - return filepath.Base(osutil.DefaultTLSCert()) -} - -// keyFilename returns the HTTPS key name -func keyFilename() string { - return filepath.Base(osutil.DefaultTLSKey()) -} - -// NewOAuthConfig returns an OAuth configuration template. -func NewOAuthConfig(clientID, clientSecret string) *oauth2.Config { - return &oauth2.Config{ - Scopes: []string{ - logging.WriteScope, - compute.DevstorageFullControlScope, - compute.ComputeScope, - cloudresourcemanager.CloudPlatformScope, - servicemanagement.CloudPlatformScope, - "https://www.googleapis.com/auth/sqlservice", - "https://www.googleapis.com/auth/sqlservice.admin", - }, - Endpoint: google.Endpoint, - ClientID: clientID, - ClientSecret: clientSecret, - } -} - -// InstanceConf is the configuration for the Google Compute Engine instance that will be deployed. -type InstanceConf struct { - Name string // Name given to the virtual machine instance. - Project string // Google project ID where the instance is created. - CreateProject bool // CreateProject defines whether to first create project. - Machine string // Machine type. - Zone string // GCE zone; see https://cloud.google.com/compute/docs/zones - Hostname string // Fully qualified domain name. - - configDir string // bucketBase() + "/config" - blobDir string // bucketBase() + "/blobs" - - Ctime time.Time // Timestamp for this configuration. - - WIP bool // Whether to use the perkeepd-WORKINPROGRESS.tar.gz tarball instead of the "production" one -} - -// Deployer creates and starts an instance such as defined in Conf. -type Deployer struct { - // Client is an OAuth2 client, authenticated for working with - // the user's Google Cloud resources. - Client *http.Client - - Conf *InstanceConf - - *log.Logger // Cannot be nil. -} - -// Get returns the Instance corresponding to the Project, Zone, and Name defined in the -// Deployer's Conf. -func (d *Deployer) Get() (*compute.Instance, error) { - computeService, err := compute.New(d.Client) - if err != nil { - return nil, err - } - return computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() -} - -type instanceExistsError struct { - project string - zone string - name string -} - -func (e instanceExistsError) Error() string { - if e.project == "" { - panic("instanceExistsErr has no project") - } - msg := "some instance(s) already exist as (" + e.project - if e.zone != "" { - msg += ", " + e.zone - } - if e.name != "" { - msg += ", " + e.name - } - msg += "), you need to delete them first." - return msg -} - -// projectHasInstance checks for all the possible zones if there's already an instance for the project. -// It returns the name of the zone at the first instance it finds, if any. -func (d *Deployer) projectHasInstance() (zone string, err error) { - s, err := compute.New(d.Client) - if err != nil { - return "", err - } - // TODO(mpl): make use of the handler's cached zones. - zl, err := compute.NewZonesService(s).List(d.Conf.Project).Do() - if err != nil { - return "", fmt.Errorf("could not get a list of zones: %v", err) - } - computeService, _ := compute.New(d.Client) - var zoneOnce sync.Once - var grp syncutil.Group - errc := make(chan error, 1) - zonec := make(chan string, 1) - timeout := time.NewTimer(30 * time.Second) - defer timeout.Stop() - for _, z := range zl.Items { - z := z - grp.Go(func() error { - list, err := computeService.Instances.List(d.Conf.Project, z.Name).Do() - if err != nil { - return fmt.Errorf("could not list existing instances: %v", err) - } - if len(list.Items) > 0 { - zoneOnce.Do(func() { - zonec <- z.Name - }) - } - return nil - }) - } - go func() { - errc <- grp.Err() - }() - // We block until either an instance was found in a zone, or all the instance - // listing is done. Or we timed-out. - select { - case err = <-errc: - return "", err - case zone = <-zonec: - // We voluntarily ignore any listing error if we found at least one instance - // because that's what we primarily want to report about. - return zone, nil - case <-timeout.C: - return "", errors.New("timed out") - } -} - -type projectIDError struct { - id string - cause error -} - -func (e projectIDError) Error() string { - if e.id == "" { - panic("projectIDError without an id") - } - if e.cause != nil { - return fmt.Sprintf("project ID error for %v: %v", e.id, e.cause) - } - return fmt.Sprintf("project ID error for %v", e.id) -} - -// CreateProject creates a new Google Cloud Project. It returns the project ID, -// which is a random number in (0,1e10), prefixed with "camlistore-launcher-". -func (d *Deployer) CreateProject(ctx context.Context) (string, error) { - s, err := cloudresourcemanager.New(d.Client) - if err != nil { - return "", err - } - // Allow for a few retries, when we generated an already taken project ID - creationTimeout := time.Now().Add(time.Minute) - var projectID, projectName string - for { - if d.Conf.Project != "" { - projectID = d.Conf.Project - projectName = projectID - } else { - projectID = genRandomProjectID() - projectName = strings.Replace(projectID, "camlistore-launcher-", "Camlistore ", 1) - } - project := cloudresourcemanager.Project{ - Name: projectName, - ProjectId: projectID, - } - if time.Now().After(creationTimeout) { - return "", errors.New("timeout while trying to create project") - } - d.Printf("Trying to create project %v", projectID) - op, err := cloudresourcemanager.NewProjectsService(s).Create(&project).Do() - if err != nil { - gerr, ok := err.(*googleapi.Error) - if !ok { - return "", fmt.Errorf("could not create project: %v", err) - } - if gerr.Code != 409 { - return "", fmt.Errorf("could not create project: %v", gerr.Message) - } - // it's ok using time.Sleep, and no backoff, as the - // timeout is pretty short, and we only retry on a 409. - time.Sleep(time.Second) - // retry if project ID already exists. - d.Printf("Project %v already exists, will retry with a new project ID", project.ProjectId) - continue - } - - // as per - // https://cloud.google.com/resource-manager/reference/rest/v1/projects/create - // recommendation - timeout := time.Now().Add(30 * time.Second) - backoff := time.Second - startPolling := 5 * time.Second - time.Sleep(startPolling) - for { - if time.Now().After(timeout) { - return "", fmt.Errorf("timeout while trying to check project creation") - } - if !op.Done { - // it's ok to just sleep, as our timeout is pretty short. - time.Sleep(backoff) - backoff *= 2 - op, err = cloudresourcemanager.NewOperationsService(s).Get(op.Name).Do() - if err != nil { - return "", fmt.Errorf("could not check project creation status: %v", err) - } - continue - } - if op.Error != nil { - // TODO(mpl): ghetto logging for now. detect at least the quota errors. - var details string - for _, v := range op.Error.Details { - details += string(v) - } - return "", fmt.Errorf("could not create project: %v, %v", op.Error.Message, details) - } - break - } - break - } - d.Printf("Success creating project %v", projectID) - return projectID, nil -} - -func genRandomProjectID() string { - // we're allowed up to 30 characters, and we already consume 20 with - // "camlistore-launcher-", so we've got 10 chars left of randomness. Should - // be plenty enough I think. - var n *big.Int - var err error - zero := big.NewInt(0) - for { - n, err = rand.Int(rand.Reader, big.NewInt(1e10)) // max is 1e10 - 1 - if err != nil { - panic(fmt.Sprintf("rand.Int error: %v", err)) - } - if n.Cmp(zero) > 0 { - break - } - } - return fmt.Sprintf("camlistore-launcher-%d", n) -} - -func (d *Deployer) enableAPIs() error { - // TODO(mpl): For now we're lucky enough that servicemanagement seems to - // work even when the Service Management API hasn't been enabled for the - // project. If/when it does not anymore, then we should use serviceuser - // instead. http://stackoverflow.com/a/43503392/1775619 - s, err := servicemanagement.New(d.Client) - if err != nil { - return err - } - - list, err := servicemanagement.NewServicesService(s).List().ConsumerId("project:" + d.Conf.Project).Do() - if err != nil { - return err - } - - requiredServices := map[string]string{ - "storage-component.googleapis.com": "Google Cloud Storage", - "storage-api.googleapis.com": "Google Cloud Storage JSON", - "logging.googleapis.com": "Stackdriver Logging", - "compute.googleapis.com": "Google Compute Engine", - "geocoding-backend.googleapis.com": "Google Maps Geocoding", - } - enabledServices := make(map[string]bool) - for _, v := range list.Services { - enabledServices[v.ServiceName] = true - } - errc := make(chan error, len(requiredServices)) - var wg sync.WaitGroup - for k, v := range requiredServices { - if _, ok := enabledServices[k]; ok { - continue - } - d.Printf("%v API not enabled; enabling it with Service Management", v) - op, err := servicemanagement.NewServicesService(s). - Enable(k, &servicemanagement.EnableServiceRequest{ConsumerId: "project:" + d.Conf.Project}).Do() - if err != nil { - gerr, ok := err.(*googleapi.Error) - if !ok { - return err - } - if gerr.Code != http.StatusBadRequest { - return err - } - for _, v := range gerr.Errors { - if v.Reason == "failedPrecondition" && strings.Contains(v.Message, "billing-enabled") { - return fmt.Errorf("you need to enabling billing for project %v: https://console.cloud.google.com/billing/?project=%v", d.Conf.Project, d.Conf.Project) - } - } - return err - } - - wg.Add(1) - go func(service, opName string) { - defer wg.Done() - timeout := time.Now().Add(2 * time.Minute) - backoff := time.Second - startPolling := 5 * time.Second - time.Sleep(startPolling) - for { - if time.Now().After(timeout) { - errc <- fmt.Errorf("timeout while trying to enable service: %v", service) - return - } - op, err := servicemanagement.NewOperationsService(s).Get(opName).Do() - if err != nil { - errc <- fmt.Errorf("could not check service enabling status: %v", err) - return - } - if !op.Done { - // it's ok to just sleep, as our timeout is pretty short. - time.Sleep(backoff) - backoff *= 2 - continue - } - if op.Error != nil { - errc <- fmt.Errorf("could not enable service %v: %v", service, op.Error.Message) - return - } - d.Printf("%v service successfully enabled", service) - return - } - }(k, op.Name) - } - wg.Wait() - close(errc) - for err := range errc { - if err != nil { - return err - } - } - return nil -} - -func (d *Deployer) checkProjectID() error { - // TODO(mpl): cache the computeService in Deployer, instead of recreating a new one everytime? - s, err := compute.New(d.Client) - if err != nil { - return projectIDError{ - id: d.Conf.Project, - cause: err, - } - } - project, err := compute.NewProjectsService(s).Get(d.Conf.Project).Do() - if err != nil { - return projectIDError{ - id: d.Conf.Project, - cause: err, - } - } - if project.Name != d.Conf.Project { - return projectIDError{ - id: d.Conf.Project, - cause: fmt.Errorf("project IDs do not match: got %q, wanted %q", project.Name, d.Conf.Project), - } - } - return nil -} - -var errAttrNotFound = errors.New("attribute not found") - -// getInstanceAttribute returns the value for attr in the custom metadata of the -// instance. It returns errAttrNotFound is such a metadata attributed does not -// exist. -func (d *Deployer) getInstanceAttribute(attr string) (string, error) { - s, err := compute.New(d.Client) - if err != nil { - return "", fmt.Errorf("error getting compute service: %v", err) - } - inst, err := compute.NewInstancesService(s).Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() - if err != nil { - return "", fmt.Errorf("error getting instance: %v", err) - } - for _, v := range inst.Metadata.Items { - if v.Key == attr { - return *(v.Value), nil - } - } - return "", errAttrNotFound -} - -// Create sets up and starts a Google Compute Engine instance as defined in d.Conf. It -// creates the necessary Google Storage buckets beforehand. -func (d *Deployer) Create(ctx context.Context) (*compute.Instance, error) { - if err := d.enableAPIs(); err != nil { - return nil, projectIDError{ - id: d.Conf.Project, - cause: err, - } - } - if err := d.checkProjectID(); err != nil { - return nil, err - } - - computeService, _ := compute.New(d.Client) - storageService, _ := storage.New(d.Client) - - fwc := make(chan error, 1) - go func() { - fwc <- d.setFirewall(ctx, computeService) - }() - - config := cloudConfig(d.Conf) - const maxCloudConfig = 32 << 10 // per compute API docs - if len(config) > maxCloudConfig { - return nil, fmt.Errorf("cloud config length of %d bytes is over %d byte limit", len(config), maxCloudConfig) - } - - if zone, err := d.projectHasInstance(); zone != "" { - return nil, instanceExistsError{ - project: d.Conf.Project, - zone: zone, - } - } else if err != nil { - return nil, fmt.Errorf("could not scan project for existing instances: %v", err) - } - - if err := d.setBuckets(ctx, storageService); err != nil { - return nil, fmt.Errorf("could not create buckets: %v", err) - } - - if err := d.createInstance(ctx, computeService); err != nil { - return nil, fmt.Errorf("could not create compute instance: %v", err) - } - - inst, err := computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() - if err != nil { - return nil, fmt.Errorf("error getting instance after creation: %v", err) - } - if Verbose { - ij, _ := json.MarshalIndent(inst, "", " ") - d.Printf("Instance: %s", ij) - } - - if err = <-fwc; err != nil { - return nil, fmt.Errorf("could not create firewall rules: %v", err) - } - return inst, nil -} - -func randPassword() string { - buf := make([]byte, 5) - if n, err := rand.Read(buf); err != nil || n != len(buf) { - log.Fatalf("crypto/rand.Read = %v, %v", n, err) - } - return fmt.Sprintf("%x", buf) -} - -// LooksLikeRegion reports whether s looks like a GCE region. -func LooksLikeRegion(s string) bool { - return strings.Count(s, "-") == 1 -} - -// createInstance starts the creation of the Compute Engine instance and waits for the -// result of the creation operation. It should be called after setBuckets and setupHTTPS. -func (d *Deployer) createInstance(ctx context.Context, computeService *compute.Service) error { - coreosImgURL, err := gceutil.CoreOSImageURL(d.Client) - if err != nil { - return fmt.Errorf("error looking up latest CoreOS stable image: %v", err) - } - prefix := projectsAPIURL + d.Conf.Project - machType := prefix + "/zones/" + d.Conf.Zone + "/machineTypes/" + d.Conf.Machine - config := cloudConfig(d.Conf) - instance := &compute.Instance{ - Name: d.Conf.Name, - Description: "Camlistore server", - MachineType: machType, - Disks: []*compute.AttachedDisk{ - { - AutoDelete: true, - Boot: true, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: d.Conf.Name + "-coreos-stateless-pd", - SourceImage: coreosImgURL, - }, - }, - }, - Tags: &compute.Tags{ - Items: []string{"http-server", "https-server"}, - }, - Metadata: &compute.Metadata{ - Items: []*compute.MetadataItems{ - { - Key: "camlistore-username", - Value: googleapi.String(camliUsername), - }, - { - Key: "camlistore-password", - Value: googleapi.String(randPassword()), - }, - { - Key: "camlistore-blob-dir", - Value: googleapi.String("gs://" + d.Conf.blobDir), - }, - { - Key: "camlistore-config-dir", - Value: googleapi.String("gs://" + d.Conf.configDir), - }, - { - Key: "perkeep-config-version", - // perkeep-config-version being defined requires this launcher to deploy Perkeep - // at rev >= 7eda9fd5027fda88166d6c03b6490cffbf2de5fb , so that any newly deployed Perkeep - // knows it can use the new configuration without DBNames. - // TODO(mpl): but how do we enforce it, or at least make it more obvious/documented? - // With a flag defaulting to "1" maybe? - Value: googleapi.String("1"), - }, - { - Key: "user-data", - Value: googleapi.String(config), - }, - }, - }, - NetworkInterfaces: []*compute.NetworkInterface{ - { - AccessConfigs: []*compute.AccessConfig{ - { - Type: "ONE_TO_ONE_NAT", - Name: "External NAT", - }, - }, - Network: prefix + "/global/networks/default", - }, - }, - ServiceAccounts: []*compute.ServiceAccount{ - { - Email: "default", - Scopes: []string{ - logging.WriteScope, - compute.DevstorageFullControlScope, - compute.ComputeScope, - "https://www.googleapis.com/auth/sqlservice", - "https://www.googleapis.com/auth/sqlservice.admin", - }, - }, - }, - } - if d.Conf.Hostname != "" { - instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{ - Key: "camlistore-hostname", - Value: googleapi.String(d.Conf.Hostname), - }) - } - const localMySQL = false // later - if localMySQL { - instance.Disks = append(instance.Disks, &compute.AttachedDisk{ - AutoDelete: false, - Boot: false, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: "camlistore-mysql-index-pd", - DiskSizeGb: 4, - }, - }) - } - - if Verbose { - d.Print("Creating instance...") - } - op, err := computeService.Instances.Insert(d.Conf.Project, d.Conf.Zone, instance).Do() - if err != nil { - return fmt.Errorf("failed to create instance: %v", err) - } - opName := op.Name - if Verbose { - d.Printf("Created. Waiting on operation %v", opName) - } -OpLoop: - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - time.Sleep(2 * time.Second) - op, err := computeService.ZoneOperations.Get(d.Conf.Project, d.Conf.Zone, opName).Do() - if err != nil { - return fmt.Errorf("failed to get op %s: %v", opName, err) - } - switch op.Status { - case "PENDING", "RUNNING": - if Verbose { - d.Printf("Waiting on operation %v", opName) - } - continue - case "DONE": - if op.Error != nil { - for _, operr := range op.Error.Errors { - d.Printf("Error: %+v", operr) - } - return fmt.Errorf("failed to start") - } - if Verbose { - d.Printf("Success. %+v", op) - } - break OpLoop - default: - return fmt.Errorf("unknown status %q: %+v", op.Status, op) - } - } - return nil -} - -func cloudConfig(conf *InstanceConf) string { - config := strings.Replace(baseInstanceConfig, "INNODB_BUFFER_POOL_SIZE=NNN", "INNODB_BUFFER_POOL_SIZE="+strconv.Itoa(innodbBufferPoolSize(conf.Machine)), -1) - perkeepdTarball := "https://storage.googleapis.com/camlistore-release/docker/" - if conf.WIP { - perkeepdTarball += "perkeepd-WORKINPROGRESS.tar.gz" - } else { - perkeepdTarball += "perkeepd.tar.gz" - } - config = strings.Replace(config, "CAMLISTORED_TARBALL", perkeepdTarball, 1) - return config -} - -// setBuckets defines the buckets needed by the instance and creates them. -func (d *Deployer) setBuckets(ctx context.Context, storageService *storage.Service) error { - projBucket := d.Conf.Project + "-camlistore" - - needBucket := map[string]bool{ - projBucket: true, - } - - buckets, err := storageService.Buckets.List(d.Conf.Project).Do() - if err != nil { - return fmt.Errorf("error listing buckets: %v", err) - } - for _, it := range buckets.Items { - delete(needBucket, it.Name) - } - if len(needBucket) > 0 { - if Verbose { - d.Printf("Need to create buckets: %v", needBucket) - } - var waitBucket sync.WaitGroup - var bucketErr error - for name := range needBucket { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - name := name - waitBucket.Add(1) - go func() { - defer waitBucket.Done() - if Verbose { - d.Printf("Creating bucket %s", name) - } - b, err := storageService.Buckets.Insert(d.Conf.Project, &storage.Bucket{ - Id: name, - Name: name, - }).Do() - if err != nil && bucketErr == nil { - bucketErr = fmt.Errorf("error creating bucket %s: %v", name, err) - return - } - if Verbose { - d.Printf("Created bucket %s: %+v", name, b) - } - }() - } - waitBucket.Wait() - if bucketErr != nil { - return bucketErr - } - } - - d.Conf.configDir = path.Join(projBucket, configDir) - d.Conf.blobDir = path.Join(projBucket, "blobs") - return nil -} - -// setFirewall adds the firewall rules needed for ports 80 & 433 to the default network. -func (d *Deployer) setFirewall(ctx context.Context, computeService *compute.Service) error { - defaultNet, err := computeService.Networks.Get(d.Conf.Project, "default").Do() - if err != nil { - return fmt.Errorf("error getting default network: %v", err) - } - - needRules := map[string]compute.Firewall{ - "default-allow-http": { - Name: "default-allow-http", - SourceRanges: []string{"0.0.0.0/0"}, - SourceTags: []string{"http-server"}, - Allowed: []*compute.FirewallAllowed{{IPProtocol: "tcp", Ports: []string{"80"}}}, - Network: defaultNet.SelfLink, - }, - "default-allow-https": { - Name: "default-allow-https", - SourceRanges: []string{"0.0.0.0/0"}, - SourceTags: []string{"https-server"}, - Allowed: []*compute.FirewallAllowed{{IPProtocol: "tcp", Ports: []string{"443"}}}, - Network: defaultNet.SelfLink, - }, - } - - rules, err := computeService.Firewalls.List(d.Conf.Project).Do() - if err != nil { - return fmt.Errorf("error listing rules: %v", err) - } - for _, it := range rules.Items { - delete(needRules, it.Name) - } - if len(needRules) == 0 { - return nil - } - - if Verbose { - d.Printf("Need to create rules: %v", needRules) - } - var wg syncutil.Group - for name, rule := range needRules { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - name, rule := name, rule - wg.Go(func() error { - if Verbose { - d.Printf("Creating rule %s", name) - } - r, err := computeService.Firewalls.Insert(d.Conf.Project, &rule).Do() - if err != nil { - return fmt.Errorf("error creating rule %s: %v", name, err) - } - if Verbose { - d.Printf("Created rule %s: %+v", name, r) - } - return nil - }) - } - return wg.Err() -} - -// returns the MySQL InnoDB buffer pool size (in bytes) as a function -// of the GCE machine type. -func innodbBufferPoolSize(machine string) int { - // Totally arbitrary. We don't need much here because - // perkeepd slurps this all into its RAM on start-up - // anyway. So this is all prety overkill and more than the - // 8MB default. - switch machine { - case "f1-micro": - return 32 << 20 - case "g1-small": - return 64 << 20 - default: - return 128 << 20 - } -} - -const baseInstanceConfig = `#cloud-config -write_files: - - path: /var/lib/camlistore/tmp/README - permissions: 0644 - content: | - This is the Camlistore /tmp directory. - - path: /var/lib/camlistore/mysql/README - permissions: 0644 - content: | - This is the Camlistore MySQL data directory. -coreos: - units: - - name: cam-journal-gatewayd.service - content: | - [Unit] - Description=Journal Gateway Service - Requires=cam-journal-gatewayd.socket - - [Service] - ExecStart=/usr/lib/systemd/systemd-journal-gatewayd - User=systemd-journal-gateway - Group=systemd-journal-gateway - SupplementaryGroups=systemd-journal - PrivateTmp=yes - PrivateDevices=yes - PrivateNetwork=yes - ProtectSystem=full - ProtectHome=yes - - [Install] - Also=cam-journal-gatewayd.socket - - name: cam-journal-gatewayd.socket - command: start - content: | - [Unit] - Description=Journal Gateway Service Socket - - [Socket] - ListenStream=/run/camjournald.sock - - [Install] - WantedBy=sockets.target - - name: mysql.service - command: start - content: | - [Unit] - Description=MySQL - After=docker.service - Requires=docker.service - - [Service] - ExecStartPre=/bin/bash -c '/usr/bin/curl https://storage.googleapis.com/camlistore-release/docker/systemd-docker.tar.gz | /bin/gunzip -c | /usr/bin/docker load' - ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin camlistore/systemd-docker - ExecStart=/opt/bin/systemd-docker run --rm --name %n -v /var/lib/camlistore/mysql:/mysql -e INNODB_BUFFER_POOL_SIZE=NNN camlistore/mysql - RestartSec=1s - Restart=always - Type=notify - NotifyAccess=all - - [Install] - WantedBy=multi-user.target - - name: perkeepd.service - command: start - content: | - [Unit] - Description=Camlistore - After=docker.service mysql.service - Requires=docker.service mysql.service - - [Service] - ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin camlistore/systemd-docker - ExecStartPre=/bin/bash -c '/usr/bin/curl CAMLISTORED_TARBALL | /bin/gunzip -c | /usr/bin/docker load' - ExecStart=/opt/bin/systemd-docker run --rm -p 80:80 -p 443:443 --name %n -v /run/camjournald.sock:/run/camjournald.sock -v /var/lib/camlistore/tmp:/tmp --link=mysql.service:mysqldb perkeep/server - RestartSec=1s - Restart=always - Type=notify - NotifyAccess=all - - [Install] - WantedBy=multi-user.target -` diff --git a/pkg/deploy/gce/handler.go b/pkg/deploy/gce/handler.go deleted file mode 100644 index 287613ba9..000000000 --- a/pkg/deploy/gce/handler.go +++ /dev/null @@ -1,1325 +0,0 @@ -/* -Copyright 2015 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gce - -import ( - "bytes" - "context" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "html/template" - "io" - "log" - "math/rand" - "net/http" - "os" - "path" - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "time" - - "perkeep.org/internal/httputil" - "perkeep.org/pkg/auth" - "perkeep.org/pkg/blob" - "perkeep.org/pkg/blobserver" - "perkeep.org/pkg/blobserver/localdisk" - "perkeep.org/pkg/sorted" - "perkeep.org/pkg/sorted/leveldb" - - "cloud.google.com/go/compute/metadata" - "golang.org/x/net/xsrftoken" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - compute "google.golang.org/api/compute/v1" -) - -const cookieExpiration = 24 * time.Hour - -var ( - helpMachineTypes = "https://cloud.google.com/compute/docs/machine-types" - helpZones = "https://cloud.google.com/compute/docs/zones#available" - - machineValues = []string{ - "f1-micro", - "g1-small", - "n1-highcpu-2", - } - - backupZones = map[string][]string{ - "us-central1": {"-a", "-b", "-f"}, - "europe-west1": {"-b", "-c", "-d"}, - "asia-east1": {"-a", "-b", "-c"}, - } -) - -// DeployHandler serves a wizard that helps with the deployment of Perkeep on Google -// Compute Engine. It must be initialized with NewDeployHandler. -type DeployHandler struct { - scheme string // URL scheme for the URLs served by this handler. Defaults to "https". - host string // URL host for the URLs served by this handler. - prefix string // prefix is the pattern for which this handler is registered as an http.Handler. - help map[string]template.HTML // various help bits used in the served pages, keyed by relevant names. - xsrfKey string // for XSRF protection. - piggyGIF string // path to the piggy gif file, defaults to /static/piggy.gif - mux *http.ServeMux - - tplMu sync.RWMutex - tpl *template.Template - - // Our wizard's credentials, acting on behalf of the user. - // Obtained from the environment for now. - clientID string - clientSecret string - - // stores the user submitted configuration as a JSON-encoded InstanceConf - instConf blobserver.Storage - // key is blobRef of the relevant InstanceConf, value is the current state of - // the instance creation process, as JSON-encoded creationState - instState sorted.KeyValue - - recordStateErrMu sync.RWMutex - // recordStateErr maps the blobRef of the relevant InstanceConf to the error - // that occurred when recording the creation state. - recordStateErr map[string]error - - zonesMu sync.RWMutex - // maps a region to all its zones suffixes (e.g. "asia-east1" -> "-a","-b"). updated in the - // background every 24 hours. defaults to backupZones. - zones map[string][]string - regions []string - - camliVersionMu sync.RWMutex - camliVersion string // git revision found in https://storage.googleapis.com/camlistore-release/docker/VERSION - - logger *log.Logger // should not be nil. -} - -// Config is the set of parameters to initialize the DeployHandler. -type Config struct { - ClientID string `json:"clientID"` // handler's credentials for OAuth. Required. - ClientSecret string `json:"clientSecret"` // handler's credentials for OAuth. Required. - Project string `json:"project"` // any Google Cloud project we can query to get the valid Google Cloud zones. Optional. Set from metadata on GCE. - ServiceAccount string `json:"serviceAccount"` // JSON file with credentials to Project. Optional. Unused on GCE. - DataDir string `json:"dataDir"` // where to store the instances configurations and states. Optional. -} - -// NewDeployHandlerFromConfig initializes a DeployHandler from cfg. -// Host and prefix have the same meaning as for NewDeployHandler. cfg should not be nil. -func NewDeployHandlerFromConfig(host, prefix string, cfg *Config) (*DeployHandler, error) { - if cfg == nil { - panic("NewDeployHandlerFromConfig: nil config") - } - if cfg.ClientID == "" { - return nil, errors.New("oauth2 clientID required in config") - } - if cfg.ClientSecret == "" { - return nil, errors.New("oauth2 clientSecret required in config") - } - os.Setenv("CAMLI_GCE_CLIENTID", cfg.ClientID) - os.Setenv("CAMLI_GCE_CLIENTSECRET", cfg.ClientSecret) - os.Setenv("CAMLI_GCE_PROJECT", cfg.Project) - os.Setenv("CAMLI_GCE_SERVICE_ACCOUNT", cfg.ServiceAccount) - os.Setenv("CAMLI_GCE_DATA", cfg.DataDir) - return NewDeployHandler(host, prefix) -} - -// NewDeployHandler initializes a DeployHandler that serves at https://host/prefix/ and returns it. -// A Google account client ID should be set in CAMLI_GCE_CLIENTID with its corresponding client -// secret in CAMLI_GCE_CLIENTSECRET. -func NewDeployHandler(host, prefix string) (*DeployHandler, error) { - clientID := os.Getenv("CAMLI_GCE_CLIENTID") - if clientID == "" { - return nil, errors.New("Need an oauth2 client ID defined in CAMLI_GCE_CLIENTID") - } - clientSecret := os.Getenv("CAMLI_GCE_CLIENTSECRET") - if clientSecret == "" { - return nil, errors.New("Need an oauth2 client secret defined in CAMLI_GCE_CLIENTSECRET") - } - tpl, err := template.New("root").Parse(noTheme + tplHTML()) - if err != nil { - return nil, fmt.Errorf("could not parse template: %v", err) - } - host = strings.TrimSuffix(host, "/") - prefix = strings.TrimSuffix(prefix, "/") - scheme := "https" - xsrfKey := os.Getenv("CAMLI_GCE_XSRFKEY") - if xsrfKey == "" { - xsrfKey = auth.RandToken(20) - log.Printf("xsrf key not provided as env var CAMLI_GCE_XSRFKEY, so generating one instead: %v", xsrfKey) - } - instConf, instState, err := dataStores() - if err != nil { - return nil, fmt.Errorf("could not initialize conf or state storage: %v", err) - } - h := &DeployHandler{ - host: host, - xsrfKey: xsrfKey, - instConf: instConf, - instState: instState, - recordStateErr: make(map[string]error), - scheme: scheme, - prefix: prefix, - help: map[string]template.HTML{ - "machineTypes": template.HTML(helpMachineTypes), - "zones": template.HTML(helpZones), - }, - clientID: clientID, - clientSecret: clientSecret, - tpl: tpl, - piggyGIF: "/static/piggy.gif", - } - mux := http.NewServeMux() - mux.HandleFunc(prefix+"/callback", func(w http.ResponseWriter, r *http.Request) { - h.serveCallback(w, r) - }) - mux.HandleFunc(prefix+"/instance", func(w http.ResponseWriter, r *http.Request) { - h.serveInstanceState(w, r) - }) - mux.HandleFunc(prefix+"/", func(w http.ResponseWriter, r *http.Request) { - h.serveRoot(w, r) - }) - h.mux = mux - h.SetLogger(log.New(os.Stderr, "GCE DEPLOYER: ", log.LstdFlags)) - h.zones = backupZones - var refreshZonesFn func() - refreshZonesFn = func() { - if err := h.refreshZones(); err != nil { - h.logger.Printf("error while refreshing zones: %v", err) - } - time.AfterFunc(24*time.Hour, refreshZonesFn) - } - go refreshZonesFn() - var refreshCamliVersionFn func() - refreshCamliVersionFn = func() { - if err := h.refreshCamliVersion(); err != nil { - h.logger.Printf("error while refreshing Perkeep version: %v", err) - } - time.AfterFunc(time.Hour, refreshCamliVersionFn) - } - go refreshCamliVersionFn() - return h, nil -} - -func (h *DeployHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if h.mux == nil { - http.Error(w, "handler not properly initialized", http.StatusInternalServerError) - return - } - h.mux.ServeHTTP(w, r) -} - -func (h *DeployHandler) SetScheme(scheme string) { h.scheme = scheme } - -// authenticatedClient returns the GCE project running the /launch/ -// app (e.g. "camlistore-website" usually for the main instance) and -// an authenticated OAuth2 client acting as that service account. -// This is only used for refreshing the list of valid zones to give to -// the user in a drop-down. - -// If we're not running on GCE (e.g. dev mode on localhost) and have -// no other way to get the info, the error value is is errNoRefresh. -func (h *DeployHandler) authenticatedClient() (project string, hc *http.Client, err error) { - var ctx = context.Background() - - project = os.Getenv("CAMLI_GCE_PROJECT") - accountFile := os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT") - if project != "" && accountFile != "" { - data, errr := os.ReadFile(accountFile) - err = errr - if err != nil { - return - } - jwtConf, errr := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/compute.readonly") - err = errr - if err != nil { - return - } - hc = jwtConf.Client(ctx) - return - } - if !metadata.OnGCE() { - err = errNoRefresh - return - } - project, _ = metadata.ProjectID() - hc, err = google.DefaultClient(ctx) - return project, hc, err -} - -var gitRevRgx = regexp.MustCompile(`^[a-z0-9]{40}?$`) - -func (h *DeployHandler) refreshCamliVersion() error { - h.camliVersionMu.Lock() - defer h.camliVersionMu.Unlock() - resp, err := http.Get("https://storage.googleapis.com/camlistore-release/docker/VERSION") - if err != nil { - return err - } - defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - version := strings.TrimSpace(string(data)) - if !gitRevRgx.MatchString(version) { - return fmt.Errorf("wrong revision format in VERSION file: %q", version) - } - h.camliVersion = version - return nil -} - -func (h *DeployHandler) camliRev() string { - h.camliVersionMu.RLock() - defer h.camliVersionMu.RUnlock() - return h.camliVersion -} - -var errNoRefresh error = errors.New("not on GCE, and at least one of CAMLI_GCE_PROJECT or CAMLI_GCE_SERVICE_ACCOUNT not defined") - -func (h *DeployHandler) refreshZones() error { - h.zonesMu.Lock() - defer h.zonesMu.Unlock() - defer func() { - h.regions = make([]string, 0, len(h.zones)) - for r := range h.zones { - h.regions = append(h.regions, r) - } - }() - project, hc, err := h.authenticatedClient() - if err != nil { - if err == errNoRefresh { - h.zones = backupZones - h.logger.Printf("Cannot refresh zones. Using hard-coded ones instead.") - return nil - } - return err - } - s, err := compute.New(hc) - if err != nil { - return err - } - rl, err := compute.NewRegionsService(s).List(project).Do() - if err != nil { - return fmt.Errorf("could not get a list of regions: %v", err) - } - h.zones = make(map[string][]string) - for _, r := range rl.Items { - zones := make([]string, 0, len(r.Zones)) - for _, z := range r.Zones { - zone := path.Base(z) - if zone == "europe-west1-a" { - // Because even though the docs mark it as deprecated, it still shows up here, go figure. - continue - } - zone = strings.Replace(zone, r.Name, "", 1) - zones = append(zones, zone) - } - h.zones[r.Name] = zones - } - return nil -} - -func (h *DeployHandler) zoneValues() []string { - h.zonesMu.RLock() - defer h.zonesMu.RUnlock() - return h.regions -} - -// if there's project as a query parameter, it means we've just created a -// project for them and we're redirecting them to the form, but with the projectID -// field pre-filled for them this time. -func (h *DeployHandler) serveRoot(w http.ResponseWriter, r *http.Request) { - if r.Method == "POST" { - h.serveSetup(w, r) - return - } - _, err := r.Cookie("user") - if err != nil { - http.SetCookie(w, newCookie()) - } - camliRev := h.camliRev() - if r.FormValue("WIP") == "1" { - camliRev = "WORKINPROGRESS" - } - - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if err := h.tpl.ExecuteTemplate(w, "withform", &TemplateData{ - ProjectID: r.FormValue("project"), - Prefix: h.prefix, - Help: h.help, - ZoneValues: h.zoneValues(), - MachineValues: machineValues, - CamliVersion: camliRev, - }); err != nil { - h.logger.Print(err) - } -} - -func (h *DeployHandler) serveSetup(w http.ResponseWriter, r *http.Request) { - if r.FormValue("mode") != "setupproject" { - h.serveError(w, r, errors.New("bad form")) - return - } - ck, err := r.Cookie("user") - if err != nil { - h.serveFormError(w, errors.New("cookie expired, or CSRF attempt. Please reload and retry")) - h.logger.Printf("Cookie expired, or CSRF attempt on form.") - return - } - - ctx := r.Context() - instConf, err := h.confFromForm(r) - if err != nil { - h.serveFormError(w, err) - return - } - - br, err := h.storeInstanceConf(ctx, instConf) - if err != nil { - h.serveError(w, r, fmt.Errorf("could not store instance configuration: %v", err)) - return - } - - xsrfToken := xsrftoken.Generate(h.xsrfKey, ck.Value, br.String()) - state := fmt.Sprintf("%s:%x", br.String(), xsrfToken) - redirectURL := h.oAuthConfig().AuthCodeURL(state) - http.Redirect(w, r, redirectURL, http.StatusFound) - return -} - -func (h *DeployHandler) serveCallback(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - ck, err := r.Cookie("user") - if err != nil { - http.Error(w, - fmt.Sprintf("Cookie expired, or CSRF attempt. Restart from %s://%s%s", h.scheme, h.host, h.prefix), - http.StatusBadRequest) - h.logger.Printf("Cookie expired, or CSRF attempt on callback.") - return - } - code := r.FormValue("code") - if code == "" { - h.serveError(w, r, errors.New("No oauth code parameter in callback URL")) - return - } - h.logger.Printf("successful authentication: %v", r.URL.RawQuery) - - br, tk, err := fromState(r) - if err != nil { - h.serveError(w, r, err) - return - } - if !xsrftoken.Valid(tk, h.xsrfKey, ck.Value, br.String()) { - h.serveError(w, r, fmt.Errorf("Invalid xsrf token: %q", tk)) - return - } - - oAuthConf := h.oAuthConfig() - tok, err := oAuthConf.Exchange(ctx, code) - if err != nil { - h.serveError(w, r, fmt.Errorf("could not obtain a token: %v", err)) - return - } - h.logger.Printf("successful authorization with token: %v", tok) - - instConf, err := h.instanceConf(ctx, br) - if err != nil { - h.serveError(w, r, err) - return - } - - depl := &Deployer{ - Client: oAuthConf.Client(ctx, tok), - Conf: instConf, - Logger: h.logger, - } - - // They've requested that we create a project for them. - if instConf.CreateProject { - // So we try to do so. - projectID, err := depl.CreateProject(ctx) - if err != nil { - h.logger.Printf("error creating project: %v", err) - // TODO(mpl): we log the errors, but none of them are - // visible to the user (they just get a 500). I should - // probably at least detect and report them the project - // creation quota errors. - h.serveError(w, r, err) - return - } - // And serve the form again if we succeeded. - http.Redirect(w, r, fmt.Sprintf("%s/?project=%s", h.prefix, projectID), http.StatusFound) - return - } - - if found := h.serveOldInstance(w, br, depl); found { - return - } - - if err := h.recordState(br, &creationState{ - InstConf: br, - }); err != nil { - h.serveError(w, r, err) - return - } - - go func() { - inst, err := depl.Create(context.Background()) - state := &creationState{ - InstConf: br, - } - if err != nil { - h.logger.Printf("could not create instance: %v", err) - switch e := err.(type) { - case instanceExistsError: - state.Err = fmt.Sprintf("%v %v", e, helpDeleteInstance) - case projectIDError: - state.Err = fmt.Sprintf("%v", e) - default: - state.Err = fmt.Sprintf("%v. %v", err, fileIssue(br.String())) - } - } else { - state.InstAddr = addr(inst) - state.Success = true - if instConf.Hostname != "" { - state.InstHostname = instConf.Hostname - } - } - if err := h.recordState(br, state); err != nil { - h.logger.Printf("Could not record creation state for %v: %v", br, err) - h.recordStateErrMu.Lock() - defer h.recordStateErrMu.Unlock() - h.recordStateErr[br.String()] = err - return - } - if state.Err != "" { - return - } - if instConf.Hostname != "" { - return - } - // We also try to get the "camlistore-hostname" from the - // instance, so we can tell the user what their hostname is. It can - // take a while as perkeepd itself sets it after it has - // registered with the camlistore.net DNS. - giveupTime := time.Now().Add(time.Hour) - pause := time.Second - for { - hostname, err := depl.getInstanceAttribute("camlistore-hostname") - if err != nil && err != errAttrNotFound { - h.logger.Printf("could not get camlistore-hostname of instance: %v", err) - state.Success = false - state.Err = fmt.Sprintf("could not get camlistore-hostname of instance: %v", err) - break - } - if err == nil { - state.InstHostname = hostname - break - } - if time.Now().After(giveupTime) { - h.logger.Printf("Giving up on getting camlistore-hostname of instance") - state.Success = false - state.Err = fmt.Sprintf("could not get camlistore-hostname of instance") - break - } - time.Sleep(pause) - pause *= 2 - } - if err := h.recordState(br, state); err != nil { - h.logger.Printf("Could not record hostname for %v: %v", br, err) - h.recordStateErrMu.Lock() - defer h.recordStateErrMu.Unlock() - h.recordStateErr[br.String()] = err - return - } - }() - h.serveProgress(w, br) -} - -// serveOldInstance looks on GCE for an instance such as defined in depl.Conf, and if -// found, serves the appropriate page depending on whether the instance is usable. It does -// not serve anything if the instance is not found. -func (h *DeployHandler) serveOldInstance(w http.ResponseWriter, br blob.Ref, depl *Deployer) (found bool) { - inst, err := depl.Get() - if err != nil { - // TODO(mpl,bradfitz): log or do something more - // drastic if the error is something other than - // instance not found. - return false - } - h.logger.Printf("Reusing existing instance for (%v, %v, %v)", depl.Conf.Project, depl.Conf.Name, depl.Conf.Zone) - - if err := h.recordState(br, &creationState{ - InstConf: br, - InstAddr: addr(inst), - Exists: true, - }); err != nil { - h.logger.Printf("Could not record creation state for %v: %v", br, err) - h.serveErrorPage(w, fmt.Errorf("An error occurred while recording the state of your instance. %v", fileIssue(br.String()))) - return true - } - h.serveProgress(w, br) - return true -} - -func (h *DeployHandler) serveFormError(w http.ResponseWriter, err error, hints ...string) { - var topHints []string - topHints = append(topHints, hints...) - h.logger.Print(err) - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if tplErr := h.tpl.ExecuteTemplate(w, "withform", &TemplateData{ - Prefix: h.prefix, - Help: h.help, - Err: err, - Hints: topHints, - ZoneValues: h.zoneValues(), - MachineValues: machineValues, - }); tplErr != nil { - h.logger.Printf("Could not serve form error %q because: %v", err, tplErr) - } -} - -func fileIssue(br string) string { - return fmt.Sprintf("Please file an issue with your instance key (%v) at https://perkeep.org/issue", br) -} - -// serveInstanceState serves the state of the requested Google Cloud Engine VM creation -// process. If the operation was successful, it serves a success page. If it failed, it -// serves an error page. If it isn't finished yet, it replies with "running". -func (h *DeployHandler) serveInstanceState(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - if r.Method != "GET" { - h.serveError(w, r, fmt.Errorf("Wrong method: %v", r.Method)) - return - } - br := r.URL.Query().Get("instancekey") - stateValue, err := h.instState.Get(br) - if err != nil { - http.Error(w, "unknown instance", http.StatusNotFound) - return - } - var state creationState - if err := json.Unmarshal([]byte(stateValue), &state); err != nil { - h.serveError(w, r, fmt.Errorf("could not json decode instance state: %v", err)) - return - } - if state.Err != "" { - // No need to log that error here since we're already doing it in serveCallback - h.serveErrorPage(w, fmt.Errorf("an error occurred while creating your instance: %v", state.Err)) - return - } - if state.Success || state.Exists { - conf, err := h.instanceConf(ctx, state.InstConf) - if err != nil { - h.logger.Printf("Could not get parameters for success message: %v", err) - h.serveErrorPage(w, fmt.Errorf("your instance was created and should soon be up at https://%s but there might have been a problem in the creation process. %v", state.Err, fileIssue(br))) - return - } - h.serveSuccess(w, &TemplateData{ - Prefix: h.prefix, - Help: h.help, - InstanceIP: state.InstAddr, - InstanceHostname: state.InstHostname, - ProjectConsoleURL: fmt.Sprintf("%s/project/%s/compute", ConsoleURL, conf.Project), - Conf: conf, - ZoneValues: h.zoneValues(), - MachineValues: machineValues, - }) - return - } - h.recordStateErrMu.RLock() - defer h.recordStateErrMu.RUnlock() - if _, ok := h.recordStateErr[br]; ok { - // No need to log that error here since we're already doing it in serveCallback - h.serveErrorPage(w, fmt.Errorf("An error occurred while recording the state of your instance. %v", fileIssue(br))) - return - } - fmt.Fprintf(w, "running") -} - -// serveProgress serves a page with some javascript code that regularly queries -// the server about the progress of the requested Google Cloud Engine VM creation. -// The server replies through serveInstanceState. -func (h *DeployHandler) serveProgress(w http.ResponseWriter, instanceKey blob.Ref) { - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if err := h.tpl.ExecuteTemplate(w, "withform", &TemplateData{ - Prefix: h.prefix, - InstanceKey: instanceKey.String(), - PiggyGIF: h.piggyGIF, - }); err != nil { - h.logger.Printf("Could not serve progress: %v", err) - } -} - -func (h *DeployHandler) serveErrorPage(w http.ResponseWriter, err error, hints ...string) { - var topHints []string - topHints = append(topHints, hints...) - h.logger.Print(err) - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if tplErr := h.tpl.ExecuteTemplate(w, "noform", &TemplateData{ - Prefix: h.prefix, - Err: err, - Hints: topHints, - }); tplErr != nil { - h.logger.Printf("Could not serve error %q because: %v", err, tplErr) - } -} - -func (h *DeployHandler) serveSuccess(w http.ResponseWriter, data *TemplateData) { - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if err := h.tpl.ExecuteTemplate(w, "noform", data); err != nil { - h.logger.Printf("Could not serve success: %v", err) - } -} - -func newCookie() *http.Cookie { - expiration := cookieExpiration - return &http.Cookie{ - Name: "user", - Value: auth.RandToken(15), - Expires: time.Now().Add(expiration), - } -} - -func formValueOrDefault(r *http.Request, formField, defValue string) string { - val := r.FormValue(formField) - if val == "" { - return defValue - } - return val -} - -func (h *DeployHandler) confFromForm(r *http.Request) (*InstanceConf, error) { - newProject, err := strconv.ParseBool(r.FormValue("newproject")) - if err != nil { - return nil, fmt.Errorf("could not convert \"newproject\" value to bool: %v", err) - } - var projectID string - if newProject { - projectID = r.FormValue("newprojectid") - } else { - projectID = r.FormValue("projectid") - if projectID == "" { - return nil, errors.New("missing project ID parameter") - } - } - var zone string - zoneReg := formValueOrDefault(r, "zone", DefaultRegion) - if LooksLikeRegion(zoneReg) { - region := zoneReg - zone = h.randomZone(region) - } else if strings.Count(zoneReg, "-") == 2 { - zone = zoneReg - } else { - return nil, errors.New("invalid zone or region") - } - isFreeTier, err := strconv.ParseBool(formValueOrDefault(r, "freetier", "false")) - if err != nil { - return nil, fmt.Errorf("could not convert \"freetier\" value to bool: %v", err) - } - machine := formValueOrDefault(r, "machine", DefaultMachineType) - if isFreeTier { - if !strings.HasPrefix(zone, "us-") { - return nil, fmt.Errorf("The %v zone was selected, but the Google Cloud Free Tier is only available for US zones", zone) - } - if machine != "f1-micro" { - return nil, fmt.Errorf("The %v machine type was selected, but the Google Cloud Free Tier is only available for f1-micro", machine) - } - } - return &InstanceConf{ - CreateProject: newProject, - Name: formValueOrDefault(r, "name", DefaultInstanceName), - Project: projectID, - Machine: machine, - Zone: zone, - Hostname: formValueOrDefault(r, "hostname", ""), - Ctime: time.Now(), - WIP: r.FormValue("WIP") == "1", - }, nil -} - -// randomZone picks one of the zone suffixes for region and returns it -// appended to region, as a fully-qualified zone name. -// If the given region is invalid, the default Zone is returned instead. -func (h *DeployHandler) randomZone(region string) string { - h.zonesMu.RLock() - defer h.zonesMu.RUnlock() - zones, ok := h.zones[region] - if !ok { - return fallbackZone - } - return region + zones[rand.Intn(len(zones))] -} - -func (h *DeployHandler) SetLogger(lg *log.Logger) { - h.logger = lg -} - -func (h *DeployHandler) serveError(w http.ResponseWriter, r *http.Request, err error) { - if h.logger != nil { - h.logger.Printf("%v", err) - } - httputil.ServeError(w, r, err) -} - -func (h *DeployHandler) oAuthConfig() *oauth2.Config { - oauthConfig := NewOAuthConfig(h.clientID, h.clientSecret) - oauthConfig.RedirectURL = fmt.Sprintf("%s://%s%s/callback", h.scheme, h.host, h.prefix) - return oauthConfig -} - -// fromState parses the oauth state parameter from r to extract the blobRef of the -// instance configuration and the xsrftoken that were stored during serveSetup. -func fromState(r *http.Request) (br blob.Ref, xsrfToken string, err error) { - params := strings.Split(r.FormValue("state"), ":") - if len(params) != 2 { - return br, "", fmt.Errorf("Invalid format for state parameter: %q, wanted blobRef:xsrfToken", r.FormValue("state")) - } - br, ok := blob.Parse(params[0]) - if !ok { - return br, "", fmt.Errorf("Invalid blobRef in state parameter: %q", params[0]) - } - token, err := hex.DecodeString(params[1]) - if err != nil { - return br, "", fmt.Errorf("can't decode hex xsrftoken %q: %v", params[1], err) - } - return br, string(token), nil -} - -func (h *DeployHandler) storeInstanceConf(ctx context.Context, conf *InstanceConf) (blob.Ref, error) { - contents, err := json.Marshal(conf) - if err != nil { - return blob.Ref{}, fmt.Errorf("could not json encode instance config: %v", err) - } - hash := blob.NewHash() - _, err = io.Copy(hash, bytes.NewReader(contents)) - if err != nil { - return blob.Ref{}, fmt.Errorf("could not hash blob contents: %v", err) - } - br := blob.RefFromHash(hash) - if _, err := blobserver.Receive(ctx, h.instConf, br, bytes.NewReader(contents)); err != nil { - return blob.Ref{}, fmt.Errorf("could not store instance config blob: %v", err) - } - return br, nil -} - -func (h *DeployHandler) instanceConf(ctx context.Context, br blob.Ref) (*InstanceConf, error) { - rc, _, err := h.instConf.Fetch(ctx, br) - if err != nil { - return nil, fmt.Errorf("could not fetch conf at %v: %v", br, err) - } - defer rc.Close() - contents, err := io.ReadAll(rc) - if err != nil { - return nil, fmt.Errorf("could not read conf in blob %v: %v", br, err) - } - var instConf InstanceConf - if err := json.Unmarshal(contents, &instConf); err != nil { - return nil, fmt.Errorf("could not json decode instance config: %v", err) - } - return &instConf, nil -} - -func (h *DeployHandler) recordState(br blob.Ref, state *creationState) error { - val, err := json.Marshal(state) - if err != nil { - return fmt.Errorf("could not json encode instance state: %v", err) - } - if err := h.instState.Set(br.String(), string(val)); err != nil { - return fmt.Errorf("could not record instance state: %v", err) - } - return nil -} - -func addr(inst *compute.Instance) string { - if inst == nil { - return "" - } - if len(inst.NetworkInterfaces) == 0 || inst.NetworkInterfaces[0] == nil { - return "" - } - if len(inst.NetworkInterfaces[0].AccessConfigs) == 0 || inst.NetworkInterfaces[0].AccessConfigs[0] == nil { - return "" - } - return inst.NetworkInterfaces[0].AccessConfigs[0].NatIP -} - -// creationState keeps information all along the creation process of the instance. The -// fields are only exported because we json encode them. -type creationState struct { - Err string `json:",omitempty"` // if non blank, creation failed. - InstConf blob.Ref // key to the user provided instance configuration. - InstAddr string // ip address of the instance. - InstHostname string // hostame (in the camlistore.net domain) of the instance - Success bool // whether new instance creation was successful. - Exists bool // true if an instance with same zone, same project name, and same instance name already exists. -} - -// dataStores returns the blobserver that stores the instances configurations, and the kv -// store for the instances states. -func dataStores() (blobserver.Storage, sorted.KeyValue, error) { - dataDir := os.Getenv("CAMLI_GCE_DATA") - if dataDir == "" { - var err error - dataDir, err = os.MkdirTemp("", "camli-gcedeployer-data") - if err != nil { - return nil, nil, err - } - log.Printf("data dir not provided as env var CAMLI_GCE_DATA, so defaulting to %v", dataDir) - } - blobsDir := filepath.Join(dataDir, "instance-conf") - if err := os.MkdirAll(blobsDir, 0700); err != nil { - return nil, nil, err - } - instConf, err := localdisk.New(blobsDir) - if err != nil { - return nil, nil, err - } - instState, err := leveldb.NewStorage(filepath.Join(dataDir, "instance-state")) - if err != nil { - return nil, nil, err - } - return instConf, instState, nil -} - -// TODO(mpl): AddTemplateTheme is a mistake, since the text argument is user -// input and hence can contain just any field, that is not a known field of -// TemplateData. Which will make the execution of the template fail. We should -// probably just somehow hardcode website/tmpl/page.html as the template. -// Or better, probably hardcode our own version of website/tmpl/page.html, -// because we don't want to be bound to whatever new template fields the -// website may need that we don't (such as .Domain). -// See issue #815, and issue #985. - -// AddTemplateTheme allows to enhance the aesthetics of the default template. To that -// effect, text can provide the template definitions for "header", "banner", "toplinks", and -// "footer". -func (h *DeployHandler) AddTemplateTheme(text string) error { - tpl, err := template.New("root").Parse(text + tplHTML()) - if err != nil { - return err - } - h.tplMu.Lock() - defer h.tplMu.Unlock() - h.tpl = tpl - return nil -} - -// TemplateData is the data passed for templates of tplHTML. -type TemplateData struct { - Title string - Help map[string]template.HTML // help bits within the form. - Hints []string // helping hints printed in case of an error. - Err error - Prefix string // handler prefix. - InstanceKey string // instance creation identifier, for the JS code to regularly poll for progress. - PiggyGIF string // URI to the piggy gif for progress animation. - Conf *InstanceConf // Configuration requested by the user - InstanceIP string `json:",omitempty"` // instance IP address that we display after successful creation. - InstanceHostname string `json:",omitempty"` - ProjectConsoleURL string - ProjectID string // set by us when we've just created a project on the behalf of the user - ZoneValues []string - MachineValues []string - CamliVersion string // git revision found in https://storage.googleapis.com/camlistore-release/docker/VERSION - - // Unused stuff, but needed by page.html. See TODO above, - // before AddTemplateTheme. - GoImportDomain string - GoImportUpstream string -} - -// empty definitions for "banner", "toplinks", and "footer" to avoid error on -// ExecuteTemplate when the definitions have not been added with AddTemplateTheme. -var noTheme = ` -{{define "header"}} - - Perkeep on Google Cloud - -{{end}} -{{define "banner"}} -{{end}} -{{define "toplinks"}} -{{end}} -{{define "footer"}} -{{end}} -` - -func tplHTML() string { - return ` - {{define "progress"}} - {{if .InstanceKey}} - - {{end}} - {{end}} - - {{define "messages"}} -
-

Perkeep on Google Cloud

- - {{if .InstanceIP}} - {{if .InstanceHostname}} -

Success. Your Perkeep instance is running at {{.InstanceHostname}}.

- {{else}} - -

Success. Your Perkeep instance is deployed at {{.InstanceIP}}. Refresh this page in a couple of minutes to know your hostname. Or go to camlistore-server instance, and look for camlistore-hostname (which might take a while to appear too) in the custom metadata section.

- {{end}} -

Please save the information on this page.

- -

First connection

-

- The password to access the web interface of your Perkeep instance was automatically generated. Go to the camlistore-server instance page to view it, and possibly change it. It is camlistore-password in the custom metadata section. Similarly, the username is camlistore-username. Then restart Perkeep from the /status page if you changed anything. -

- -

Further configuration

-

- Manage your instance at {{.ProjectConsoleURL}}. -

- -

- If you want to use your own HTTPS certificate and key, go to the storage browser. Delete "` + certFilename() + `", "` + keyFilename() + `", and replace them by uploading your own files (with the same names). Then restart Perkeep. -

- -

Perkeep should not require system -administration but to manage/add SSH keys, go to the camlistore-server -instance page. Scroll down to the SSH Keys section. Note that the -machine can be deleted or wiped at any time without losing -information. All state is stored in Cloud Storage. The index, however, -is stored in MySQL on the instance. The index can be rebuilt if lost -or corrupted.

- -

- {{end}} - {{if .Err}} -

Error: {{.Err}}

- {{range $hint := .Hints}} -

{{$hint}}

- {{end}} - {{end}} - {{end}} - -{{define "withform"}} - -{{template "header" .}} - - - - - - {{if .InstanceKey}} -
- {{end}} - {{template "banner" .}} - {{template "toplinks" .}} - {{template "progress" .}} - {{template "messages" .}} -
- - -

Deploy Perkeep

- -

This tool creates your own private -Perkeep instance running on Google Cloud Platform. Be sure to -understand Compute Engine pricing -and -Cloud Storage pricing -before proceeding. Note that Perkeep metadata adds overhead on top of the size -of any raw data added to your instance. To delete your -instance and stop paying Google for the virtual machine, visit the Google Cloud console -and visit both the "Compute Engine" and "Storage" sections for your project. -

- {{if .CamliVersion}} -

Perkeep version deployed by this tool: {{.CamliVersion}}

- {{end}} - - - - - - - - {{if .ProjectID}} - - {{else}} - - {{end}} - - -
Google Project ID: - {{if .ProjectID}} -
- Existing project ID:
- {{else}} -
- Existing project ID:
- {{end}} - You need to enable billing with Google for the selected project. -
Zone or Region: - {{if .ProjectID}} - - {{else}} - - {{end}} - - {{range $k, $v := .ZoneValues}} - - {{end}} -
If a region is specified, a random zone (-a, -b, -c, etc) in that region will be selected. -
Machine type: - {{if .ProjectID}} - - {{else}} - - {{end}} - - {{range $k, $v := .MachineValues}} - - {{end}} -
As of 2015-12-27, a g1-small is $13.88 (USD) per month, before storage usage charges. See current pricing. -
Use Google Cloud Free Tier
Use Google Cloud Free Tier
The Free Tier implies using an f1-micro instance, which might not be powerful enough in the long run. Also, the free storage is limited to 5GB, and must be in a US region. -
- {{if .ProjectID}} -
(it will ask for permissions) - {{else}} -
(it will ask for permissions) - {{end}} -
-
-
- {{template "footer" .}} - {{if .InstanceKey}} -
- {{end}} - - -{{end}} - -{{define "noform"}} - -{{template "header" .}} - - {{if .InstanceKey}} -
- {{end}} - {{template "banner" .}} - {{template "toplinks" .}} - {{template "progress" .}} - {{template "messages" .}} - {{template "footer" .}} - {{if .InstanceKey}} -
- {{end}} - - -{{end}} -` -} - -// TODO(bradfitz,mpl): move this to go4.org/cloud/google/gceutil -func ZonesOfRegion(hc *http.Client, project, region string) (zones []string, err error) { - s, err := compute.New(hc) - if err != nil { - return nil, err - } - zl, err := compute.NewZonesService(s).List(project).Do() - if err != nil { - return nil, fmt.Errorf("could not get a list of zones: %v", err) - } - if zl.NextPageToken != "" { - return nil, errors.New("TODO: more than one page of zones found; use NextPageToken") - } - for _, z := range zl.Items { - if path.Base(z.Region) != region { - continue - } - zones = append(zones, z.Name) - } - return zones, nil -} diff --git a/pkg/deploy/gce/notes.txt b/pkg/deploy/gce/notes.txt deleted file mode 100644 index e95ccee1c..000000000 --- a/pkg/deploy/gce/notes.txt +++ /dev/null @@ -1,31 +0,0 @@ -non-core dev: -gcutil --service_version="v1" --project="camanaged" addinstance "camlistore" --zone="us-central1-b" --machine_type="n1-standard-1" --network="default" --external_ip_address="107.178.214.163" --metadata="cam-key-1:cam-value-1" --metadata="cam-key-2:cam-value-2" --metadata="sshKeys:bradfitz:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw6Dk3iskKylP2zginCOAzIunMA38vGL9b/i18UG/Iuq+jKczZXB/1dlcZGSOs3+LtGh/C341TXTioydxTw+ux1AbmUk4c6L404skl85XFOys/GLxA4sHxBSb5we0Q57yohSgeZNlQd+Scmu5v7WC0N7I3hOK0lJgtxRNyC2nncGC0UOm+IGPTWcqPJERTauH/OhoAddWQehf1ugxTJYFU9atl3Op/mDXfyGBSLweWAQ84fhVKRZnl4i9Yhk1b357Q8cVKH6UQUADVamo7CQOsenzx99UL0thFRTSbuKALyf9e+SPwJrtIxZaX+skVSR+CzooRbypIamLbNXhfbxNz bradfitz@Bradleys-MacBook-Air.local" --service_account_scopes="https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/compute.readonly,https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/sqlservice,https://www.googleapis.com/auth/sqlservice.admin,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/datastore,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/compute,https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/taskqueue,https://www.googleapis.com/auth/bigquery,https://www.googleapis.com/auth/sqlservice,https://www.googleapis.com/auth/datastore" --tags="tag,tag2,http-server,https-server" --persistent_boot_disk="true" --auto_delete_boot_disk="false" --image=projects/debian-cloud/global/images/backports-debian-7-wheezy-v20140718 - -$ curl -H "Metadata-Flavor:Google" http://metadata/computeMetadata/v1/instance/service-accounts/default/scopes -https://www.googleapis.com/auth/bigquery -https://www.googleapis.com/auth/cloud-platform -https://www.googleapis.com/auth/compute -https://www.googleapis.com/auth/compute.readonly -https://www.googleapis.com/auth/datastore -https://www.googleapis.com/auth/devstorage.full_control -https://www.googleapis.com/auth/sqlservice -https://www.googleapis.com/auth/sqlservice.admin -https://www.googleapis.com/auth/taskqueue -https://www.googleapis.com/auth/userinfo.email - -gcutil --project=camanaged addinstance \ - --image=projects/coreos-cloud/global/images/coreos-alpha-394-0-0-v20140801 \ - --persistent_boot_disk \ - --zone=us-central1-a --machine_type=n1-standard-1 \ - --external_ip_address=107.178.208.16 \ - --auto_delete_boot_disk \ - --tags=http-server,https-server \ - --metadata_from_file=user-data:cloud-config.yaml core1 - -TODO: -- allow config from /gcs/bucket/key; add pkg for os.Stat/os.Open wrappers checking - prefix -- use that package for: - "httpsCert": "/home/bradfitz/keys/camlihouse/ssl.crt", - "httpsKey": "/home/bradfitz/keys/camlihouse/ssl.key", - "identitySecretRing": "/home/bradfitz/.config/perkeep/identity-secring.gpg", diff --git a/pkg/serverinit/env.go b/pkg/serverinit/env.go index 8f305e6c2..1544d7ead 100644 --- a/pkg/serverinit/env.go +++ b/pkg/serverinit/env.go @@ -85,7 +85,7 @@ func DefaultEnvConfig() (*Config, error) { externalIP, _ := metadata.ExternalIP() hostName, _ := metadata.InstanceAttributeValue("camlistore-hostname") - // If they specified a hostname (probably with pk-deploy), then: + // If they specified a hostname (previously common with old pk-deploy), then: // if it looks like an FQDN, perkeepd is going to rely on Let's // Encrypt, else perkeepd is going to generate some self-signed for that // hostname. diff --git a/website/pk-web/pkweb.go b/website/pk-web/pkweb.go index e45f125ed..c4000813c 100644 --- a/website/pk-web/pkweb.go +++ b/website/pk-web/pkweb.go @@ -21,7 +21,6 @@ import ( "context" "crypto/rand" "crypto/tls" - "encoding/json" "errors" "flag" "fmt" @@ -40,26 +39,19 @@ import ( txttemplate "text/template" "time" - "perkeep.org/internal/netutil" "perkeep.org/internal/osutil" "perkeep.org/pkg/buildinfo" - "perkeep.org/pkg/deploy/gce" "perkeep.org/pkg/types/camtypes" "cloud.google.com/go/compute/metadata" - "cloud.google.com/go/datastore" "cloud.google.com/go/logging" "cloud.google.com/go/storage" "github.com/mailgun/mailgun-go" "github.com/russross/blackfriday" - "go4.org/cloud/cloudlaunch" "go4.org/writerutil" "golang.org/x/crypto/acme/autocert" "golang.org/x/oauth2/google" - cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" - compute "google.golang.org/api/compute/v1" "google.golang.org/api/option" - storageapi "google.golang.org/api/storage/v1" ) const ( @@ -80,7 +72,6 @@ var ( tlsKeyFile = flag.String("tlskey", "", "TLS private key file") alsoRun = flag.String("also_run", "", "[optiona] Path to run as a child process. (Used to run perkeep.org's ./scripts/run-blob-server)") devMode = flag.Bool("dev", false, "in dev mode") - flagStaging = flag.Bool("staging", false, "Deploy to a test GCE instance. Requires -cloudlaunch=true") flagVersion = flag.Bool("version", false, "show version") gceProjectID = flag.String("gce_project_id", "", "GCE project ID; required if not running on GCE and gce_log_name is specified.") @@ -502,104 +493,6 @@ func runAsChild(res string) { }() } -func gceDeployHandlerConfig() (*gce.Config, error) { - if inProd { - return deployerCredsFromGCS() - } - clientId := os.Getenv("CAMLI_GCE_CLIENTID") - if clientId != "" { - return &gce.Config{ - ClientID: clientId, - ClientSecret: os.Getenv("CAMLI_GCE_CLIENTSECRET"), - Project: os.Getenv("CAMLI_GCE_PROJECT"), - ServiceAccount: os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT"), - DataDir: os.Getenv("CAMLI_GCE_DATA"), - }, nil - } - configDir, err := osutil.PerkeepConfigDir() - if err != nil { - return nil, err - } - configFile := filepath.Join(configDir, "launcher-config.json") - data, err := os.ReadFile(configFile) - if err != nil { - return nil, fmt.Errorf("error reading launcher-config.json (expected of type https://godoc.org/"+prodDomain+"/pkg/deploy/gce#Config): %v", err) - } - var config gce.Config - if err := json.Unmarshal(data, &config); err != nil { - return nil, err - } - return &config, nil -} - -var cloudLauncherEnabled = false - -// gceDeployHandler returns an http.Handler for a GCE launcher, -// configured to run at /prefix/ (the trailing slash can be omitted). -// The launcher is not initialized if: -// - in production, the launcher-config.json file is not found in the relevant bucket -// - neither CAMLI_GCE_CLIENTID is set, nor launcher-config.json is found in the -// camlistore server config dir. -func gceDeployHandler(prefix string) (*gce.DeployHandler, error) { - if !cloudLauncherEnabled { - return nil, errors.New("The Perkeep Cloud Launcher is no longer available.") - } - var hostPort string - var err error - scheme := "https" - if inProd { - if inStaging { - hostPort = stagingHostname + ":443" - } else { - hostPort = prodDomain + ":443" - } - } else { - addr := *httpsAddr - if *devMode && *httpsAddr == "" { - addr = *httpAddr - scheme = "http" - } - hostPort, err = netutil.ListenHostPort(addr) - if err != nil { - // the deploy handler needs to know its own - // hostname or IP for the oauth2 callback. - return nil, fmt.Errorf("invalid -https flag: %v", err) - } - } - config, err := gceDeployHandlerConfig() - if config == nil { - return nil, err - } - gceh, err := gce.NewDeployHandlerFromConfig(hostPort, prefix, config) - if err != nil { - return nil, fmt.Errorf("NewDeployHandlerFromConfig: %v", err) - } - - pageBytes, err := os.ReadFile(filepath.Join(*root, "tmpl", "page.html")) - if err != nil { - return nil, err - } - if err := gceh.AddTemplateTheme(string(pageBytes)); err != nil { - return nil, fmt.Errorf("AddTemplateTheme: %v", err) - } - gceh.SetScheme(scheme) - log.Printf("Starting Perkeep launcher on %s://%s%s", scheme, hostPort, prefix) - return gceh, nil -} - -var launchConfig = &cloudlaunch.Config{ - Name: "camweb", - BinaryBucket: prodBucket, - GCEProjectID: "camlistore-website", - Scopes: []string{ - storageapi.DevstorageFullControlScope, - compute.ComputeScope, - logging.WriteScope, - datastore.ScopeDatastore, - cloudresourcemanager.CloudPlatformScope, - }, -} - func checkInProduction() bool { if !metadata.OnGCE() { return false @@ -815,29 +708,6 @@ func projectID() string { return projID } -func initStaging() error { - if *flagStaging { - launchConfig.Name = stagingInstName - return nil - } - // If we are the instance that has just been deployed, we can't rely on - // *flagStaging, since there's no way to pass flags through launchConfig. - // And we need to know if we're a staging instance, so we can set - // launchConfig.Name properly before we get into restartLoop from - // MaybeDeploy. So we use our own instance name as a hint. - if !metadata.OnGCE() { - return nil - } - instName, err := metadata.InstanceName() - if err != nil { - return fmt.Errorf("Instance could not get its Instance Name: %v", err) - } - if instName == stagingInstName { - launchConfig.Name = stagingInstName - } - return nil -} - func main() { flag.Parse() if *flagVersion { @@ -845,10 +715,6 @@ func main() { buildinfo.Summary(), runtime.Version(), runtime.GOOS, runtime.GOARCH) return } - if err := initStaging(); err != nil { - log.Fatalf("Error setting up staging: %v", err) - } - launchConfig.MaybeDeploy() setProdFlags() if *root == "" { @@ -904,16 +770,9 @@ func main() { }) } - gceLauncher, err := gceDeployHandler("/launch/") - if err != nil { - log.Printf("Not installing GCE /launch/ handler: %v", err) - err := err - mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) { - http.Error(w, fmt.Sprintf("GCE launcher disabled: %v", err), 500) - }) - } else { - mux.Handle("/launch/", gceLauncher) - } + mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "GCE launcher no longer supported", 500) + }) var handler http.Handler = &redirectRootHandler{Handler: mux} if *logDir != "" || *logStdout { @@ -939,23 +798,6 @@ func main() { log.Fatalf("Failed to ping Google Cloud Logging: %v", err) } handler = NewLoggingHandler(handler, gceLogger{logc.Logger(*gceLogName)}) - if gceLauncher != nil { - var logc *logging.Client - if metadata.OnGCE() { - logc, err = logging.NewClient(ctx, projID) - } else { - logc, err = logging.NewClient(ctx, projID, option.WithHTTPClient(hc)) - } - if err != nil { - log.Fatal(err) - } - commonLabels := logging.CommonLabels(map[string]string{ - "from": "camli-gce-launcher", - }) - logger := logc.Logger(*gceLogName, commonLabels).StandardLogger(logging.Default) - logger.SetPrefix("launcher: ") - gceLauncher.SetLogger(logger) - } } emailErr := make(chan error) @@ -1092,18 +934,6 @@ func fromGCS(filename string) ([]byte, error) { return slurp(filename) } -func deployerCredsFromGCS() (*gce.Config, error) { - var cfg gce.Config - data, err := fromGCS("launcher-config.json") - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("could not JSON decode camli GCE launcher config: %v", err) - } - return &cfg, nil -} - var issueNum = regexp.MustCompile(`^/(?:issue|bug)s?(/\d*)?$`) // issueRedirect returns whether the request should be redirected to the