diff --git a/.github/workflows/tests-lint.yaml b/.github/workflows/tests-lint.yaml index 9f825095b..2404c2e56 100644 --- a/.github/workflows/tests-lint.yaml +++ b/.github/workflows/tests-lint.yaml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/lint" jobs: diff --git a/.github/workflows/tests-linux.yml b/.github/workflows/tests-linux.yml index 876d3b36c..3d53050af 100644 --- a/.github/workflows/tests-linux.yml +++ b/.github/workflows/tests-linux.yml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/linux" jobs: diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3431fa6ce..35ca82f08 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/macos" jobs: diff --git a/.github/workflows/tests-tidy.yaml b/.github/workflows/tests-tidy.yaml index 5dec275f8..b4a161708 100644 --- a/.github/workflows/tests-tidy.yaml +++ b/.github/workflows/tests-tidy.yaml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/tidy" jobs: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 933fbdcbe..3d8ef8a8e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,9 @@ -on: [push, pull_request] +on: + push: + branches: + - "main" + pull_request: + # all PRs on all branches name: "tests/windows" jobs: diff --git a/cmd/pk-deploy/camdeploy.go b/cmd/pk-deploy/camdeploy.go deleted file mode 100644 index 6f0fb1429..000000000 --- a/cmd/pk-deploy/camdeploy.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2014 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// The pk-deploy program deploys Perkeep on cloud computing platforms -// such as Google Compute Engine or Amazon EC2. -package main // import "perkeep.org/cmd/pk-deploy" - -import ( - "perkeep.org/pkg/cmdmain" -) - -func main() { - cmdmain.Main() -} diff --git a/cmd/pk-deploy/gce.go b/cmd/pk-deploy/gce.go deleted file mode 100644 index f6613ec9e..000000000 --- a/cmd/pk-deploy/gce.go +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2014 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bufio" - "context" - "flag" - "fmt" - "log" - "math/rand" - "os" - "strings" - - "perkeep.org/pkg/cmdmain" - "perkeep.org/pkg/deploy/gce" - - "go4.org/oauthutil" - "golang.org/x/oauth2" -) - -type gceCmd struct { - project string - zone string - machine string - instName string - verbose bool - hostname string - wip bool -} - -func init() { - cmdmain.RegisterMode("gce", func(flags *flag.FlagSet) cmdmain.CommandRunner { - cmd := new(gceCmd) - flags.StringVar(&cmd.project, "project", "", "Name of Project.") - flags.StringVar(&cmd.zone, "zone", gce.DefaultRegion, "GCE zone or region. If a region is given, a random zone in that region is selected. See https://cloud.google.com/compute/docs/zones") - flags.StringVar(&cmd.machine, "machine", gce.DefaultMachineType, "GCE machine type (e.g. n1-standard-1, f1-micro, g1-small); see https://cloud.google.com/compute/docs/machine-types") - flags.StringVar(&cmd.instName, "instance_name", gce.DefaultInstanceName, "Name of VM instance.") - flags.StringVar(&cmd.hostname, "hostname", "", "Optional hostname for the instance. If set, the custom metadata variable \"camlistore-hostname\" on the instance takes that value. Otherwise, perkeepd sets that variable to the hostname we get from the camliNet DNS.") - flags.BoolVar(&cmd.wip, "wip", false, "Developer option. Deploy the WORKINPROGRESS perkeepd tarball.") - flags.BoolVar(&cmd.verbose, "verbose", false, "Be verbose.") - return cmd - }) -} - -const ( - clientIdDat = "client-id.dat" - clientSecretDat = "client-secret.dat" - helpEnableAuth = `Enable authentication: in your project console, navigate to "APIs and auth", "Credentials", click on "Create new Client ID", and pick "Installed application", with type "Other". Copy the CLIENT ID to ` + clientIdDat + `, and the CLIENT SECRET to ` + clientSecretDat - helpCreateProject = "Go to " + gce.ConsoleURL + " to create a new Google Cloud project" - helpEnableAPIs = `Enable the project APIs: in your project console, navigate to "APIs and auth", "APIs". In the list, enable "Google Cloud Storage", "Google Cloud Storage", "Google Cloud Storage JSON API", and "Google Compute Engine".` -) - -func (c *gceCmd) Describe() string { - return "Deploy Perkeep on Google Compute Engine." -} - -func (c *gceCmd) Usage() { - fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n %s\n\n", - "pk-deploy gce --project= --hostname= [options]", - "pk-deploy gce --project= --cert= --key= [options]") - flag.PrintDefaults() - fmt.Fprintln(os.Stderr, "\nTo get started:") - printHelp() -} - -func printHelp() { - for _, v := range []string{helpCreateProject, helpEnableAuth, helpEnableAPIs} { - fmt.Fprintf(os.Stderr, "%v\n", v) - } -} - -func (c *gceCmd) RunCommand(args []string) error { - if c.verbose { - gce.Verbose = true - } - if c.project == "" { - return cmdmain.UsageError("Missing --project flag.") - } - - // We embed the client ID and client secret, per - // https://developers.google.com/identity/protocols/OAuth2InstalledApp - // Notably: "The client ID and client secret obtained from the - // Developers Console are embedded in the source code of your - // application. In this context, the client secret is - // obviously not treated as a secret." - // - // These were created at: - // https://console.developers.google.com/apis/credentials?project=camlistore-website - // (Notes for Brad and Mathieu) - const ( - clientID = "574004351801-9qqoggh6b5v3jqt722v43ikmgmtv60h3.apps.googleusercontent.com" - clientSecret = "Gf1zwaOcbJnRTE5zD4feKaTI" // NOT a secret, despite name - ) - config := gce.NewOAuthConfig(clientID, clientSecret) - config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob" - - hc := oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{ - Config: config, - CacheFile: c.project + "-token.json", - AuthCode: func() string { - fmt.Println("Get auth code from:") - fmt.Printf("%v\n\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce)) - fmt.Print("Enter auth code: ") - sc := bufio.NewScanner(os.Stdin) - sc.Scan() - return strings.TrimSpace(sc.Text()) - }, - })) - - zone := c.zone - if gce.LooksLikeRegion(zone) { - region := zone - zones, err := gce.ZonesOfRegion(hc, c.project, region) - if err != nil { - return err - } - if len(zones) == 0 { - return fmt.Errorf("no zones found in region %q; invalid region?", region) - } - zone = zones[rand.Intn(len(zones))] - } - - instConf := &gce.InstanceConf{ - Name: c.instName, - Project: c.project, - Machine: c.machine, - Zone: zone, - Hostname: c.hostname, - WIP: c.wip, - } - - log.Printf("Creating instance %s (in project %s) in zone %s ...", c.instName, c.project, zone) - depl := &gce.Deployer{ - Client: hc, - Conf: instConf, - Logger: log.New(cmdmain.Stderr, "", log.Flags()), - } - inst, err := depl.Create(context.Background()) - if err != nil { - return err - } - - log.Printf("Instance created; starting up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP) - return nil -} diff --git a/doc/environment-vars.md b/doc/environment-vars.md index ad1a756b4..81118d1d8 100644 --- a/doc/environment-vars.md +++ b/doc/environment-vars.md @@ -70,44 +70,6 @@ or `CAMLI_FORCE_OSARCH` (bool) Used by make.go to force building an unrecommended OS/ARCH pair. -`CAMLI_GCE_\*` -: Variables prefixed with `CAMLI_GCE_` concern the Google Compute Engine deploy - handler in [pkg/deploy/gce](/pkg/deploy/gce), which is only used by camweb to - launch Perkeep on Google Compute Engine. They do not affect Perkeep's - behaviour. - -`CAMLI_GCE_CLIENTID` (string) -: See `CAMLI_GCE_*` first. This string is used by gce.DeployHandler as the - application's OAuth Client ID. If blank, camweb does not enable the Google - Compute Engine launcher. - -`CAMLI_GCE_CLIENTSECRET` (string) -: See `CAMLI_GCE_*` first. Used by gce.DeployHandler as the application's OAuth - Client Secret. If blank, gce.NewDeployHandler returns an error, and camweb - fails to start if the Google Compute Engine launcher was enabled. - -`CAMLI_GCE_DATA` (string) -: See `CAMLI_GCE_*` first. Path to the directory where gce.DeployHandler stores - the instances configuration and state. If blank, the "camli-gce-data" default - is used instead. - -`CAMLI_GCE_PROJECT` (string) -: See `CAMLI_GCE_*` first. ID of the Google Project that provides the above - client ID and secret. It is used when we query for the list of all the - existing zones, since such a query requires a project ID. If blank, a - hard-coded list of zones is used instead. - -`CAMLI_GCE_SERVICE_ACCOUNT` (string) -: See `CAMLI_GCE_*` first. Path to a Google service account JSON file. This - account should have at least compute.readonly permissions on the Google - Project with ID CAMLI_GCE_PROJECT. It is used to authenticate when querying - for the list of all the existing zones. If blank, a hard-coded list of zones - is used instead. - -`CAMLI_GCE_XSRFKEY` (string) -: See `CAMLI_GCE_*` first. Used by gce.DeployHandler as the XSRF protection key. - If blank, gce.NewDeployHandler generates a new random key instead. - `CAMLI_GOPHERJS_GOROOT` (string) : As gopherjs does not build with go tip, when make.go is run with go devel, CAMLI_GOPHERJS_GOROOT should be set to a Go 1.10 root so that gopherjs can be diff --git a/make.go b/make.go index 9b4f51721..f9708c16a 100644 --- a/make.go +++ b/make.go @@ -104,7 +104,6 @@ func main() { "perkeep.org/cmd/pk-get", "perkeep.org/cmd/pk-put", "perkeep.org/cmd/pk", - "perkeep.org/cmd/pk-deploy", "perkeep.org/server/perkeepd", "perkeep.org/app/hello", "perkeep.org/app/publisher", diff --git a/pkg/deploy/gce/cloud-config.yaml b/pkg/deploy/gce/cloud-config.yaml deleted file mode 100644 index 78c4ebb95..000000000 --- a/pkg/deploy/gce/cloud-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -#cloud-config - -coreos: - units: - - name: hello-docker.service - command: start - content: | - [Unit] - Description=Camlistore - After=docker.service - Requires=docker.service - - [Service] - ExecStart=/usr/bin/docker run -p 80:80 camlistore/hello - RestartSec=500ms - Restart=always - - [Install] - WantedBy=multi-user.target diff --git a/pkg/deploy/gce/deploy.go b/pkg/deploy/gce/deploy.go deleted file mode 100644 index 359f2227f..000000000 --- a/pkg/deploy/gce/deploy.go +++ /dev/null @@ -1,917 +0,0 @@ -/* -Copyright 2014 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package gce provides tools to deploy Perkeep on Google Compute Engine. -package gce // import "perkeep.org/pkg/deploy/gce" - -import ( - "context" - "crypto/rand" - "encoding/json" - "errors" - "fmt" - "log" - "math/big" - "net/http" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "perkeep.org/internal/osutil" - - "cloud.google.com/go/logging" - "go4.org/cloud/google/gceutil" - "go4.org/syncutil" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" - compute "google.golang.org/api/compute/v1" - "google.golang.org/api/googleapi" - servicemanagement "google.golang.org/api/servicemanagement/v1" - storage "google.golang.org/api/storage/v1" -) - -const ( - DefaultInstanceName = "camlistore-server" - DefaultMachineType = "g1-small" - DefaultRegion = "us-central1" - - projectsAPIURL = "https://www.googleapis.com/compute/v1/projects/" - - fallbackZone = "us-central1-a" - - camliUsername = "camlistore" // directly set in compute metadata, so not user settable. - - configDir = "config" - - ConsoleURL = "https://console.developers.google.com" - helpDeleteInstance = `To delete an existing Compute Engine instance: in your project console, navigate to "Compute", "Compute Engine", and "VM instances". Select your instance and click "Delete".` -) - -var ( - // Verbose enables more info to be printed. - Verbose bool -) - -// certFilename returns the HTTPS certificate file name -func certFilename() string { - return filepath.Base(osutil.DefaultTLSCert()) -} - -// keyFilename returns the HTTPS key name -func keyFilename() string { - return filepath.Base(osutil.DefaultTLSKey()) -} - -// NewOAuthConfig returns an OAuth configuration template. -func NewOAuthConfig(clientID, clientSecret string) *oauth2.Config { - return &oauth2.Config{ - Scopes: []string{ - logging.WriteScope, - compute.DevstorageFullControlScope, - compute.ComputeScope, - cloudresourcemanager.CloudPlatformScope, - servicemanagement.CloudPlatformScope, - "https://www.googleapis.com/auth/sqlservice", - "https://www.googleapis.com/auth/sqlservice.admin", - }, - Endpoint: google.Endpoint, - ClientID: clientID, - ClientSecret: clientSecret, - } -} - -// InstanceConf is the configuration for the Google Compute Engine instance that will be deployed. -type InstanceConf struct { - Name string // Name given to the virtual machine instance. - Project string // Google project ID where the instance is created. - CreateProject bool // CreateProject defines whether to first create project. - Machine string // Machine type. - Zone string // GCE zone; see https://cloud.google.com/compute/docs/zones - Hostname string // Fully qualified domain name. - - configDir string // bucketBase() + "/config" - blobDir string // bucketBase() + "/blobs" - - Ctime time.Time // Timestamp for this configuration. - - WIP bool // Whether to use the perkeepd-WORKINPROGRESS.tar.gz tarball instead of the "production" one -} - -// Deployer creates and starts an instance such as defined in Conf. -type Deployer struct { - // Client is an OAuth2 client, authenticated for working with - // the user's Google Cloud resources. - Client *http.Client - - Conf *InstanceConf - - *log.Logger // Cannot be nil. -} - -// Get returns the Instance corresponding to the Project, Zone, and Name defined in the -// Deployer's Conf. -func (d *Deployer) Get() (*compute.Instance, error) { - computeService, err := compute.New(d.Client) - if err != nil { - return nil, err - } - return computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() -} - -type instanceExistsError struct { - project string - zone string - name string -} - -func (e instanceExistsError) Error() string { - if e.project == "" { - panic("instanceExistsErr has no project") - } - msg := "some instance(s) already exist as (" + e.project - if e.zone != "" { - msg += ", " + e.zone - } - if e.name != "" { - msg += ", " + e.name - } - msg += "), you need to delete them first." - return msg -} - -// projectHasInstance checks for all the possible zones if there's already an instance for the project. -// It returns the name of the zone at the first instance it finds, if any. -func (d *Deployer) projectHasInstance() (zone string, err error) { - s, err := compute.New(d.Client) - if err != nil { - return "", err - } - // TODO(mpl): make use of the handler's cached zones. - zl, err := compute.NewZonesService(s).List(d.Conf.Project).Do() - if err != nil { - return "", fmt.Errorf("could not get a list of zones: %v", err) - } - computeService, _ := compute.New(d.Client) - var zoneOnce sync.Once - var grp syncutil.Group - errc := make(chan error, 1) - zonec := make(chan string, 1) - timeout := time.NewTimer(30 * time.Second) - defer timeout.Stop() - for _, z := range zl.Items { - z := z - grp.Go(func() error { - list, err := computeService.Instances.List(d.Conf.Project, z.Name).Do() - if err != nil { - return fmt.Errorf("could not list existing instances: %v", err) - } - if len(list.Items) > 0 { - zoneOnce.Do(func() { - zonec <- z.Name - }) - } - return nil - }) - } - go func() { - errc <- grp.Err() - }() - // We block until either an instance was found in a zone, or all the instance - // listing is done. Or we timed-out. - select { - case err = <-errc: - return "", err - case zone = <-zonec: - // We voluntarily ignore any listing error if we found at least one instance - // because that's what we primarily want to report about. - return zone, nil - case <-timeout.C: - return "", errors.New("timed out") - } -} - -type projectIDError struct { - id string - cause error -} - -func (e projectIDError) Error() string { - if e.id == "" { - panic("projectIDError without an id") - } - if e.cause != nil { - return fmt.Sprintf("project ID error for %v: %v", e.id, e.cause) - } - return fmt.Sprintf("project ID error for %v", e.id) -} - -// CreateProject creates a new Google Cloud Project. It returns the project ID, -// which is a random number in (0,1e10), prefixed with "camlistore-launcher-". -func (d *Deployer) CreateProject(ctx context.Context) (string, error) { - s, err := cloudresourcemanager.New(d.Client) - if err != nil { - return "", err - } - // Allow for a few retries, when we generated an already taken project ID - creationTimeout := time.Now().Add(time.Minute) - var projectID, projectName string - for { - if d.Conf.Project != "" { - projectID = d.Conf.Project - projectName = projectID - } else { - projectID = genRandomProjectID() - projectName = strings.Replace(projectID, "camlistore-launcher-", "Camlistore ", 1) - } - project := cloudresourcemanager.Project{ - Name: projectName, - ProjectId: projectID, - } - if time.Now().After(creationTimeout) { - return "", errors.New("timeout while trying to create project") - } - d.Printf("Trying to create project %v", projectID) - op, err := cloudresourcemanager.NewProjectsService(s).Create(&project).Do() - if err != nil { - gerr, ok := err.(*googleapi.Error) - if !ok { - return "", fmt.Errorf("could not create project: %v", err) - } - if gerr.Code != 409 { - return "", fmt.Errorf("could not create project: %v", gerr.Message) - } - // it's ok using time.Sleep, and no backoff, as the - // timeout is pretty short, and we only retry on a 409. - time.Sleep(time.Second) - // retry if project ID already exists. - d.Printf("Project %v already exists, will retry with a new project ID", project.ProjectId) - continue - } - - // as per - // https://cloud.google.com/resource-manager/reference/rest/v1/projects/create - // recommendation - timeout := time.Now().Add(30 * time.Second) - backoff := time.Second - startPolling := 5 * time.Second - time.Sleep(startPolling) - for { - if time.Now().After(timeout) { - return "", fmt.Errorf("timeout while trying to check project creation") - } - if !op.Done { - // it's ok to just sleep, as our timeout is pretty short. - time.Sleep(backoff) - backoff *= 2 - op, err = cloudresourcemanager.NewOperationsService(s).Get(op.Name).Do() - if err != nil { - return "", fmt.Errorf("could not check project creation status: %v", err) - } - continue - } - if op.Error != nil { - // TODO(mpl): ghetto logging for now. detect at least the quota errors. - var details string - for _, v := range op.Error.Details { - details += string(v) - } - return "", fmt.Errorf("could not create project: %v, %v", op.Error.Message, details) - } - break - } - break - } - d.Printf("Success creating project %v", projectID) - return projectID, nil -} - -func genRandomProjectID() string { - // we're allowed up to 30 characters, and we already consume 20 with - // "camlistore-launcher-", so we've got 10 chars left of randomness. Should - // be plenty enough I think. - var n *big.Int - var err error - zero := big.NewInt(0) - for { - n, err = rand.Int(rand.Reader, big.NewInt(1e10)) // max is 1e10 - 1 - if err != nil { - panic(fmt.Sprintf("rand.Int error: %v", err)) - } - if n.Cmp(zero) > 0 { - break - } - } - return fmt.Sprintf("camlistore-launcher-%d", n) -} - -func (d *Deployer) enableAPIs() error { - // TODO(mpl): For now we're lucky enough that servicemanagement seems to - // work even when the Service Management API hasn't been enabled for the - // project. If/when it does not anymore, then we should use serviceuser - // instead. http://stackoverflow.com/a/43503392/1775619 - s, err := servicemanagement.New(d.Client) - if err != nil { - return err - } - - list, err := servicemanagement.NewServicesService(s).List().ConsumerId("project:" + d.Conf.Project).Do() - if err != nil { - return err - } - - requiredServices := map[string]string{ - "storage-component.googleapis.com": "Google Cloud Storage", - "storage-api.googleapis.com": "Google Cloud Storage JSON", - "logging.googleapis.com": "Stackdriver Logging", - "compute.googleapis.com": "Google Compute Engine", - "geocoding-backend.googleapis.com": "Google Maps Geocoding", - } - enabledServices := make(map[string]bool) - for _, v := range list.Services { - enabledServices[v.ServiceName] = true - } - errc := make(chan error, len(requiredServices)) - var wg sync.WaitGroup - for k, v := range requiredServices { - if _, ok := enabledServices[k]; ok { - continue - } - d.Printf("%v API not enabled; enabling it with Service Management", v) - op, err := servicemanagement.NewServicesService(s). - Enable(k, &servicemanagement.EnableServiceRequest{ConsumerId: "project:" + d.Conf.Project}).Do() - if err != nil { - gerr, ok := err.(*googleapi.Error) - if !ok { - return err - } - if gerr.Code != http.StatusBadRequest { - return err - } - for _, v := range gerr.Errors { - if v.Reason == "failedPrecondition" && strings.Contains(v.Message, "billing-enabled") { - return fmt.Errorf("you need to enabling billing for project %v: https://console.cloud.google.com/billing/?project=%v", d.Conf.Project, d.Conf.Project) - } - } - return err - } - - wg.Add(1) - go func(service, opName string) { - defer wg.Done() - timeout := time.Now().Add(2 * time.Minute) - backoff := time.Second - startPolling := 5 * time.Second - time.Sleep(startPolling) - for { - if time.Now().After(timeout) { - errc <- fmt.Errorf("timeout while trying to enable service: %v", service) - return - } - op, err := servicemanagement.NewOperationsService(s).Get(opName).Do() - if err != nil { - errc <- fmt.Errorf("could not check service enabling status: %v", err) - return - } - if !op.Done { - // it's ok to just sleep, as our timeout is pretty short. - time.Sleep(backoff) - backoff *= 2 - continue - } - if op.Error != nil { - errc <- fmt.Errorf("could not enable service %v: %v", service, op.Error.Message) - return - } - d.Printf("%v service successfully enabled", service) - return - } - }(k, op.Name) - } - wg.Wait() - close(errc) - for err := range errc { - if err != nil { - return err - } - } - return nil -} - -func (d *Deployer) checkProjectID() error { - // TODO(mpl): cache the computeService in Deployer, instead of recreating a new one everytime? - s, err := compute.New(d.Client) - if err != nil { - return projectIDError{ - id: d.Conf.Project, - cause: err, - } - } - project, err := compute.NewProjectsService(s).Get(d.Conf.Project).Do() - if err != nil { - return projectIDError{ - id: d.Conf.Project, - cause: err, - } - } - if project.Name != d.Conf.Project { - return projectIDError{ - id: d.Conf.Project, - cause: fmt.Errorf("project IDs do not match: got %q, wanted %q", project.Name, d.Conf.Project), - } - } - return nil -} - -var errAttrNotFound = errors.New("attribute not found") - -// getInstanceAttribute returns the value for attr in the custom metadata of the -// instance. It returns errAttrNotFound is such a metadata attributed does not -// exist. -func (d *Deployer) getInstanceAttribute(attr string) (string, error) { - s, err := compute.New(d.Client) - if err != nil { - return "", fmt.Errorf("error getting compute service: %v", err) - } - inst, err := compute.NewInstancesService(s).Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() - if err != nil { - return "", fmt.Errorf("error getting instance: %v", err) - } - for _, v := range inst.Metadata.Items { - if v.Key == attr { - return *(v.Value), nil - } - } - return "", errAttrNotFound -} - -// Create sets up and starts a Google Compute Engine instance as defined in d.Conf. It -// creates the necessary Google Storage buckets beforehand. -func (d *Deployer) Create(ctx context.Context) (*compute.Instance, error) { - if err := d.enableAPIs(); err != nil { - return nil, projectIDError{ - id: d.Conf.Project, - cause: err, - } - } - if err := d.checkProjectID(); err != nil { - return nil, err - } - - computeService, _ := compute.New(d.Client) - storageService, _ := storage.New(d.Client) - - fwc := make(chan error, 1) - go func() { - fwc <- d.setFirewall(ctx, computeService) - }() - - config := cloudConfig(d.Conf) - const maxCloudConfig = 32 << 10 // per compute API docs - if len(config) > maxCloudConfig { - return nil, fmt.Errorf("cloud config length of %d bytes is over %d byte limit", len(config), maxCloudConfig) - } - - if zone, err := d.projectHasInstance(); zone != "" { - return nil, instanceExistsError{ - project: d.Conf.Project, - zone: zone, - } - } else if err != nil { - return nil, fmt.Errorf("could not scan project for existing instances: %v", err) - } - - if err := d.setBuckets(ctx, storageService); err != nil { - return nil, fmt.Errorf("could not create buckets: %v", err) - } - - if err := d.createInstance(ctx, computeService); err != nil { - return nil, fmt.Errorf("could not create compute instance: %v", err) - } - - inst, err := computeService.Instances.Get(d.Conf.Project, d.Conf.Zone, d.Conf.Name).Do() - if err != nil { - return nil, fmt.Errorf("error getting instance after creation: %v", err) - } - if Verbose { - ij, _ := json.MarshalIndent(inst, "", " ") - d.Printf("Instance: %s", ij) - } - - if err = <-fwc; err != nil { - return nil, fmt.Errorf("could not create firewall rules: %v", err) - } - return inst, nil -} - -func randPassword() string { - buf := make([]byte, 5) - if n, err := rand.Read(buf); err != nil || n != len(buf) { - log.Fatalf("crypto/rand.Read = %v, %v", n, err) - } - return fmt.Sprintf("%x", buf) -} - -// LooksLikeRegion reports whether s looks like a GCE region. -func LooksLikeRegion(s string) bool { - return strings.Count(s, "-") == 1 -} - -// createInstance starts the creation of the Compute Engine instance and waits for the -// result of the creation operation. It should be called after setBuckets and setupHTTPS. -func (d *Deployer) createInstance(ctx context.Context, computeService *compute.Service) error { - coreosImgURL, err := gceutil.CoreOSImageURL(d.Client) - if err != nil { - return fmt.Errorf("error looking up latest CoreOS stable image: %v", err) - } - prefix := projectsAPIURL + d.Conf.Project - machType := prefix + "/zones/" + d.Conf.Zone + "/machineTypes/" + d.Conf.Machine - config := cloudConfig(d.Conf) - instance := &compute.Instance{ - Name: d.Conf.Name, - Description: "Camlistore server", - MachineType: machType, - Disks: []*compute.AttachedDisk{ - { - AutoDelete: true, - Boot: true, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: d.Conf.Name + "-coreos-stateless-pd", - SourceImage: coreosImgURL, - }, - }, - }, - Tags: &compute.Tags{ - Items: []string{"http-server", "https-server"}, - }, - Metadata: &compute.Metadata{ - Items: []*compute.MetadataItems{ - { - Key: "camlistore-username", - Value: googleapi.String(camliUsername), - }, - { - Key: "camlistore-password", - Value: googleapi.String(randPassword()), - }, - { - Key: "camlistore-blob-dir", - Value: googleapi.String("gs://" + d.Conf.blobDir), - }, - { - Key: "camlistore-config-dir", - Value: googleapi.String("gs://" + d.Conf.configDir), - }, - { - Key: "perkeep-config-version", - // perkeep-config-version being defined requires this launcher to deploy Perkeep - // at rev >= 7eda9fd5027fda88166d6c03b6490cffbf2de5fb , so that any newly deployed Perkeep - // knows it can use the new configuration without DBNames. - // TODO(mpl): but how do we enforce it, or at least make it more obvious/documented? - // With a flag defaulting to "1" maybe? - Value: googleapi.String("1"), - }, - { - Key: "user-data", - Value: googleapi.String(config), - }, - }, - }, - NetworkInterfaces: []*compute.NetworkInterface{ - { - AccessConfigs: []*compute.AccessConfig{ - { - Type: "ONE_TO_ONE_NAT", - Name: "External NAT", - }, - }, - Network: prefix + "/global/networks/default", - }, - }, - ServiceAccounts: []*compute.ServiceAccount{ - { - Email: "default", - Scopes: []string{ - logging.WriteScope, - compute.DevstorageFullControlScope, - compute.ComputeScope, - "https://www.googleapis.com/auth/sqlservice", - "https://www.googleapis.com/auth/sqlservice.admin", - }, - }, - }, - } - if d.Conf.Hostname != "" { - instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{ - Key: "camlistore-hostname", - Value: googleapi.String(d.Conf.Hostname), - }) - } - const localMySQL = false // later - if localMySQL { - instance.Disks = append(instance.Disks, &compute.AttachedDisk{ - AutoDelete: false, - Boot: false, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: "camlistore-mysql-index-pd", - DiskSizeGb: 4, - }, - }) - } - - if Verbose { - d.Print("Creating instance...") - } - op, err := computeService.Instances.Insert(d.Conf.Project, d.Conf.Zone, instance).Do() - if err != nil { - return fmt.Errorf("failed to create instance: %v", err) - } - opName := op.Name - if Verbose { - d.Printf("Created. Waiting on operation %v", opName) - } -OpLoop: - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - time.Sleep(2 * time.Second) - op, err := computeService.ZoneOperations.Get(d.Conf.Project, d.Conf.Zone, opName).Do() - if err != nil { - return fmt.Errorf("failed to get op %s: %v", opName, err) - } - switch op.Status { - case "PENDING", "RUNNING": - if Verbose { - d.Printf("Waiting on operation %v", opName) - } - continue - case "DONE": - if op.Error != nil { - for _, operr := range op.Error.Errors { - d.Printf("Error: %+v", operr) - } - return fmt.Errorf("failed to start") - } - if Verbose { - d.Printf("Success. %+v", op) - } - break OpLoop - default: - return fmt.Errorf("unknown status %q: %+v", op.Status, op) - } - } - return nil -} - -func cloudConfig(conf *InstanceConf) string { - config := strings.Replace(baseInstanceConfig, "INNODB_BUFFER_POOL_SIZE=NNN", "INNODB_BUFFER_POOL_SIZE="+strconv.Itoa(innodbBufferPoolSize(conf.Machine)), -1) - perkeepdTarball := "https://storage.googleapis.com/camlistore-release/docker/" - if conf.WIP { - perkeepdTarball += "perkeepd-WORKINPROGRESS.tar.gz" - } else { - perkeepdTarball += "perkeepd.tar.gz" - } - config = strings.Replace(config, "CAMLISTORED_TARBALL", perkeepdTarball, 1) - return config -} - -// setBuckets defines the buckets needed by the instance and creates them. -func (d *Deployer) setBuckets(ctx context.Context, storageService *storage.Service) error { - projBucket := d.Conf.Project + "-camlistore" - - needBucket := map[string]bool{ - projBucket: true, - } - - buckets, err := storageService.Buckets.List(d.Conf.Project).Do() - if err != nil { - return fmt.Errorf("error listing buckets: %v", err) - } - for _, it := range buckets.Items { - delete(needBucket, it.Name) - } - if len(needBucket) > 0 { - if Verbose { - d.Printf("Need to create buckets: %v", needBucket) - } - var waitBucket sync.WaitGroup - var bucketErr error - for name := range needBucket { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - name := name - waitBucket.Add(1) - go func() { - defer waitBucket.Done() - if Verbose { - d.Printf("Creating bucket %s", name) - } - b, err := storageService.Buckets.Insert(d.Conf.Project, &storage.Bucket{ - Id: name, - Name: name, - }).Do() - if err != nil && bucketErr == nil { - bucketErr = fmt.Errorf("error creating bucket %s: %v", name, err) - return - } - if Verbose { - d.Printf("Created bucket %s: %+v", name, b) - } - }() - } - waitBucket.Wait() - if bucketErr != nil { - return bucketErr - } - } - - d.Conf.configDir = path.Join(projBucket, configDir) - d.Conf.blobDir = path.Join(projBucket, "blobs") - return nil -} - -// setFirewall adds the firewall rules needed for ports 80 & 433 to the default network. -func (d *Deployer) setFirewall(ctx context.Context, computeService *compute.Service) error { - defaultNet, err := computeService.Networks.Get(d.Conf.Project, "default").Do() - if err != nil { - return fmt.Errorf("error getting default network: %v", err) - } - - needRules := map[string]compute.Firewall{ - "default-allow-http": { - Name: "default-allow-http", - SourceRanges: []string{"0.0.0.0/0"}, - SourceTags: []string{"http-server"}, - Allowed: []*compute.FirewallAllowed{{IPProtocol: "tcp", Ports: []string{"80"}}}, - Network: defaultNet.SelfLink, - }, - "default-allow-https": { - Name: "default-allow-https", - SourceRanges: []string{"0.0.0.0/0"}, - SourceTags: []string{"https-server"}, - Allowed: []*compute.FirewallAllowed{{IPProtocol: "tcp", Ports: []string{"443"}}}, - Network: defaultNet.SelfLink, - }, - } - - rules, err := computeService.Firewalls.List(d.Conf.Project).Do() - if err != nil { - return fmt.Errorf("error listing rules: %v", err) - } - for _, it := range rules.Items { - delete(needRules, it.Name) - } - if len(needRules) == 0 { - return nil - } - - if Verbose { - d.Printf("Need to create rules: %v", needRules) - } - var wg syncutil.Group - for name, rule := range needRules { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - name, rule := name, rule - wg.Go(func() error { - if Verbose { - d.Printf("Creating rule %s", name) - } - r, err := computeService.Firewalls.Insert(d.Conf.Project, &rule).Do() - if err != nil { - return fmt.Errorf("error creating rule %s: %v", name, err) - } - if Verbose { - d.Printf("Created rule %s: %+v", name, r) - } - return nil - }) - } - return wg.Err() -} - -// returns the MySQL InnoDB buffer pool size (in bytes) as a function -// of the GCE machine type. -func innodbBufferPoolSize(machine string) int { - // Totally arbitrary. We don't need much here because - // perkeepd slurps this all into its RAM on start-up - // anyway. So this is all prety overkill and more than the - // 8MB default. - switch machine { - case "f1-micro": - return 32 << 20 - case "g1-small": - return 64 << 20 - default: - return 128 << 20 - } -} - -const baseInstanceConfig = `#cloud-config -write_files: - - path: /var/lib/camlistore/tmp/README - permissions: 0644 - content: | - This is the Camlistore /tmp directory. - - path: /var/lib/camlistore/mysql/README - permissions: 0644 - content: | - This is the Camlistore MySQL data directory. -coreos: - units: - - name: cam-journal-gatewayd.service - content: | - [Unit] - Description=Journal Gateway Service - Requires=cam-journal-gatewayd.socket - - [Service] - ExecStart=/usr/lib/systemd/systemd-journal-gatewayd - User=systemd-journal-gateway - Group=systemd-journal-gateway - SupplementaryGroups=systemd-journal - PrivateTmp=yes - PrivateDevices=yes - PrivateNetwork=yes - ProtectSystem=full - ProtectHome=yes - - [Install] - Also=cam-journal-gatewayd.socket - - name: cam-journal-gatewayd.socket - command: start - content: | - [Unit] - Description=Journal Gateway Service Socket - - [Socket] - ListenStream=/run/camjournald.sock - - [Install] - WantedBy=sockets.target - - name: mysql.service - command: start - content: | - [Unit] - Description=MySQL - After=docker.service - Requires=docker.service - - [Service] - ExecStartPre=/bin/bash -c '/usr/bin/curl https://storage.googleapis.com/camlistore-release/docker/systemd-docker.tar.gz | /bin/gunzip -c | /usr/bin/docker load' - ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin camlistore/systemd-docker - ExecStart=/opt/bin/systemd-docker run --rm --name %n -v /var/lib/camlistore/mysql:/mysql -e INNODB_BUFFER_POOL_SIZE=NNN camlistore/mysql - RestartSec=1s - Restart=always - Type=notify - NotifyAccess=all - - [Install] - WantedBy=multi-user.target - - name: perkeepd.service - command: start - content: | - [Unit] - Description=Camlistore - After=docker.service mysql.service - Requires=docker.service mysql.service - - [Service] - ExecStartPre=/usr/bin/docker run --rm -v /opt/bin:/opt/bin camlistore/systemd-docker - ExecStartPre=/bin/bash -c '/usr/bin/curl CAMLISTORED_TARBALL | /bin/gunzip -c | /usr/bin/docker load' - ExecStart=/opt/bin/systemd-docker run --rm -p 80:80 -p 443:443 --name %n -v /run/camjournald.sock:/run/camjournald.sock -v /var/lib/camlistore/tmp:/tmp --link=mysql.service:mysqldb perkeep/server - RestartSec=1s - Restart=always - Type=notify - NotifyAccess=all - - [Install] - WantedBy=multi-user.target -` diff --git a/pkg/deploy/gce/handler.go b/pkg/deploy/gce/handler.go deleted file mode 100644 index 287613ba9..000000000 --- a/pkg/deploy/gce/handler.go +++ /dev/null @@ -1,1325 +0,0 @@ -/* -Copyright 2015 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gce - -import ( - "bytes" - "context" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "html/template" - "io" - "log" - "math/rand" - "net/http" - "os" - "path" - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "time" - - "perkeep.org/internal/httputil" - "perkeep.org/pkg/auth" - "perkeep.org/pkg/blob" - "perkeep.org/pkg/blobserver" - "perkeep.org/pkg/blobserver/localdisk" - "perkeep.org/pkg/sorted" - "perkeep.org/pkg/sorted/leveldb" - - "cloud.google.com/go/compute/metadata" - "golang.org/x/net/xsrftoken" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - compute "google.golang.org/api/compute/v1" -) - -const cookieExpiration = 24 * time.Hour - -var ( - helpMachineTypes = "https://cloud.google.com/compute/docs/machine-types" - helpZones = "https://cloud.google.com/compute/docs/zones#available" - - machineValues = []string{ - "f1-micro", - "g1-small", - "n1-highcpu-2", - } - - backupZones = map[string][]string{ - "us-central1": {"-a", "-b", "-f"}, - "europe-west1": {"-b", "-c", "-d"}, - "asia-east1": {"-a", "-b", "-c"}, - } -) - -// DeployHandler serves a wizard that helps with the deployment of Perkeep on Google -// Compute Engine. It must be initialized with NewDeployHandler. -type DeployHandler struct { - scheme string // URL scheme for the URLs served by this handler. Defaults to "https". - host string // URL host for the URLs served by this handler. - prefix string // prefix is the pattern for which this handler is registered as an http.Handler. - help map[string]template.HTML // various help bits used in the served pages, keyed by relevant names. - xsrfKey string // for XSRF protection. - piggyGIF string // path to the piggy gif file, defaults to /static/piggy.gif - mux *http.ServeMux - - tplMu sync.RWMutex - tpl *template.Template - - // Our wizard's credentials, acting on behalf of the user. - // Obtained from the environment for now. - clientID string - clientSecret string - - // stores the user submitted configuration as a JSON-encoded InstanceConf - instConf blobserver.Storage - // key is blobRef of the relevant InstanceConf, value is the current state of - // the instance creation process, as JSON-encoded creationState - instState sorted.KeyValue - - recordStateErrMu sync.RWMutex - // recordStateErr maps the blobRef of the relevant InstanceConf to the error - // that occurred when recording the creation state. - recordStateErr map[string]error - - zonesMu sync.RWMutex - // maps a region to all its zones suffixes (e.g. "asia-east1" -> "-a","-b"). updated in the - // background every 24 hours. defaults to backupZones. - zones map[string][]string - regions []string - - camliVersionMu sync.RWMutex - camliVersion string // git revision found in https://storage.googleapis.com/camlistore-release/docker/VERSION - - logger *log.Logger // should not be nil. -} - -// Config is the set of parameters to initialize the DeployHandler. -type Config struct { - ClientID string `json:"clientID"` // handler's credentials for OAuth. Required. - ClientSecret string `json:"clientSecret"` // handler's credentials for OAuth. Required. - Project string `json:"project"` // any Google Cloud project we can query to get the valid Google Cloud zones. Optional. Set from metadata on GCE. - ServiceAccount string `json:"serviceAccount"` // JSON file with credentials to Project. Optional. Unused on GCE. - DataDir string `json:"dataDir"` // where to store the instances configurations and states. Optional. -} - -// NewDeployHandlerFromConfig initializes a DeployHandler from cfg. -// Host and prefix have the same meaning as for NewDeployHandler. cfg should not be nil. -func NewDeployHandlerFromConfig(host, prefix string, cfg *Config) (*DeployHandler, error) { - if cfg == nil { - panic("NewDeployHandlerFromConfig: nil config") - } - if cfg.ClientID == "" { - return nil, errors.New("oauth2 clientID required in config") - } - if cfg.ClientSecret == "" { - return nil, errors.New("oauth2 clientSecret required in config") - } - os.Setenv("CAMLI_GCE_CLIENTID", cfg.ClientID) - os.Setenv("CAMLI_GCE_CLIENTSECRET", cfg.ClientSecret) - os.Setenv("CAMLI_GCE_PROJECT", cfg.Project) - os.Setenv("CAMLI_GCE_SERVICE_ACCOUNT", cfg.ServiceAccount) - os.Setenv("CAMLI_GCE_DATA", cfg.DataDir) - return NewDeployHandler(host, prefix) -} - -// NewDeployHandler initializes a DeployHandler that serves at https://host/prefix/ and returns it. -// A Google account client ID should be set in CAMLI_GCE_CLIENTID with its corresponding client -// secret in CAMLI_GCE_CLIENTSECRET. -func NewDeployHandler(host, prefix string) (*DeployHandler, error) { - clientID := os.Getenv("CAMLI_GCE_CLIENTID") - if clientID == "" { - return nil, errors.New("Need an oauth2 client ID defined in CAMLI_GCE_CLIENTID") - } - clientSecret := os.Getenv("CAMLI_GCE_CLIENTSECRET") - if clientSecret == "" { - return nil, errors.New("Need an oauth2 client secret defined in CAMLI_GCE_CLIENTSECRET") - } - tpl, err := template.New("root").Parse(noTheme + tplHTML()) - if err != nil { - return nil, fmt.Errorf("could not parse template: %v", err) - } - host = strings.TrimSuffix(host, "/") - prefix = strings.TrimSuffix(prefix, "/") - scheme := "https" - xsrfKey := os.Getenv("CAMLI_GCE_XSRFKEY") - if xsrfKey == "" { - xsrfKey = auth.RandToken(20) - log.Printf("xsrf key not provided as env var CAMLI_GCE_XSRFKEY, so generating one instead: %v", xsrfKey) - } - instConf, instState, err := dataStores() - if err != nil { - return nil, fmt.Errorf("could not initialize conf or state storage: %v", err) - } - h := &DeployHandler{ - host: host, - xsrfKey: xsrfKey, - instConf: instConf, - instState: instState, - recordStateErr: make(map[string]error), - scheme: scheme, - prefix: prefix, - help: map[string]template.HTML{ - "machineTypes": template.HTML(helpMachineTypes), - "zones": template.HTML(helpZones), - }, - clientID: clientID, - clientSecret: clientSecret, - tpl: tpl, - piggyGIF: "/static/piggy.gif", - } - mux := http.NewServeMux() - mux.HandleFunc(prefix+"/callback", func(w http.ResponseWriter, r *http.Request) { - h.serveCallback(w, r) - }) - mux.HandleFunc(prefix+"/instance", func(w http.ResponseWriter, r *http.Request) { - h.serveInstanceState(w, r) - }) - mux.HandleFunc(prefix+"/", func(w http.ResponseWriter, r *http.Request) { - h.serveRoot(w, r) - }) - h.mux = mux - h.SetLogger(log.New(os.Stderr, "GCE DEPLOYER: ", log.LstdFlags)) - h.zones = backupZones - var refreshZonesFn func() - refreshZonesFn = func() { - if err := h.refreshZones(); err != nil { - h.logger.Printf("error while refreshing zones: %v", err) - } - time.AfterFunc(24*time.Hour, refreshZonesFn) - } - go refreshZonesFn() - var refreshCamliVersionFn func() - refreshCamliVersionFn = func() { - if err := h.refreshCamliVersion(); err != nil { - h.logger.Printf("error while refreshing Perkeep version: %v", err) - } - time.AfterFunc(time.Hour, refreshCamliVersionFn) - } - go refreshCamliVersionFn() - return h, nil -} - -func (h *DeployHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if h.mux == nil { - http.Error(w, "handler not properly initialized", http.StatusInternalServerError) - return - } - h.mux.ServeHTTP(w, r) -} - -func (h *DeployHandler) SetScheme(scheme string) { h.scheme = scheme } - -// authenticatedClient returns the GCE project running the /launch/ -// app (e.g. "camlistore-website" usually for the main instance) and -// an authenticated OAuth2 client acting as that service account. -// This is only used for refreshing the list of valid zones to give to -// the user in a drop-down. - -// If we're not running on GCE (e.g. dev mode on localhost) and have -// no other way to get the info, the error value is is errNoRefresh. -func (h *DeployHandler) authenticatedClient() (project string, hc *http.Client, err error) { - var ctx = context.Background() - - project = os.Getenv("CAMLI_GCE_PROJECT") - accountFile := os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT") - if project != "" && accountFile != "" { - data, errr := os.ReadFile(accountFile) - err = errr - if err != nil { - return - } - jwtConf, errr := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/compute.readonly") - err = errr - if err != nil { - return - } - hc = jwtConf.Client(ctx) - return - } - if !metadata.OnGCE() { - err = errNoRefresh - return - } - project, _ = metadata.ProjectID() - hc, err = google.DefaultClient(ctx) - return project, hc, err -} - -var gitRevRgx = regexp.MustCompile(`^[a-z0-9]{40}?$`) - -func (h *DeployHandler) refreshCamliVersion() error { - h.camliVersionMu.Lock() - defer h.camliVersionMu.Unlock() - resp, err := http.Get("https://storage.googleapis.com/camlistore-release/docker/VERSION") - if err != nil { - return err - } - defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - version := strings.TrimSpace(string(data)) - if !gitRevRgx.MatchString(version) { - return fmt.Errorf("wrong revision format in VERSION file: %q", version) - } - h.camliVersion = version - return nil -} - -func (h *DeployHandler) camliRev() string { - h.camliVersionMu.RLock() - defer h.camliVersionMu.RUnlock() - return h.camliVersion -} - -var errNoRefresh error = errors.New("not on GCE, and at least one of CAMLI_GCE_PROJECT or CAMLI_GCE_SERVICE_ACCOUNT not defined") - -func (h *DeployHandler) refreshZones() error { - h.zonesMu.Lock() - defer h.zonesMu.Unlock() - defer func() { - h.regions = make([]string, 0, len(h.zones)) - for r := range h.zones { - h.regions = append(h.regions, r) - } - }() - project, hc, err := h.authenticatedClient() - if err != nil { - if err == errNoRefresh { - h.zones = backupZones - h.logger.Printf("Cannot refresh zones. Using hard-coded ones instead.") - return nil - } - return err - } - s, err := compute.New(hc) - if err != nil { - return err - } - rl, err := compute.NewRegionsService(s).List(project).Do() - if err != nil { - return fmt.Errorf("could not get a list of regions: %v", err) - } - h.zones = make(map[string][]string) - for _, r := range rl.Items { - zones := make([]string, 0, len(r.Zones)) - for _, z := range r.Zones { - zone := path.Base(z) - if zone == "europe-west1-a" { - // Because even though the docs mark it as deprecated, it still shows up here, go figure. - continue - } - zone = strings.Replace(zone, r.Name, "", 1) - zones = append(zones, zone) - } - h.zones[r.Name] = zones - } - return nil -} - -func (h *DeployHandler) zoneValues() []string { - h.zonesMu.RLock() - defer h.zonesMu.RUnlock() - return h.regions -} - -// if there's project as a query parameter, it means we've just created a -// project for them and we're redirecting them to the form, but with the projectID -// field pre-filled for them this time. -func (h *DeployHandler) serveRoot(w http.ResponseWriter, r *http.Request) { - if r.Method == "POST" { - h.serveSetup(w, r) - return - } - _, err := r.Cookie("user") - if err != nil { - http.SetCookie(w, newCookie()) - } - camliRev := h.camliRev() - if r.FormValue("WIP") == "1" { - camliRev = "WORKINPROGRESS" - } - - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if err := h.tpl.ExecuteTemplate(w, "withform", &TemplateData{ - ProjectID: r.FormValue("project"), - Prefix: h.prefix, - Help: h.help, - ZoneValues: h.zoneValues(), - MachineValues: machineValues, - CamliVersion: camliRev, - }); err != nil { - h.logger.Print(err) - } -} - -func (h *DeployHandler) serveSetup(w http.ResponseWriter, r *http.Request) { - if r.FormValue("mode") != "setupproject" { - h.serveError(w, r, errors.New("bad form")) - return - } - ck, err := r.Cookie("user") - if err != nil { - h.serveFormError(w, errors.New("cookie expired, or CSRF attempt. Please reload and retry")) - h.logger.Printf("Cookie expired, or CSRF attempt on form.") - return - } - - ctx := r.Context() - instConf, err := h.confFromForm(r) - if err != nil { - h.serveFormError(w, err) - return - } - - br, err := h.storeInstanceConf(ctx, instConf) - if err != nil { - h.serveError(w, r, fmt.Errorf("could not store instance configuration: %v", err)) - return - } - - xsrfToken := xsrftoken.Generate(h.xsrfKey, ck.Value, br.String()) - state := fmt.Sprintf("%s:%x", br.String(), xsrfToken) - redirectURL := h.oAuthConfig().AuthCodeURL(state) - http.Redirect(w, r, redirectURL, http.StatusFound) - return -} - -func (h *DeployHandler) serveCallback(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - ck, err := r.Cookie("user") - if err != nil { - http.Error(w, - fmt.Sprintf("Cookie expired, or CSRF attempt. Restart from %s://%s%s", h.scheme, h.host, h.prefix), - http.StatusBadRequest) - h.logger.Printf("Cookie expired, or CSRF attempt on callback.") - return - } - code := r.FormValue("code") - if code == "" { - h.serveError(w, r, errors.New("No oauth code parameter in callback URL")) - return - } - h.logger.Printf("successful authentication: %v", r.URL.RawQuery) - - br, tk, err := fromState(r) - if err != nil { - h.serveError(w, r, err) - return - } - if !xsrftoken.Valid(tk, h.xsrfKey, ck.Value, br.String()) { - h.serveError(w, r, fmt.Errorf("Invalid xsrf token: %q", tk)) - return - } - - oAuthConf := h.oAuthConfig() - tok, err := oAuthConf.Exchange(ctx, code) - if err != nil { - h.serveError(w, r, fmt.Errorf("could not obtain a token: %v", err)) - return - } - h.logger.Printf("successful authorization with token: %v", tok) - - instConf, err := h.instanceConf(ctx, br) - if err != nil { - h.serveError(w, r, err) - return - } - - depl := &Deployer{ - Client: oAuthConf.Client(ctx, tok), - Conf: instConf, - Logger: h.logger, - } - - // They've requested that we create a project for them. - if instConf.CreateProject { - // So we try to do so. - projectID, err := depl.CreateProject(ctx) - if err != nil { - h.logger.Printf("error creating project: %v", err) - // TODO(mpl): we log the errors, but none of them are - // visible to the user (they just get a 500). I should - // probably at least detect and report them the project - // creation quota errors. - h.serveError(w, r, err) - return - } - // And serve the form again if we succeeded. - http.Redirect(w, r, fmt.Sprintf("%s/?project=%s", h.prefix, projectID), http.StatusFound) - return - } - - if found := h.serveOldInstance(w, br, depl); found { - return - } - - if err := h.recordState(br, &creationState{ - InstConf: br, - }); err != nil { - h.serveError(w, r, err) - return - } - - go func() { - inst, err := depl.Create(context.Background()) - state := &creationState{ - InstConf: br, - } - if err != nil { - h.logger.Printf("could not create instance: %v", err) - switch e := err.(type) { - case instanceExistsError: - state.Err = fmt.Sprintf("%v %v", e, helpDeleteInstance) - case projectIDError: - state.Err = fmt.Sprintf("%v", e) - default: - state.Err = fmt.Sprintf("%v. %v", err, fileIssue(br.String())) - } - } else { - state.InstAddr = addr(inst) - state.Success = true - if instConf.Hostname != "" { - state.InstHostname = instConf.Hostname - } - } - if err := h.recordState(br, state); err != nil { - h.logger.Printf("Could not record creation state for %v: %v", br, err) - h.recordStateErrMu.Lock() - defer h.recordStateErrMu.Unlock() - h.recordStateErr[br.String()] = err - return - } - if state.Err != "" { - return - } - if instConf.Hostname != "" { - return - } - // We also try to get the "camlistore-hostname" from the - // instance, so we can tell the user what their hostname is. It can - // take a while as perkeepd itself sets it after it has - // registered with the camlistore.net DNS. - giveupTime := time.Now().Add(time.Hour) - pause := time.Second - for { - hostname, err := depl.getInstanceAttribute("camlistore-hostname") - if err != nil && err != errAttrNotFound { - h.logger.Printf("could not get camlistore-hostname of instance: %v", err) - state.Success = false - state.Err = fmt.Sprintf("could not get camlistore-hostname of instance: %v", err) - break - } - if err == nil { - state.InstHostname = hostname - break - } - if time.Now().After(giveupTime) { - h.logger.Printf("Giving up on getting camlistore-hostname of instance") - state.Success = false - state.Err = fmt.Sprintf("could not get camlistore-hostname of instance") - break - } - time.Sleep(pause) - pause *= 2 - } - if err := h.recordState(br, state); err != nil { - h.logger.Printf("Could not record hostname for %v: %v", br, err) - h.recordStateErrMu.Lock() - defer h.recordStateErrMu.Unlock() - h.recordStateErr[br.String()] = err - return - } - }() - h.serveProgress(w, br) -} - -// serveOldInstance looks on GCE for an instance such as defined in depl.Conf, and if -// found, serves the appropriate page depending on whether the instance is usable. It does -// not serve anything if the instance is not found. -func (h *DeployHandler) serveOldInstance(w http.ResponseWriter, br blob.Ref, depl *Deployer) (found bool) { - inst, err := depl.Get() - if err != nil { - // TODO(mpl,bradfitz): log or do something more - // drastic if the error is something other than - // instance not found. - return false - } - h.logger.Printf("Reusing existing instance for (%v, %v, %v)", depl.Conf.Project, depl.Conf.Name, depl.Conf.Zone) - - if err := h.recordState(br, &creationState{ - InstConf: br, - InstAddr: addr(inst), - Exists: true, - }); err != nil { - h.logger.Printf("Could not record creation state for %v: %v", br, err) - h.serveErrorPage(w, fmt.Errorf("An error occurred while recording the state of your instance. %v", fileIssue(br.String()))) - return true - } - h.serveProgress(w, br) - return true -} - -func (h *DeployHandler) serveFormError(w http.ResponseWriter, err error, hints ...string) { - var topHints []string - topHints = append(topHints, hints...) - h.logger.Print(err) - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if tplErr := h.tpl.ExecuteTemplate(w, "withform", &TemplateData{ - Prefix: h.prefix, - Help: h.help, - Err: err, - Hints: topHints, - ZoneValues: h.zoneValues(), - MachineValues: machineValues, - }); tplErr != nil { - h.logger.Printf("Could not serve form error %q because: %v", err, tplErr) - } -} - -func fileIssue(br string) string { - return fmt.Sprintf("Please file an issue with your instance key (%v) at https://perkeep.org/issue", br) -} - -// serveInstanceState serves the state of the requested Google Cloud Engine VM creation -// process. If the operation was successful, it serves a success page. If it failed, it -// serves an error page. If it isn't finished yet, it replies with "running". -func (h *DeployHandler) serveInstanceState(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - if r.Method != "GET" { - h.serveError(w, r, fmt.Errorf("Wrong method: %v", r.Method)) - return - } - br := r.URL.Query().Get("instancekey") - stateValue, err := h.instState.Get(br) - if err != nil { - http.Error(w, "unknown instance", http.StatusNotFound) - return - } - var state creationState - if err := json.Unmarshal([]byte(stateValue), &state); err != nil { - h.serveError(w, r, fmt.Errorf("could not json decode instance state: %v", err)) - return - } - if state.Err != "" { - // No need to log that error here since we're already doing it in serveCallback - h.serveErrorPage(w, fmt.Errorf("an error occurred while creating your instance: %v", state.Err)) - return - } - if state.Success || state.Exists { - conf, err := h.instanceConf(ctx, state.InstConf) - if err != nil { - h.logger.Printf("Could not get parameters for success message: %v", err) - h.serveErrorPage(w, fmt.Errorf("your instance was created and should soon be up at https://%s but there might have been a problem in the creation process. %v", state.Err, fileIssue(br))) - return - } - h.serveSuccess(w, &TemplateData{ - Prefix: h.prefix, - Help: h.help, - InstanceIP: state.InstAddr, - InstanceHostname: state.InstHostname, - ProjectConsoleURL: fmt.Sprintf("%s/project/%s/compute", ConsoleURL, conf.Project), - Conf: conf, - ZoneValues: h.zoneValues(), - MachineValues: machineValues, - }) - return - } - h.recordStateErrMu.RLock() - defer h.recordStateErrMu.RUnlock() - if _, ok := h.recordStateErr[br]; ok { - // No need to log that error here since we're already doing it in serveCallback - h.serveErrorPage(w, fmt.Errorf("An error occurred while recording the state of your instance. %v", fileIssue(br))) - return - } - fmt.Fprintf(w, "running") -} - -// serveProgress serves a page with some javascript code that regularly queries -// the server about the progress of the requested Google Cloud Engine VM creation. -// The server replies through serveInstanceState. -func (h *DeployHandler) serveProgress(w http.ResponseWriter, instanceKey blob.Ref) { - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if err := h.tpl.ExecuteTemplate(w, "withform", &TemplateData{ - Prefix: h.prefix, - InstanceKey: instanceKey.String(), - PiggyGIF: h.piggyGIF, - }); err != nil { - h.logger.Printf("Could not serve progress: %v", err) - } -} - -func (h *DeployHandler) serveErrorPage(w http.ResponseWriter, err error, hints ...string) { - var topHints []string - topHints = append(topHints, hints...) - h.logger.Print(err) - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if tplErr := h.tpl.ExecuteTemplate(w, "noform", &TemplateData{ - Prefix: h.prefix, - Err: err, - Hints: topHints, - }); tplErr != nil { - h.logger.Printf("Could not serve error %q because: %v", err, tplErr) - } -} - -func (h *DeployHandler) serveSuccess(w http.ResponseWriter, data *TemplateData) { - h.tplMu.RLock() - defer h.tplMu.RUnlock() - if err := h.tpl.ExecuteTemplate(w, "noform", data); err != nil { - h.logger.Printf("Could not serve success: %v", err) - } -} - -func newCookie() *http.Cookie { - expiration := cookieExpiration - return &http.Cookie{ - Name: "user", - Value: auth.RandToken(15), - Expires: time.Now().Add(expiration), - } -} - -func formValueOrDefault(r *http.Request, formField, defValue string) string { - val := r.FormValue(formField) - if val == "" { - return defValue - } - return val -} - -func (h *DeployHandler) confFromForm(r *http.Request) (*InstanceConf, error) { - newProject, err := strconv.ParseBool(r.FormValue("newproject")) - if err != nil { - return nil, fmt.Errorf("could not convert \"newproject\" value to bool: %v", err) - } - var projectID string - if newProject { - projectID = r.FormValue("newprojectid") - } else { - projectID = r.FormValue("projectid") - if projectID == "" { - return nil, errors.New("missing project ID parameter") - } - } - var zone string - zoneReg := formValueOrDefault(r, "zone", DefaultRegion) - if LooksLikeRegion(zoneReg) { - region := zoneReg - zone = h.randomZone(region) - } else if strings.Count(zoneReg, "-") == 2 { - zone = zoneReg - } else { - return nil, errors.New("invalid zone or region") - } - isFreeTier, err := strconv.ParseBool(formValueOrDefault(r, "freetier", "false")) - if err != nil { - return nil, fmt.Errorf("could not convert \"freetier\" value to bool: %v", err) - } - machine := formValueOrDefault(r, "machine", DefaultMachineType) - if isFreeTier { - if !strings.HasPrefix(zone, "us-") { - return nil, fmt.Errorf("The %v zone was selected, but the Google Cloud Free Tier is only available for US zones", zone) - } - if machine != "f1-micro" { - return nil, fmt.Errorf("The %v machine type was selected, but the Google Cloud Free Tier is only available for f1-micro", machine) - } - } - return &InstanceConf{ - CreateProject: newProject, - Name: formValueOrDefault(r, "name", DefaultInstanceName), - Project: projectID, - Machine: machine, - Zone: zone, - Hostname: formValueOrDefault(r, "hostname", ""), - Ctime: time.Now(), - WIP: r.FormValue("WIP") == "1", - }, nil -} - -// randomZone picks one of the zone suffixes for region and returns it -// appended to region, as a fully-qualified zone name. -// If the given region is invalid, the default Zone is returned instead. -func (h *DeployHandler) randomZone(region string) string { - h.zonesMu.RLock() - defer h.zonesMu.RUnlock() - zones, ok := h.zones[region] - if !ok { - return fallbackZone - } - return region + zones[rand.Intn(len(zones))] -} - -func (h *DeployHandler) SetLogger(lg *log.Logger) { - h.logger = lg -} - -func (h *DeployHandler) serveError(w http.ResponseWriter, r *http.Request, err error) { - if h.logger != nil { - h.logger.Printf("%v", err) - } - httputil.ServeError(w, r, err) -} - -func (h *DeployHandler) oAuthConfig() *oauth2.Config { - oauthConfig := NewOAuthConfig(h.clientID, h.clientSecret) - oauthConfig.RedirectURL = fmt.Sprintf("%s://%s%s/callback", h.scheme, h.host, h.prefix) - return oauthConfig -} - -// fromState parses the oauth state parameter from r to extract the blobRef of the -// instance configuration and the xsrftoken that were stored during serveSetup. -func fromState(r *http.Request) (br blob.Ref, xsrfToken string, err error) { - params := strings.Split(r.FormValue("state"), ":") - if len(params) != 2 { - return br, "", fmt.Errorf("Invalid format for state parameter: %q, wanted blobRef:xsrfToken", r.FormValue("state")) - } - br, ok := blob.Parse(params[0]) - if !ok { - return br, "", fmt.Errorf("Invalid blobRef in state parameter: %q", params[0]) - } - token, err := hex.DecodeString(params[1]) - if err != nil { - return br, "", fmt.Errorf("can't decode hex xsrftoken %q: %v", params[1], err) - } - return br, string(token), nil -} - -func (h *DeployHandler) storeInstanceConf(ctx context.Context, conf *InstanceConf) (blob.Ref, error) { - contents, err := json.Marshal(conf) - if err != nil { - return blob.Ref{}, fmt.Errorf("could not json encode instance config: %v", err) - } - hash := blob.NewHash() - _, err = io.Copy(hash, bytes.NewReader(contents)) - if err != nil { - return blob.Ref{}, fmt.Errorf("could not hash blob contents: %v", err) - } - br := blob.RefFromHash(hash) - if _, err := blobserver.Receive(ctx, h.instConf, br, bytes.NewReader(contents)); err != nil { - return blob.Ref{}, fmt.Errorf("could not store instance config blob: %v", err) - } - return br, nil -} - -func (h *DeployHandler) instanceConf(ctx context.Context, br blob.Ref) (*InstanceConf, error) { - rc, _, err := h.instConf.Fetch(ctx, br) - if err != nil { - return nil, fmt.Errorf("could not fetch conf at %v: %v", br, err) - } - defer rc.Close() - contents, err := io.ReadAll(rc) - if err != nil { - return nil, fmt.Errorf("could not read conf in blob %v: %v", br, err) - } - var instConf InstanceConf - if err := json.Unmarshal(contents, &instConf); err != nil { - return nil, fmt.Errorf("could not json decode instance config: %v", err) - } - return &instConf, nil -} - -func (h *DeployHandler) recordState(br blob.Ref, state *creationState) error { - val, err := json.Marshal(state) - if err != nil { - return fmt.Errorf("could not json encode instance state: %v", err) - } - if err := h.instState.Set(br.String(), string(val)); err != nil { - return fmt.Errorf("could not record instance state: %v", err) - } - return nil -} - -func addr(inst *compute.Instance) string { - if inst == nil { - return "" - } - if len(inst.NetworkInterfaces) == 0 || inst.NetworkInterfaces[0] == nil { - return "" - } - if len(inst.NetworkInterfaces[0].AccessConfigs) == 0 || inst.NetworkInterfaces[0].AccessConfigs[0] == nil { - return "" - } - return inst.NetworkInterfaces[0].AccessConfigs[0].NatIP -} - -// creationState keeps information all along the creation process of the instance. The -// fields are only exported because we json encode them. -type creationState struct { - Err string `json:",omitempty"` // if non blank, creation failed. - InstConf blob.Ref // key to the user provided instance configuration. - InstAddr string // ip address of the instance. - InstHostname string // hostame (in the camlistore.net domain) of the instance - Success bool // whether new instance creation was successful. - Exists bool // true if an instance with same zone, same project name, and same instance name already exists. -} - -// dataStores returns the blobserver that stores the instances configurations, and the kv -// store for the instances states. -func dataStores() (blobserver.Storage, sorted.KeyValue, error) { - dataDir := os.Getenv("CAMLI_GCE_DATA") - if dataDir == "" { - var err error - dataDir, err = os.MkdirTemp("", "camli-gcedeployer-data") - if err != nil { - return nil, nil, err - } - log.Printf("data dir not provided as env var CAMLI_GCE_DATA, so defaulting to %v", dataDir) - } - blobsDir := filepath.Join(dataDir, "instance-conf") - if err := os.MkdirAll(blobsDir, 0700); err != nil { - return nil, nil, err - } - instConf, err := localdisk.New(blobsDir) - if err != nil { - return nil, nil, err - } - instState, err := leveldb.NewStorage(filepath.Join(dataDir, "instance-state")) - if err != nil { - return nil, nil, err - } - return instConf, instState, nil -} - -// TODO(mpl): AddTemplateTheme is a mistake, since the text argument is user -// input and hence can contain just any field, that is not a known field of -// TemplateData. Which will make the execution of the template fail. We should -// probably just somehow hardcode website/tmpl/page.html as the template. -// Or better, probably hardcode our own version of website/tmpl/page.html, -// because we don't want to be bound to whatever new template fields the -// website may need that we don't (such as .Domain). -// See issue #815, and issue #985. - -// AddTemplateTheme allows to enhance the aesthetics of the default template. To that -// effect, text can provide the template definitions for "header", "banner", "toplinks", and -// "footer". -func (h *DeployHandler) AddTemplateTheme(text string) error { - tpl, err := template.New("root").Parse(text + tplHTML()) - if err != nil { - return err - } - h.tplMu.Lock() - defer h.tplMu.Unlock() - h.tpl = tpl - return nil -} - -// TemplateData is the data passed for templates of tplHTML. -type TemplateData struct { - Title string - Help map[string]template.HTML // help bits within the form. - Hints []string // helping hints printed in case of an error. - Err error - Prefix string // handler prefix. - InstanceKey string // instance creation identifier, for the JS code to regularly poll for progress. - PiggyGIF string // URI to the piggy gif for progress animation. - Conf *InstanceConf // Configuration requested by the user - InstanceIP string `json:",omitempty"` // instance IP address that we display after successful creation. - InstanceHostname string `json:",omitempty"` - ProjectConsoleURL string - ProjectID string // set by us when we've just created a project on the behalf of the user - ZoneValues []string - MachineValues []string - CamliVersion string // git revision found in https://storage.googleapis.com/camlistore-release/docker/VERSION - - // Unused stuff, but needed by page.html. See TODO above, - // before AddTemplateTheme. - GoImportDomain string - GoImportUpstream string -} - -// empty definitions for "banner", "toplinks", and "footer" to avoid error on -// ExecuteTemplate when the definitions have not been added with AddTemplateTheme. -var noTheme = ` -{{define "header"}} - - Perkeep on Google Cloud - -{{end}} -{{define "banner"}} -{{end}} -{{define "toplinks"}} -{{end}} -{{define "footer"}} -{{end}} -` - -func tplHTML() string { - return ` - {{define "progress"}} - {{if .InstanceKey}} - - {{end}} - {{end}} - - {{define "messages"}} -
-

Perkeep on Google Cloud

- - {{if .InstanceIP}} - {{if .InstanceHostname}} -

Success. Your Perkeep instance is running at {{.InstanceHostname}}.

- {{else}} - -

Success. Your Perkeep instance is deployed at {{.InstanceIP}}. Refresh this page in a couple of minutes to know your hostname. Or go to camlistore-server instance, and look for camlistore-hostname (which might take a while to appear too) in the custom metadata section.

- {{end}} -

Please save the information on this page.

- -

First connection

-

- The password to access the web interface of your Perkeep instance was automatically generated. Go to the camlistore-server instance page to view it, and possibly change it. It is camlistore-password in the custom metadata section. Similarly, the username is camlistore-username. Then restart Perkeep from the /status page if you changed anything. -

- -

Further configuration

-

- Manage your instance at {{.ProjectConsoleURL}}. -

- -

- If you want to use your own HTTPS certificate and key, go to the storage browser. Delete "` + certFilename() + `", "` + keyFilename() + `", and replace them by uploading your own files (with the same names). Then restart Perkeep. -

- -

Perkeep should not require system -administration but to manage/add SSH keys, go to the camlistore-server -instance page. Scroll down to the SSH Keys section. Note that the -machine can be deleted or wiped at any time without losing -information. All state is stored in Cloud Storage. The index, however, -is stored in MySQL on the instance. The index can be rebuilt if lost -or corrupted.

- -

- {{end}} - {{if .Err}} -

Error: {{.Err}}

- {{range $hint := .Hints}} -

{{$hint}}

- {{end}} - {{end}} - {{end}} - -{{define "withform"}} - -{{template "header" .}} - - - - - - {{if .InstanceKey}} -
- {{end}} - {{template "banner" .}} - {{template "toplinks" .}} - {{template "progress" .}} - {{template "messages" .}} -
- - -

Deploy Perkeep

- -

This tool creates your own private -Perkeep instance running on Google Cloud Platform. Be sure to -understand Compute Engine pricing -and -Cloud Storage pricing -before proceeding. Note that Perkeep metadata adds overhead on top of the size -of any raw data added to your instance. To delete your -instance and stop paying Google for the virtual machine, visit the Google Cloud console -and visit both the "Compute Engine" and "Storage" sections for your project. -

- {{if .CamliVersion}} -

Perkeep version deployed by this tool: {{.CamliVersion}}

- {{end}} - - - - - - - - {{if .ProjectID}} - - {{else}} - - {{end}} - - -
Google Project ID: - {{if .ProjectID}} -
- Existing project ID:
- {{else}} -
- Existing project ID:
- {{end}} - You need to enable billing with Google for the selected project. -
Zone or Region: - {{if .ProjectID}} - - {{else}} - - {{end}} - - {{range $k, $v := .ZoneValues}} - - {{end}} -
If a region is specified, a random zone (-a, -b, -c, etc) in that region will be selected. -
Machine type: - {{if .ProjectID}} - - {{else}} - - {{end}} - - {{range $k, $v := .MachineValues}} - - {{end}} -
As of 2015-12-27, a g1-small is $13.88 (USD) per month, before storage usage charges. See current pricing. -
Use Google Cloud Free Tier
Use Google Cloud Free Tier
The Free Tier implies using an f1-micro instance, which might not be powerful enough in the long run. Also, the free storage is limited to 5GB, and must be in a US region. -
- {{if .ProjectID}} -
(it will ask for permissions) - {{else}} -
(it will ask for permissions) - {{end}} -
-
-
- {{template "footer" .}} - {{if .InstanceKey}} -
- {{end}} - - -{{end}} - -{{define "noform"}} - -{{template "header" .}} - - {{if .InstanceKey}} -
- {{end}} - {{template "banner" .}} - {{template "toplinks" .}} - {{template "progress" .}} - {{template "messages" .}} - {{template "footer" .}} - {{if .InstanceKey}} -
- {{end}} - - -{{end}} -` -} - -// TODO(bradfitz,mpl): move this to go4.org/cloud/google/gceutil -func ZonesOfRegion(hc *http.Client, project, region string) (zones []string, err error) { - s, err := compute.New(hc) - if err != nil { - return nil, err - } - zl, err := compute.NewZonesService(s).List(project).Do() - if err != nil { - return nil, fmt.Errorf("could not get a list of zones: %v", err) - } - if zl.NextPageToken != "" { - return nil, errors.New("TODO: more than one page of zones found; use NextPageToken") - } - for _, z := range zl.Items { - if path.Base(z.Region) != region { - continue - } - zones = append(zones, z.Name) - } - return zones, nil -} diff --git a/pkg/deploy/gce/notes.txt b/pkg/deploy/gce/notes.txt deleted file mode 100644 index e95ccee1c..000000000 --- a/pkg/deploy/gce/notes.txt +++ /dev/null @@ -1,31 +0,0 @@ -non-core dev: -gcutil --service_version="v1" --project="camanaged" addinstance "camlistore" --zone="us-central1-b" --machine_type="n1-standard-1" --network="default" --external_ip_address="107.178.214.163" --metadata="cam-key-1:cam-value-1" --metadata="cam-key-2:cam-value-2" --metadata="sshKeys:bradfitz:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw6Dk3iskKylP2zginCOAzIunMA38vGL9b/i18UG/Iuq+jKczZXB/1dlcZGSOs3+LtGh/C341TXTioydxTw+ux1AbmUk4c6L404skl85XFOys/GLxA4sHxBSb5we0Q57yohSgeZNlQd+Scmu5v7WC0N7I3hOK0lJgtxRNyC2nncGC0UOm+IGPTWcqPJERTauH/OhoAddWQehf1ugxTJYFU9atl3Op/mDXfyGBSLweWAQ84fhVKRZnl4i9Yhk1b357Q8cVKH6UQUADVamo7CQOsenzx99UL0thFRTSbuKALyf9e+SPwJrtIxZaX+skVSR+CzooRbypIamLbNXhfbxNz bradfitz@Bradleys-MacBook-Air.local" --service_account_scopes="https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/compute.readonly,https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/sqlservice,https://www.googleapis.com/auth/sqlservice.admin,https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/datastore,https://www.googleapis.com/auth/userinfo.email,https://www.googleapis.com/auth/compute,https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/taskqueue,https://www.googleapis.com/auth/bigquery,https://www.googleapis.com/auth/sqlservice,https://www.googleapis.com/auth/datastore" --tags="tag,tag2,http-server,https-server" --persistent_boot_disk="true" --auto_delete_boot_disk="false" --image=projects/debian-cloud/global/images/backports-debian-7-wheezy-v20140718 - -$ curl -H "Metadata-Flavor:Google" http://metadata/computeMetadata/v1/instance/service-accounts/default/scopes -https://www.googleapis.com/auth/bigquery -https://www.googleapis.com/auth/cloud-platform -https://www.googleapis.com/auth/compute -https://www.googleapis.com/auth/compute.readonly -https://www.googleapis.com/auth/datastore -https://www.googleapis.com/auth/devstorage.full_control -https://www.googleapis.com/auth/sqlservice -https://www.googleapis.com/auth/sqlservice.admin -https://www.googleapis.com/auth/taskqueue -https://www.googleapis.com/auth/userinfo.email - -gcutil --project=camanaged addinstance \ - --image=projects/coreos-cloud/global/images/coreos-alpha-394-0-0-v20140801 \ - --persistent_boot_disk \ - --zone=us-central1-a --machine_type=n1-standard-1 \ - --external_ip_address=107.178.208.16 \ - --auto_delete_boot_disk \ - --tags=http-server,https-server \ - --metadata_from_file=user-data:cloud-config.yaml core1 - -TODO: -- allow config from /gcs/bucket/key; add pkg for os.Stat/os.Open wrappers checking - prefix -- use that package for: - "httpsCert": "/home/bradfitz/keys/camlihouse/ssl.crt", - "httpsKey": "/home/bradfitz/keys/camlihouse/ssl.key", - "identitySecretRing": "/home/bradfitz/.config/perkeep/identity-secring.gpg", diff --git a/pkg/serverinit/env.go b/pkg/serverinit/env.go index 8f305e6c2..1544d7ead 100644 --- a/pkg/serverinit/env.go +++ b/pkg/serverinit/env.go @@ -85,7 +85,7 @@ func DefaultEnvConfig() (*Config, error) { externalIP, _ := metadata.ExternalIP() hostName, _ := metadata.InstanceAttributeValue("camlistore-hostname") - // If they specified a hostname (probably with pk-deploy), then: + // If they specified a hostname (previously common with old pk-deploy), then: // if it looks like an FQDN, perkeepd is going to rely on Let's // Encrypt, else perkeepd is going to generate some self-signed for that // hostname. diff --git a/website/pk-web/pkweb.go b/website/pk-web/pkweb.go index e45f125ed..c4000813c 100644 --- a/website/pk-web/pkweb.go +++ b/website/pk-web/pkweb.go @@ -21,7 +21,6 @@ import ( "context" "crypto/rand" "crypto/tls" - "encoding/json" "errors" "flag" "fmt" @@ -40,26 +39,19 @@ import ( txttemplate "text/template" "time" - "perkeep.org/internal/netutil" "perkeep.org/internal/osutil" "perkeep.org/pkg/buildinfo" - "perkeep.org/pkg/deploy/gce" "perkeep.org/pkg/types/camtypes" "cloud.google.com/go/compute/metadata" - "cloud.google.com/go/datastore" "cloud.google.com/go/logging" "cloud.google.com/go/storage" "github.com/mailgun/mailgun-go" "github.com/russross/blackfriday" - "go4.org/cloud/cloudlaunch" "go4.org/writerutil" "golang.org/x/crypto/acme/autocert" "golang.org/x/oauth2/google" - cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" - compute "google.golang.org/api/compute/v1" "google.golang.org/api/option" - storageapi "google.golang.org/api/storage/v1" ) const ( @@ -80,7 +72,6 @@ var ( tlsKeyFile = flag.String("tlskey", "", "TLS private key file") alsoRun = flag.String("also_run", "", "[optiona] Path to run as a child process. (Used to run perkeep.org's ./scripts/run-blob-server)") devMode = flag.Bool("dev", false, "in dev mode") - flagStaging = flag.Bool("staging", false, "Deploy to a test GCE instance. Requires -cloudlaunch=true") flagVersion = flag.Bool("version", false, "show version") gceProjectID = flag.String("gce_project_id", "", "GCE project ID; required if not running on GCE and gce_log_name is specified.") @@ -502,104 +493,6 @@ func runAsChild(res string) { }() } -func gceDeployHandlerConfig() (*gce.Config, error) { - if inProd { - return deployerCredsFromGCS() - } - clientId := os.Getenv("CAMLI_GCE_CLIENTID") - if clientId != "" { - return &gce.Config{ - ClientID: clientId, - ClientSecret: os.Getenv("CAMLI_GCE_CLIENTSECRET"), - Project: os.Getenv("CAMLI_GCE_PROJECT"), - ServiceAccount: os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT"), - DataDir: os.Getenv("CAMLI_GCE_DATA"), - }, nil - } - configDir, err := osutil.PerkeepConfigDir() - if err != nil { - return nil, err - } - configFile := filepath.Join(configDir, "launcher-config.json") - data, err := os.ReadFile(configFile) - if err != nil { - return nil, fmt.Errorf("error reading launcher-config.json (expected of type https://godoc.org/"+prodDomain+"/pkg/deploy/gce#Config): %v", err) - } - var config gce.Config - if err := json.Unmarshal(data, &config); err != nil { - return nil, err - } - return &config, nil -} - -var cloudLauncherEnabled = false - -// gceDeployHandler returns an http.Handler for a GCE launcher, -// configured to run at /prefix/ (the trailing slash can be omitted). -// The launcher is not initialized if: -// - in production, the launcher-config.json file is not found in the relevant bucket -// - neither CAMLI_GCE_CLIENTID is set, nor launcher-config.json is found in the -// camlistore server config dir. -func gceDeployHandler(prefix string) (*gce.DeployHandler, error) { - if !cloudLauncherEnabled { - return nil, errors.New("The Perkeep Cloud Launcher is no longer available.") - } - var hostPort string - var err error - scheme := "https" - if inProd { - if inStaging { - hostPort = stagingHostname + ":443" - } else { - hostPort = prodDomain + ":443" - } - } else { - addr := *httpsAddr - if *devMode && *httpsAddr == "" { - addr = *httpAddr - scheme = "http" - } - hostPort, err = netutil.ListenHostPort(addr) - if err != nil { - // the deploy handler needs to know its own - // hostname or IP for the oauth2 callback. - return nil, fmt.Errorf("invalid -https flag: %v", err) - } - } - config, err := gceDeployHandlerConfig() - if config == nil { - return nil, err - } - gceh, err := gce.NewDeployHandlerFromConfig(hostPort, prefix, config) - if err != nil { - return nil, fmt.Errorf("NewDeployHandlerFromConfig: %v", err) - } - - pageBytes, err := os.ReadFile(filepath.Join(*root, "tmpl", "page.html")) - if err != nil { - return nil, err - } - if err := gceh.AddTemplateTheme(string(pageBytes)); err != nil { - return nil, fmt.Errorf("AddTemplateTheme: %v", err) - } - gceh.SetScheme(scheme) - log.Printf("Starting Perkeep launcher on %s://%s%s", scheme, hostPort, prefix) - return gceh, nil -} - -var launchConfig = &cloudlaunch.Config{ - Name: "camweb", - BinaryBucket: prodBucket, - GCEProjectID: "camlistore-website", - Scopes: []string{ - storageapi.DevstorageFullControlScope, - compute.ComputeScope, - logging.WriteScope, - datastore.ScopeDatastore, - cloudresourcemanager.CloudPlatformScope, - }, -} - func checkInProduction() bool { if !metadata.OnGCE() { return false @@ -815,29 +708,6 @@ func projectID() string { return projID } -func initStaging() error { - if *flagStaging { - launchConfig.Name = stagingInstName - return nil - } - // If we are the instance that has just been deployed, we can't rely on - // *flagStaging, since there's no way to pass flags through launchConfig. - // And we need to know if we're a staging instance, so we can set - // launchConfig.Name properly before we get into restartLoop from - // MaybeDeploy. So we use our own instance name as a hint. - if !metadata.OnGCE() { - return nil - } - instName, err := metadata.InstanceName() - if err != nil { - return fmt.Errorf("Instance could not get its Instance Name: %v", err) - } - if instName == stagingInstName { - launchConfig.Name = stagingInstName - } - return nil -} - func main() { flag.Parse() if *flagVersion { @@ -845,10 +715,6 @@ func main() { buildinfo.Summary(), runtime.Version(), runtime.GOOS, runtime.GOARCH) return } - if err := initStaging(); err != nil { - log.Fatalf("Error setting up staging: %v", err) - } - launchConfig.MaybeDeploy() setProdFlags() if *root == "" { @@ -904,16 +770,9 @@ func main() { }) } - gceLauncher, err := gceDeployHandler("/launch/") - if err != nil { - log.Printf("Not installing GCE /launch/ handler: %v", err) - err := err - mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) { - http.Error(w, fmt.Sprintf("GCE launcher disabled: %v", err), 500) - }) - } else { - mux.Handle("/launch/", gceLauncher) - } + mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "GCE launcher no longer supported", 500) + }) var handler http.Handler = &redirectRootHandler{Handler: mux} if *logDir != "" || *logStdout { @@ -939,23 +798,6 @@ func main() { log.Fatalf("Failed to ping Google Cloud Logging: %v", err) } handler = NewLoggingHandler(handler, gceLogger{logc.Logger(*gceLogName)}) - if gceLauncher != nil { - var logc *logging.Client - if metadata.OnGCE() { - logc, err = logging.NewClient(ctx, projID) - } else { - logc, err = logging.NewClient(ctx, projID, option.WithHTTPClient(hc)) - } - if err != nil { - log.Fatal(err) - } - commonLabels := logging.CommonLabels(map[string]string{ - "from": "camli-gce-launcher", - }) - logger := logc.Logger(*gceLogName, commonLabels).StandardLogger(logging.Default) - logger.SetPrefix("launcher: ") - gceLauncher.SetLogger(logger) - } } emailErr := make(chan error) @@ -1092,18 +934,6 @@ func fromGCS(filename string) ([]byte, error) { return slurp(filename) } -func deployerCredsFromGCS() (*gce.Config, error) { - var cfg gce.Config - data, err := fromGCS("launcher-config.json") - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("could not JSON decode camli GCE launcher config: %v", err) - } - return &cfg, nil -} - var issueNum = regexp.MustCompile(`^/(?:issue|bug)s?(/\d*)?$`) // issueRedirect returns whether the request should be redirected to the