diff --git a/go.mod b/go.mod index 056bf6340..d22d47f3b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ toolchain go1.21.4 require ( bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 cloud.google.com/go/compute/metadata v0.2.3 - cloud.google.com/go/datastore v1.11.0 cloud.google.com/go/logging v1.7.0 cloud.google.com/go/storage v1.29.0 filippo.io/age v1.1.1 @@ -20,7 +19,6 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/hjfreyer/taglib-go v0.0.0-20151027170453-0ef8bba9c41b github.com/lib/pq v1.10.2 - github.com/mailgun/mailgun-go v0.0.0-20171127222028-17e8bd11e87c github.com/mattn/go-mastodon v0.0.5-0.20190517015615-8f6192e26b66 github.com/nf/cr2 v0.0.0-20140528043846-05d46fef4f2f github.com/pkg/sftp v1.13.6 @@ -75,9 +73,6 @@ require ( github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect - github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect - github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect - github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 // indirect diff --git a/go.sum b/go.sum index 2e6d8e5e2..ec6223c1d 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.11.0 h1:iF6I/HaLs3Ado8uRKMvZRvF/ZLkWaWE9i8AiHzbC774= -cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= @@ -120,12 +118,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= @@ -299,8 +291,6 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mailgun/mailgun-go v0.0.0-20171127222028-17e8bd11e87c h1:5huPh/MfWW65cx8KWNVD4mCCnwIrNiX4bFJR5OeONg0= -github.com/mailgun/mailgun-go v0.0.0-20171127222028-17e8bd11e87c/go.mod h1:NWTyU+O4aczg/nsGhQnvHL6v2n5Gy6Sv5tNDVvC6FbU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= diff --git a/website/pk-web/contributors.go b/website/pk-web/contributors.go index d7b216faa..492030ea3 100644 --- a/website/pk-web/contributors.go +++ b/website/pk-web/contributors.go @@ -6,13 +6,10 @@ import ( "fmt" "log" "net/http" - "os" "os/exec" "sort" "strconv" "strings" - - "perkeep.org/internal/osutil" ) var urlsMap = map[string]author{ @@ -77,30 +74,10 @@ func parseLine(l string) (name, email string, commits int, err error) { } func gitShortlog() *exec.Cmd { - if !*gitContainer { - return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen") + if *shortLogFile != "" { + return exec.Command("/bin/bash", "-c", "cat", *shortLogFile) } - args := []string{"run", "--rm"} - if inProd { - args = append(args, - "-v", "/var/camweb:/var/camweb", - "--workdir="+prodSrcDir, - ) - } else { - hostRoot, err := osutil.GoPackagePath(prodDomain) - if err != nil { - log.Fatal(err) - } - log.Printf("Using bind root of %q", hostRoot) - args = append(args, - "-v", hostRoot+":"+prodSrcDir, - "--workdir="+prodSrcDir, - ) - } - args = append(args, "camlistore/git", "/bin/bash", "-c", "git log | git shortlog -sen") - cmd := exec.Command("docker", args...) - cmd.Stderr = os.Stderr - return cmd + return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen") } func genContribPage() ([]byte, error) { diff --git a/website/pk-web/email.go b/website/pk-web/email.go deleted file mode 100644 index 42d60cd22..000000000 --- a/website/pk-web/email.go +++ /dev/null @@ -1,361 +0,0 @@ -/* -Copyright 2013 The Perkeep Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bytes" - "context" - "encoding/json" - "flag" - "fmt" - "log" - "net/http" - "os" - "os/exec" - "strings" - "sync" - "time" - - "cloud.google.com/go/datastore" - "github.com/mailgun/mailgun-go" - - "perkeep.org/internal/osutil" -) - -var ( - emailNow = flag.String("email_now", "", "[debug] if non-empty, this commit hash is emailed immediately, without starting the webserver.") - mailgunCfgFile = flag.String("mailgun_config", "", "[optional] Mailgun JSON configuration for sending emails on new commits.") - emailsTo = flag.String("email_dest", "", "[optional] The email address for new commit emails.") -) - -type mailgunCfg struct { - Domain string `json:"domain"` - APIKey string `json:"apiKey"` - PublicAPIKey string `json:"publicAPIKey"` -} - -// mailgun is for sending the camweb startup e-mail, and the commits e-mails. No -// e-mails are sent if it is nil. It is set in sendStartingEmail, and it is nil -// if mailgunCfgFile is not set. -var mailGun mailgun.Mailgun - -func mailgunCfgFromGCS() (*mailgunCfg, error) { - var cfg mailgunCfg - data, err := fromGCS(*mailgunCfgFile) - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("could not JSON decode website's mailgun config: %v", err) - } - return &cfg, nil -} - -func startEmailCommitLoop(errc chan<- error) { - if *emailsTo == "" { - return - } - if *emailNow != "" { - dir, err := osutil.GoPackagePath(prodDomain) - if err != nil { - log.Fatal(err) - } - if err := emailCommit(dir, *emailNow); err != nil { - log.Fatal(err) - } - os.Exit(0) - } - go func() { - errc <- commitEmailLoop() - }() -} - -// tokenc holds tokens for the /mailnow handler. -// Hitting /mailnow (unauthenticated) forces a 'git fetch origin -// master'. Because it's unauthenticated, we don't want to allow -// attackers to force us to hit git. The /mailnow handler tries to -// take a token from tokenc. -var tokenc = make(chan bool, 3) - -var fetchc = make(chan bool, 1) - -var knownCommit = map[string]bool{} // commit -> true - -var diffMarker = []byte("diff --git a/") - -func emailCommit(dir, hash string) (err error) { - if mailGun == nil { - return nil - } - - var body []byte - if err := emailOnTimeout("git show", 2*time.Minute, func() error { - cmd := execGit(dir, "show", nil, "show", hash) - body, err = cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("error running git show: %v\n%s", err, body) - } - return nil - }); err != nil { - return err - } - if !bytes.Contains(body, diffMarker) { - // Boring merge commit. Don't email. - return nil - } - - var out []byte - if err := emailOnTimeout("git show_pretty", 2*time.Minute, func() error { - cmd := execGit(dir, "show_pretty", nil, "show", "--pretty=oneline", hash) - out, err = cmd.Output() - if err != nil { - return fmt.Errorf("error running git show_pretty: %v\n%s", err, out) - } - return nil - }); err != nil { - return err - } - subj := out[41:] // remove hash and space - if i := bytes.IndexByte(subj, '\n'); i != -1 { - subj = subj[:i] - } - if len(subj) > 80 { - subj = subj[:80] - } - - contents := fmt.Sprintf(` - -https://github.com/perkeep/perkeep/commit/%s - -%s`, hash, body) - - m := mailGun.NewMessage( - "noreply@perkeep.org", - string(subj), - contents, - *emailsTo, - ) - m.SetReplyTo("camlistore-commits@googlegroups.com") - if _, _, err := mailGun.Send(m); err != nil { - return fmt.Errorf("failed to send e-mail: %v", err) - } - return nil -} - -var latestHash struct { - sync.Mutex - s string // hash of the most recent perkeep revision -} - -// dsClient is our datastore client to track which commits we've -// emailed about. It's only non-nil in production. -var dsClient *datastore.Client - -func commitEmailLoop() error { - http.HandleFunc("/mailnow", mailNowHandler) - - var err error - dsClient, err = datastore.NewClient(context.Background(), "camlistore-website") - log.Printf("datastore = %v, %v", dsClient, err) - - go func() { - for { - select { - case tokenc <- true: - default: - } - time.Sleep(15 * time.Second) - } - }() - - dir := pkSrcDir() - - http.HandleFunc("/latesthash", latestHashHandler) - http.HandleFunc("/debug/email", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "ds = %v, %v", dsClient, err) - }) - - for { - pollCommits(dir) - - // Poll every minute or whenever we're forced with the - // /mailnow handler. - select { - case <-time.After(1 * time.Minute): - case <-fetchc: - log.Printf("Polling git due to explicit trigger.") - } - } -} - -// emailOnTimeout runs fn in a goroutine. If fn is not done after d, -// a message about fnName is logged, and an e-mail about it is sent. -func emailOnTimeout(fnName string, d time.Duration, fn func() error) error { - c := make(chan error, 1) - go func() { - c <- fn() - }() - select { - case <-time.After(d): - log.Printf("timeout for %s, sending e-mail about it", fnName) - m := mailGun.NewMessage( - "noreply@perkeep.org", - "timeout for docker on pk-web", - "Because "+fnName+" is stuck.", - "mathieu.lonjaret@gmail.com", - ) - if _, _, err := mailGun.Send(m); err != nil { - return fmt.Errorf("failed to send docker restart e-mail: %v", err) - } - return nil - case err := <-c: - return err - } -} - -// execGit runs the git command with gitArgs. All the other arguments are only -// relevant if *gitContainer, in which case we run in a docker container. -func execGit(workdir string, containerName string, mounts map[string]string, gitArgs ...string) *exec.Cmd { - var cmd *exec.Cmd - if *gitContainer { - removeContainer(containerName) - args := []string{ - "run", - "--rm", - "--name=" + containerName, - } - for host, container := range mounts { - args = append(args, "-v", host+":"+container+":ro") - } - args = append(args, []string{ - "-v", workdir + ":" + workdir, - "--workdir=" + workdir, - "camlistore/git", - "git"}...) - args = append(args, gitArgs...) - cmd = exec.Command("docker", args...) - } else { - cmd = exec.Command("git", gitArgs...) - cmd.Dir = workdir - } - return cmd -} - -// GitCommit is a datastore entity to track which commits we've -// already emailed about. -type GitCommit struct { - Emailed bool -} - -func pollCommits(dir string) { - if err := emailOnTimeout("git pull_origin", 5*time.Minute, func() error { - cmd := execGit(dir, "pull_origin", nil, "pull", "origin") - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("error running git pull origin master in %s: %v\n%s", dir, err, out) - } - return nil - }); err != nil { - log.Printf("%v", err) - return - } - log.Printf("Ran git pull.") - // TODO: see if .git/refs/remotes/origin/master - // changed. (quicker than running recentCommits each time) - - hashes, err := recentCommits(dir) - if err != nil { - log.Print(err) - return - } - if len(hashes) == 0 { - return - } - latestHash.Lock() - latestHash.s = hashes[0] - latestHash.Unlock() - for _, commit := range hashes { - if knownCommit[commit] { - continue - } - if dsClient != nil { - ctx := context.Background() - key := datastore.NameKey("git_commit", commit, nil) - var gc GitCommit - if err := dsClient.Get(ctx, key, &gc); err == nil && gc.Emailed { - log.Printf("Already emailed about commit %v; skipping", commit) - knownCommit[commit] = true - continue - } - } - if err := emailCommit(dir, commit); err != nil { - log.Printf("Error with commit e-mail: %v", err) - continue - } - log.Printf("Emailed commit %s", commit) - knownCommit[commit] = true - if dsClient != nil { - ctx := context.Background() - key := datastore.NameKey("git_commit", commit, nil) - _, err := dsClient.Put(ctx, key, &GitCommit{Emailed: true}) - log.Printf("datastore put of git_commit(%v): %v", commit, err) - } - } -} - -func recentCommits(dir string) (hashes []string, err error) { - var out []byte - if err := emailOnTimeout("git log_origin_master", 2*time.Minute, func() error { - cmd := execGit(dir, "log_origin_master", nil, "log", "--since=1 month ago", "--pretty=oneline", "origin/master") - out, err = cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("error running git log in %s: %v\n%s", dir, err, out) - } - return nil - }); err != nil { - return nil, err - } - for _, line := range strings.Split(string(out), "\n") { - v := strings.SplitN(line, " ", 2) - if len(v) > 1 { - hashes = append(hashes, v[0]) - } - } - return -} - -func mailNowHandler(w http.ResponseWriter, r *http.Request) { - select { - case <-tokenc: - log.Printf("/mailnow got a token") - default: - // Too many requests. Ignore. - log.Printf("Ignoring /mailnow request; too soon.") - return - } - select { - case fetchc <- true: - log.Printf("/mailnow triggered a git fetch") - default: - } -} - -func latestHashHandler(w http.ResponseWriter, r *http.Request) { - latestHash.Lock() - defer latestHash.Unlock() - fmt.Fprint(w, latestHash.s) -} diff --git a/website/pk-web/logging.go b/website/pk-web/logging.go index 0d3164602..b08ae9a66 100644 --- a/website/pk-web/logging.go +++ b/website/pk-web/logging.go @@ -7,8 +7,6 @@ import ( "os" "strings" "time" - - "cloud.google.com/go/logging" ) type logRecord struct { @@ -146,23 +144,3 @@ func (lr *logRecord) WriteHeader(status int) { lr.responseStatus = status lr.ResponseWriter.WriteHeader(status) } - -type gceLogger struct { - c *logging.Logger -} - -func (lg gceLogger) LogEvent(lr *logRecord) { - lg.c.Log(logging.Entry{ - Timestamp: lr.time, - Payload: map[string]interface{}{ - "ip": lr.ip, - "path": lr.rawpath, - "method": lr.method, - "responseBytes": lr.responseBytes, - "status": lr.responseStatus, - "userAgent": lr.userAgent, - "referer": lr.referer, - "proto": lr.proto, - }, - }) -} diff --git a/website/pk-web/pkweb.go b/website/pk-web/pkweb.go index c4000813c..bf5251c28 100644 --- a/website/pk-web/pkweb.go +++ b/website/pk-web/pkweb.go @@ -18,8 +18,6 @@ package main // import "perkeep.org/website/pk-web" import ( "bytes" - "context" - "crypto/rand" "crypto/tls" "errors" "flag" @@ -39,19 +37,11 @@ import ( txttemplate "text/template" "time" - "perkeep.org/internal/osutil" "perkeep.org/pkg/buildinfo" "perkeep.org/pkg/types/camtypes" - "cloud.google.com/go/compute/metadata" - "cloud.google.com/go/logging" - "cloud.google.com/go/storage" - "github.com/mailgun/mailgun-go" "github.com/russross/blackfriday" - "go4.org/writerutil" "golang.org/x/crypto/acme/autocert" - "golang.org/x/oauth2/google" - "google.golang.org/api/option" ) const ( @@ -63,37 +53,21 @@ const ( var h1TitlePattern = regexp.MustCompile(`]*>([^<]+)`) var ( - httpAddr = flag.String("http", defaultAddr, "HTTP address. If using Let's Encrypt, this server needs to be able to answer the http-01 challenge on port 80.") - httpsAddr = flag.String("https", "", "HTTPS address") - root = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl)") - logDir = flag.String("logdir", "", "Directory to write log files to (one per hour), or empty to not log.") - logStdout = flag.Bool("logstdout", true, "Whether to log to stdout") - tlsCertFile = flag.String("tlscert", "", "TLS cert file") - tlsKeyFile = flag.String("tlskey", "", "TLS private key file") - alsoRun = flag.String("also_run", "", "[optiona] Path to run as a child process. (Used to run perkeep.org's ./scripts/run-blob-server)") - devMode = flag.Bool("dev", false, "in dev mode") - flagVersion = flag.Bool("version", false, "show version") - - gceProjectID = flag.String("gce_project_id", "", "GCE project ID; required if not running on GCE and gce_log_name is specified.") - gceLogName = flag.String("gce_log_name", "", "GCE Cloud Logging log name; if non-empty, logs go to Cloud Logging instead of Apache-style local disk log files") - gceJWTFile = flag.String("gce_jwt_file", "", "If non-empty, a filename to the GCE Service Account's JWT (JSON) config file.") - gitContainer = flag.Bool("git_container", false, "Use git from the `camlistore/git` Docker container; if false, the system `git` is used.") - - adminEmail = flag.String("email", "", "Address that Let's Encrypt will notify about problems with issued certificates") -) - -const ( - stagingInstName = "camweb-staging" // name of the GCE instance when testing - stagingHostname = "staging.camlistore.net" + httpAddr = flag.String("http", defaultAddr, "HTTP address. If using Let's Encrypt, this server needs to be able to answer the http-01 challenge on port 80.") + httpsAddr = flag.String("https", "", "HTTPS address") + root = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl)") + logDir = flag.String("logdir", "", "Directory to write log files to (one per hour), or empty to not log.") + logStdout = flag.Bool("logstdout", true, "Whether to log to stdout") + tlsCertFile = flag.String("tlscert", "", "TLS cert file") + tlsKeyFile = flag.String("tlskey", "", "TLS private key file") + alsoRun = flag.String("also_run", "", "[optiona] Path to run as a child process. (Used to run perkeep.org's ./scripts/run-blob-server)") + flagVersion = flag.Bool("version", false, "show version") + adminEmail = flag.String("email", "", "Address that Let's Encrypt will notify about problems with issued certificates") + shortLogFile = flag.String("gitlog-file", "", "If non-empty, the path to the `git log | git shortlog -sen output` to use. If empty, it's run as needed.") ) var ( inProd bool - // inStaging is whether this instance is the staging server. This should only be true - // if inProd is also true - they are not mutually exclusive; staging is still prod - - // because we want to test the same code paths as in production. The code then runs - // on another GCE instance, and on the stagingHostname host. - inStaging bool pageHTML, errorHTML, camliErrorHTML *template.Template packageHTML *txttemplate.Template @@ -462,10 +436,6 @@ func (h *redirectRootHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) host := strings.ToLower(r.Host) if host == "www.camlistore.org" || host == "camlistore.org" || host == "www."+prodDomain || (inProd && r.TLS == nil) { - if inStaging { - http.Redirect(rw, r, "https://"+stagingHostname+r.URL.RequestURI(), http.StatusFound) - return - } http.Redirect(rw, r, "https://"+prodDomain+r.URL.RequestURI(), http.StatusFound) return } @@ -493,126 +463,28 @@ func runAsChild(res string) { }() } -func checkInProduction() bool { - if !metadata.OnGCE() { - return false - } - proj, _ := metadata.ProjectID() - inst, _ := metadata.InstanceName() - log.Printf("Running on GCE: %v / %v", proj, inst) - prod := proj == "camlistore-website" && inst == "camweb" || inst == stagingInstName - inStaging = prod && inst == stagingInstName - return prod -} - -const ( - prodSrcDir = "/var/camweb/src/" + prodDomain - prodLECacheDir = "/var/le/letsencrypt.cache" -) - -func setProdFlags() { - inProd = checkInProduction() - if !inProd { - return - } - if *devMode { - log.Fatal("can't use dev mode in production") - } - log.Printf("Running in production; configuring prod flags & containers") - *httpAddr = ":80" - *httpsAddr = ":443" - // TODO(mpl): investigate why this proxying does not seem to be working (we end up on https://camlistore.org). - buildbotBackend = "https://travis-ci.org/perkeep/perkeep" - buildbotHost = "build.perkeep.org" - *gceLogName = "camweb-access-log" - if inStaging { - *gceLogName += "-staging" - } - *root = filepath.Join(prodSrcDir, "website") - *gitContainer = true - - *adminEmail = "mathieu.lonjaret@gmail.com" // for let's encrypt - *emailsTo = "camlistore-commits@googlegroups.com" - *mailgunCfgFile = "mailgun-config.json" - if inStaging { - // in staging, keep emailsTo so we get in the loop that does the - // git pull and refreshes the content, but no mailgunCfgFile so - // we don't actually try to send any e-mail. - *mailgunCfgFile = "" - } - - os.RemoveAll(prodSrcDir) - if err := os.MkdirAll(prodSrcDir, 0755); err != nil { - log.Fatal(err) - } - log.Printf("fetching git docker image...") - getDockerImage("camlistore/git", "docker-git.tar.gz") - getDockerImage("camlistore/demoblobserver", "docker-demoblobserver.tar.gz") - - log.Printf("cloning perkeep git tree...") - branch := "master" - if inStaging { - // We work off the staging branch, so we stay in control of the - // website contents, regardless of which commits are landing on the - // master branch in the meantime. - branch = "staging" - } - cloneArgs := []string{ - "run", - "--rm", - "-v", "/var/camweb:/var/camweb", - "camlistore/git", - "git", "clone", "-b", branch, "https://github.com/perkeep/perkeep.git", prodSrcDir, - } - out, err := exec.Command("docker", cloneArgs...).CombinedOutput() - if err != nil { - log.Fatalf("git clone: %v, %s", err, out) - } - os.Chdir(*root) - log.Printf("Starting.") - sendStartingEmail() -} - -func randHex(n int) string { - buf := make([]byte, n/2+1) - rand.Read(buf) - return fmt.Sprintf("%x", buf)[:n] -} - -func removeContainer(name string) { - if err := exec.Command("docker", "kill", name).Run(); err == nil { - // It was actually running. - log.Printf("Killed old %q container.", name) - } - if err := exec.Command("docker", "rm", name).Run(); err == nil { - // Always try to remove, in case we end up with a stale, - // non-running one (which has happened in the past). - log.Printf("Removed old %q container.", name) - } -} - // runDemoBlobServerContainer runs the demo blobserver as name in a docker // container. It is not run in daemon mode, so it never returns if successful. func runDemoBlobServerContainer(name string) error { - removeContainer(name) - cmd := exec.Command("docker", "run", - "--rm", - "--name="+name, - "-e", "CAMLI_ROOT="+prodSrcDir+"/website/blobserver-example/root", - "-e", "CAMLI_PASSWORD="+randHex(20), - "-v", pkSrcDir()+":"+prodSrcDir, - "--net=host", - "--workdir="+prodSrcDir, - "camlistore/demoblobserver", - "camlistored", - "--openbrowser=false", - "--listen=:3179", - "--configfile="+prodSrcDir+"/website/blobserver-example/example-blobserver-config.json") - stderr := &writerutil.PrefixSuffixSaver{N: 32 << 10} - cmd.Stderr = stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to run demo blob server: %v, stderr: %v", err, string(stderr.Bytes())) - } + // removeContainer(name) + // cmd := exec.Command("docker", "run", + // "--rm", + // "--name="+name, + // "-e", "CAMLI_ROOT="+prodSrcDir+"/website/blobserver-example/root", + // "-e", "CAMLI_PASSWORD="+randHex(20), + // "-v", pkSrcDir()+":"+prodSrcDir, + // "--net=host", + // "--workdir="+prodSrcDir, + // "camlistore/demoblobserver", + // "camlistored", + // "--openbrowser=false", + // "--listen=:3179", + // "--configfile="+prodSrcDir+"/website/blobserver-example/example-blobserver-config.json") + // stderr := &writerutil.PrefixSuffixSaver{N: 32 << 10} + // cmd.Stderr = stderr + // if err := cmd.Run(); err != nil { + // return fmt.Errorf("failed to run demo blob server: %v, stderr: %v", err, string(stderr.Bytes())) + // } return nil } @@ -635,79 +507,6 @@ func runDemoBlobserverLoop() { } } -func sendStartingEmail() { - if *mailgunCfgFile == "" { - return - } - contentRev, err := exec.Command("docker", "run", - "--rm", - "-v", "/var/camweb:/var/camweb", - "-w", prodSrcDir, - "camlistore/git", - "/bin/bash", "-c", - "git show --pretty=format:'%ad-%h' --abbrev-commit --date=short | head -1").Output() - - cfg, err := mailgunCfgFromGCS() - if err != nil { - log.Printf("Failed to get mailgun config: %v", err) - return - } - mailGun = mailgun.NewMailgun(cfg.Domain, cfg.APIKey, cfg.PublicAPIKey) - contents := `Perkeep website starting with binary ` + buildinfo.Summary() + ` and content at git rev ` + string(contentRev) - m := mailGun.NewMessage( - "noreply@perkeep.org (Perkeep Website)", - "Perkeep camweb restarting", - contents, - "brad@danga.com", - "mathieu.lonjaret@gmail.com", - ) - if _, _, err := mailGun.Send(m); err != nil { - log.Printf("Failed to send camweb startup e-mail: %v", err) - } -} - -func getDockerImage(tag, file string) { - have, err := exec.Command("docker", "inspect", tag).Output() - if err == nil && len(have) > 0 { - return // we have it. - } - url := "https://storage.googleapis.com/" + prodBucket + "/" + file - err = exec.Command("/bin/bash", "-c", "curl --silent "+url+" | docker load").Run() - if err != nil { - log.Fatal(err) - } -} - -// httpClient returns an http Client suitable for Google Cloud Storage or Google Cloud -// Logging calls with the projID project ID. -func httpClient(projID string) *http.Client { - if *gceJWTFile == "" { - log.Fatal("Cannot initialize an authorized http Client without --gce_jwt_file") - } - jsonSlurp, err := os.ReadFile(*gceJWTFile) - if err != nil { - log.Fatalf("Error reading --gce_jwt_file value: %v", err) - } - jwtConf, err := google.JWTConfigFromJSON(jsonSlurp, logging.WriteScope) - if err != nil { - log.Fatalf("Error reading --gce_jwt_file value: %v", err) - } - return jwtConf.Client(context.Background()) -} - -// projectID returns the GCE project ID used for running this camweb on GCE -// and/or for logging on Google Cloud Logging, if any. -func projectID() string { - if *gceProjectID != "" { - return *gceProjectID - } - projID, err := metadata.ProjectID() - if projID == "" || err != nil { - log.Fatalf("GCE project ID needed but --gce_project_id not specified (and not running on GCE); metadata error: %v", err) - } - return projID -} - func main() { flag.Parse() if *flagVersion { @@ -715,7 +514,6 @@ func main() { buildinfo.Summary(), runtime.Version(), runtime.GOOS, runtime.GOARCH) return } - setProdFlags() if *root == "" { var err error @@ -778,30 +576,6 @@ func main() { if *logDir != "" || *logStdout { handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout)) } - if *gceLogName != "" { - projID := projectID() - var hc *http.Client - if !metadata.OnGCE() { - hc = httpClient(projID) - } - ctx := context.Background() - var logc *logging.Client - if metadata.OnGCE() { - logc, err = logging.NewClient(ctx, projID) - } else { - logc, err = logging.NewClient(ctx, projID, option.WithHTTPClient(hc)) - } - if err != nil { - log.Fatal(err) - } - if err := logc.Ping(ctx); err != nil { - log.Fatalf("Failed to ping Google Cloud Logging: %v", err) - } - handler = NewLoggingHandler(handler, gceLogger{logc.Logger(*gceLogName)}) - } - - emailErr := make(chan error) - startEmailCommitLoop(emailErr) if *alsoRun != "" { runAsChild(*alsoRun) @@ -822,8 +596,6 @@ func main() { }() select { - case err := <-emailErr: - log.Fatalf("Error sending emails: %v", err) case err := <-httpsErr: log.Fatalf("Error serving HTTPS: %v", err) } @@ -874,13 +646,9 @@ func serve(httpServer *http.Server, onHTTPError func(error)) error { } hostPolicy = autocert.HostWhitelist(host) } else { - if inStaging { - hostPolicy = autocert.HostWhitelist(stagingHostname) - } else { - hostPolicy = autocert.HostWhitelist(prodDomain, "www."+prodDomain, - "www.camlistore.org", "camlistore.org") - } - cacheDir = autocert.DirCache(prodLECacheDir) + hostPolicy = autocert.HostWhitelist(prodDomain, "www."+prodDomain, + "www.camlistore.org", "camlistore.org") + cacheDir = autocert.DirCache("/var/le/letsencrypt.cache") } m := autocert.Manager{ Prompt: autocert.AcceptTOS, @@ -917,23 +685,6 @@ func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { return tc, nil } -func fromGCS(filename string) ([]byte, error) { - ctx := context.Background() - sc, err := storage.NewClient(ctx) - if err != nil { - return nil, err - } - slurp := func(key string) ([]byte, error) { - rc, err := sc.Bucket(prodBucket).Object(key).NewReader(ctx) - if err != nil { - return nil, fmt.Errorf("Error fetching GCS object %q in bucket %q: %v", key, prodBucket, err) - } - defer rc.Close() - return io.ReadAll(rc) - } - return slurp(filename) -} - var issueNum = regexp.MustCompile(`^/(?:issue|bug)s?(/\d*)?$`) // issueRedirect returns whether the request should be redirected to the @@ -1047,14 +798,3 @@ func errHandler(w http.ResponseWriter, r *http.Request) { content: contents, }) } - -func pkSrcDir() string { - if inProd { - return prodSrcDir - } - dir, err := osutil.GoPackagePath(prodDomain) - if err != nil { - log.Fatalf("Failed to find the root of the %s source code via osutil.GoPackagePath: %v", prodDomain, err) - } - return dir -}