mirror of https://github.com/perkeep/perkeep.git
website/pk-web: remove GCE+mailgun support
I moved the website to run on Fly.io some 6 months ago but lost the git commit that did so. So now I'm redoing it from memory (sigh). But we don't need any of this GCE or Mailgun stuff now. People can subscribe to git commits via other means. Signed-off-by: Brad Fitzpatrick <brad@danga.com>
This commit is contained in:
Brad Fitzpatrick
parent
0caf36bc9c
commit
e1a04f92ed
5
go.mod
5
go.mod
|
|
@ -7,7 +7,6 @@ toolchain go1.21.4
|
|||
require (
|
||||
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
|
||||
cloud.google.com/go/compute/metadata v0.2.3
|
||||
cloud.google.com/go/datastore v1.11.0
|
||||
cloud.google.com/go/logging v1.7.0
|
||||
cloud.google.com/go/storage v1.29.0
|
||||
filippo.io/age v1.1.1
|
||||
|
|
@ -20,7 +19,6 @@ require (
|
|||
github.com/gorilla/websocket v1.4.2
|
||||
github.com/hjfreyer/taglib-go v0.0.0-20151027170453-0ef8bba9c41b
|
||||
github.com/lib/pq v1.10.2
|
||||
github.com/mailgun/mailgun-go v0.0.0-20171127222028-17e8bd11e87c
|
||||
github.com/mattn/go-mastodon v0.0.5-0.20190517015615-8f6192e26b66
|
||||
github.com/nf/cr2 v0.0.0-20140528043846-05d46fef4f2f
|
||||
github.com/pkg/sftp v1.13.6
|
||||
|
|
@ -75,9 +73,6 @@ require (
|
|||
github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/edsrzf/mmap-go v1.1.0 // indirect
|
||||
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.5.0 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 // indirect
|
||||
|
|
|
|||
10
go.sum
10
go.sum
|
|
@ -18,8 +18,6 @@ cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi
|
|||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.11.0 h1:iF6I/HaLs3Ado8uRKMvZRvF/ZLkWaWE9i8AiHzbC774=
|
||||
cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c=
|
||||
cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k=
|
||||
cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
|
||||
cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I=
|
||||
|
|
@ -120,12 +118,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
|
|||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
|
||||
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
|
||||
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
|
||||
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
|
||||
github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
|
|
@ -299,8 +291,6 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
|
|||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
|
||||
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/mailgun/mailgun-go v0.0.0-20171127222028-17e8bd11e87c h1:5huPh/MfWW65cx8KWNVD4mCCnwIrNiX4bFJR5OeONg0=
|
||||
github.com/mailgun/mailgun-go v0.0.0-20171127222028-17e8bd11e87c/go.mod h1:NWTyU+O4aczg/nsGhQnvHL6v2n5Gy6Sv5tNDVvC6FbU=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
|
|
|
|||
|
|
@ -6,13 +6,10 @@ import (
|
|||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"perkeep.org/internal/osutil"
|
||||
)
|
||||
|
||||
var urlsMap = map[string]author{
|
||||
|
|
@ -77,30 +74,10 @@ func parseLine(l string) (name, email string, commits int, err error) {
|
|||
}
|
||||
|
||||
func gitShortlog() *exec.Cmd {
|
||||
if !*gitContainer {
|
||||
return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen")
|
||||
if *shortLogFile != "" {
|
||||
return exec.Command("/bin/bash", "-c", "cat", *shortLogFile)
|
||||
}
|
||||
args := []string{"run", "--rm"}
|
||||
if inProd {
|
||||
args = append(args,
|
||||
"-v", "/var/camweb:/var/camweb",
|
||||
"--workdir="+prodSrcDir,
|
||||
)
|
||||
} else {
|
||||
hostRoot, err := osutil.GoPackagePath(prodDomain)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("Using bind root of %q", hostRoot)
|
||||
args = append(args,
|
||||
"-v", hostRoot+":"+prodSrcDir,
|
||||
"--workdir="+prodSrcDir,
|
||||
)
|
||||
}
|
||||
args = append(args, "camlistore/git", "/bin/bash", "-c", "git log | git shortlog -sen")
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd
|
||||
return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen")
|
||||
}
|
||||
|
||||
func genContribPage() ([]byte, error) {
|
||||
|
|
|
|||
|
|
@ -1,361 +0,0 @@
|
|||
/*
|
||||
Copyright 2013 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/datastore"
|
||||
"github.com/mailgun/mailgun-go"
|
||||
|
||||
"perkeep.org/internal/osutil"
|
||||
)
|
||||
|
||||
var (
|
||||
emailNow = flag.String("email_now", "", "[debug] if non-empty, this commit hash is emailed immediately, without starting the webserver.")
|
||||
mailgunCfgFile = flag.String("mailgun_config", "", "[optional] Mailgun JSON configuration for sending emails on new commits.")
|
||||
emailsTo = flag.String("email_dest", "", "[optional] The email address for new commit emails.")
|
||||
)
|
||||
|
||||
type mailgunCfg struct {
|
||||
Domain string `json:"domain"`
|
||||
APIKey string `json:"apiKey"`
|
||||
PublicAPIKey string `json:"publicAPIKey"`
|
||||
}
|
||||
|
||||
// mailgun is for sending the camweb startup e-mail, and the commits e-mails. No
|
||||
// e-mails are sent if it is nil. It is set in sendStartingEmail, and it is nil
|
||||
// if mailgunCfgFile is not set.
|
||||
var mailGun mailgun.Mailgun
|
||||
|
||||
func mailgunCfgFromGCS() (*mailgunCfg, error) {
|
||||
var cfg mailgunCfg
|
||||
data, err := fromGCS(*mailgunCfgFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("could not JSON decode website's mailgun config: %v", err)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func startEmailCommitLoop(errc chan<- error) {
|
||||
if *emailsTo == "" {
|
||||
return
|
||||
}
|
||||
if *emailNow != "" {
|
||||
dir, err := osutil.GoPackagePath(prodDomain)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := emailCommit(dir, *emailNow); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
go func() {
|
||||
errc <- commitEmailLoop()
|
||||
}()
|
||||
}
|
||||
|
||||
// tokenc holds tokens for the /mailnow handler.
|
||||
// Hitting /mailnow (unauthenticated) forces a 'git fetch origin
|
||||
// master'. Because it's unauthenticated, we don't want to allow
|
||||
// attackers to force us to hit git. The /mailnow handler tries to
|
||||
// take a token from tokenc.
|
||||
var tokenc = make(chan bool, 3)
|
||||
|
||||
var fetchc = make(chan bool, 1)
|
||||
|
||||
var knownCommit = map[string]bool{} // commit -> true
|
||||
|
||||
var diffMarker = []byte("diff --git a/")
|
||||
|
||||
func emailCommit(dir, hash string) (err error) {
|
||||
if mailGun == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var body []byte
|
||||
if err := emailOnTimeout("git show", 2*time.Minute, func() error {
|
||||
cmd := execGit(dir, "show", nil, "show", hash)
|
||||
body, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error running git show: %v\n%s", err, body)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Contains(body, diffMarker) {
|
||||
// Boring merge commit. Don't email.
|
||||
return nil
|
||||
}
|
||||
|
||||
var out []byte
|
||||
if err := emailOnTimeout("git show_pretty", 2*time.Minute, func() error {
|
||||
cmd := execGit(dir, "show_pretty", nil, "show", "--pretty=oneline", hash)
|
||||
out, err = cmd.Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error running git show_pretty: %v\n%s", err, out)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
subj := out[41:] // remove hash and space
|
||||
if i := bytes.IndexByte(subj, '\n'); i != -1 {
|
||||
subj = subj[:i]
|
||||
}
|
||||
if len(subj) > 80 {
|
||||
subj = subj[:80]
|
||||
}
|
||||
|
||||
contents := fmt.Sprintf(`
|
||||
|
||||
https://github.com/perkeep/perkeep/commit/%s
|
||||
|
||||
%s`, hash, body)
|
||||
|
||||
m := mailGun.NewMessage(
|
||||
"noreply@perkeep.org",
|
||||
string(subj),
|
||||
contents,
|
||||
*emailsTo,
|
||||
)
|
||||
m.SetReplyTo("camlistore-commits@googlegroups.com")
|
||||
if _, _, err := mailGun.Send(m); err != nil {
|
||||
return fmt.Errorf("failed to send e-mail: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var latestHash struct {
|
||||
sync.Mutex
|
||||
s string // hash of the most recent perkeep revision
|
||||
}
|
||||
|
||||
// dsClient is our datastore client to track which commits we've
|
||||
// emailed about. It's only non-nil in production.
|
||||
var dsClient *datastore.Client
|
||||
|
||||
func commitEmailLoop() error {
|
||||
http.HandleFunc("/mailnow", mailNowHandler)
|
||||
|
||||
var err error
|
||||
dsClient, err = datastore.NewClient(context.Background(), "camlistore-website")
|
||||
log.Printf("datastore = %v, %v", dsClient, err)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case tokenc <- true:
|
||||
default:
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
}()
|
||||
|
||||
dir := pkSrcDir()
|
||||
|
||||
http.HandleFunc("/latesthash", latestHashHandler)
|
||||
http.HandleFunc("/debug/email", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "ds = %v, %v", dsClient, err)
|
||||
})
|
||||
|
||||
for {
|
||||
pollCommits(dir)
|
||||
|
||||
// Poll every minute or whenever we're forced with the
|
||||
// /mailnow handler.
|
||||
select {
|
||||
case <-time.After(1 * time.Minute):
|
||||
case <-fetchc:
|
||||
log.Printf("Polling git due to explicit trigger.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// emailOnTimeout runs fn in a goroutine. If fn is not done after d,
|
||||
// a message about fnName is logged, and an e-mail about it is sent.
|
||||
func emailOnTimeout(fnName string, d time.Duration, fn func() error) error {
|
||||
c := make(chan error, 1)
|
||||
go func() {
|
||||
c <- fn()
|
||||
}()
|
||||
select {
|
||||
case <-time.After(d):
|
||||
log.Printf("timeout for %s, sending e-mail about it", fnName)
|
||||
m := mailGun.NewMessage(
|
||||
"noreply@perkeep.org",
|
||||
"timeout for docker on pk-web",
|
||||
"Because "+fnName+" is stuck.",
|
||||
"mathieu.lonjaret@gmail.com",
|
||||
)
|
||||
if _, _, err := mailGun.Send(m); err != nil {
|
||||
return fmt.Errorf("failed to send docker restart e-mail: %v", err)
|
||||
}
|
||||
return nil
|
||||
case err := <-c:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// execGit runs the git command with gitArgs. All the other arguments are only
|
||||
// relevant if *gitContainer, in which case we run in a docker container.
|
||||
func execGit(workdir string, containerName string, mounts map[string]string, gitArgs ...string) *exec.Cmd {
|
||||
var cmd *exec.Cmd
|
||||
if *gitContainer {
|
||||
removeContainer(containerName)
|
||||
args := []string{
|
||||
"run",
|
||||
"--rm",
|
||||
"--name=" + containerName,
|
||||
}
|
||||
for host, container := range mounts {
|
||||
args = append(args, "-v", host+":"+container+":ro")
|
||||
}
|
||||
args = append(args, []string{
|
||||
"-v", workdir + ":" + workdir,
|
||||
"--workdir=" + workdir,
|
||||
"camlistore/git",
|
||||
"git"}...)
|
||||
args = append(args, gitArgs...)
|
||||
cmd = exec.Command("docker", args...)
|
||||
} else {
|
||||
cmd = exec.Command("git", gitArgs...)
|
||||
cmd.Dir = workdir
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
// GitCommit is a datastore entity to track which commits we've
|
||||
// already emailed about.
|
||||
type GitCommit struct {
|
||||
Emailed bool
|
||||
}
|
||||
|
||||
func pollCommits(dir string) {
|
||||
if err := emailOnTimeout("git pull_origin", 5*time.Minute, func() error {
|
||||
cmd := execGit(dir, "pull_origin", nil, "pull", "origin")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error running git pull origin master in %s: %v\n%s", dir, err, out)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Printf("%v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("Ran git pull.")
|
||||
// TODO: see if .git/refs/remotes/origin/master
|
||||
// changed. (quicker than running recentCommits each time)
|
||||
|
||||
hashes, err := recentCommits(dir)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return
|
||||
}
|
||||
if len(hashes) == 0 {
|
||||
return
|
||||
}
|
||||
latestHash.Lock()
|
||||
latestHash.s = hashes[0]
|
||||
latestHash.Unlock()
|
||||
for _, commit := range hashes {
|
||||
if knownCommit[commit] {
|
||||
continue
|
||||
}
|
||||
if dsClient != nil {
|
||||
ctx := context.Background()
|
||||
key := datastore.NameKey("git_commit", commit, nil)
|
||||
var gc GitCommit
|
||||
if err := dsClient.Get(ctx, key, &gc); err == nil && gc.Emailed {
|
||||
log.Printf("Already emailed about commit %v; skipping", commit)
|
||||
knownCommit[commit] = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := emailCommit(dir, commit); err != nil {
|
||||
log.Printf("Error with commit e-mail: %v", err)
|
||||
continue
|
||||
}
|
||||
log.Printf("Emailed commit %s", commit)
|
||||
knownCommit[commit] = true
|
||||
if dsClient != nil {
|
||||
ctx := context.Background()
|
||||
key := datastore.NameKey("git_commit", commit, nil)
|
||||
_, err := dsClient.Put(ctx, key, &GitCommit{Emailed: true})
|
||||
log.Printf("datastore put of git_commit(%v): %v", commit, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func recentCommits(dir string) (hashes []string, err error) {
|
||||
var out []byte
|
||||
if err := emailOnTimeout("git log_origin_master", 2*time.Minute, func() error {
|
||||
cmd := execGit(dir, "log_origin_master", nil, "log", "--since=1 month ago", "--pretty=oneline", "origin/master")
|
||||
out, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error running git log in %s: %v\n%s", dir, err, out)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
v := strings.SplitN(line, " ", 2)
|
||||
if len(v) > 1 {
|
||||
hashes = append(hashes, v[0])
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func mailNowHandler(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-tokenc:
|
||||
log.Printf("/mailnow got a token")
|
||||
default:
|
||||
// Too many requests. Ignore.
|
||||
log.Printf("Ignoring /mailnow request; too soon.")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case fetchc <- true:
|
||||
log.Printf("/mailnow triggered a git fetch")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func latestHashHandler(w http.ResponseWriter, r *http.Request) {
|
||||
latestHash.Lock()
|
||||
defer latestHash.Unlock()
|
||||
fmt.Fprint(w, latestHash.s)
|
||||
}
|
||||
|
|
@ -7,8 +7,6 @@ import (
|
|||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/logging"
|
||||
)
|
||||
|
||||
type logRecord struct {
|
||||
|
|
@ -146,23 +144,3 @@ func (lr *logRecord) WriteHeader(status int) {
|
|||
lr.responseStatus = status
|
||||
lr.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
type gceLogger struct {
|
||||
c *logging.Logger
|
||||
}
|
||||
|
||||
func (lg gceLogger) LogEvent(lr *logRecord) {
|
||||
lg.c.Log(logging.Entry{
|
||||
Timestamp: lr.time,
|
||||
Payload: map[string]interface{}{
|
||||
"ip": lr.ip,
|
||||
"path": lr.rawpath,
|
||||
"method": lr.method,
|
||||
"responseBytes": lr.responseBytes,
|
||||
"status": lr.responseStatus,
|
||||
"userAgent": lr.userAgent,
|
||||
"referer": lr.referer,
|
||||
"proto": lr.proto,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,6 @@ package main // import "perkeep.org/website/pk-web"
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"flag"
|
||||
|
|
@ -39,19 +37,11 @@ import (
|
|||
txttemplate "text/template"
|
||||
"time"
|
||||
|
||||
"perkeep.org/internal/osutil"
|
||||
"perkeep.org/pkg/buildinfo"
|
||||
"perkeep.org/pkg/types/camtypes"
|
||||
|
||||
"cloud.google.com/go/compute/metadata"
|
||||
"cloud.google.com/go/logging"
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/mailgun/mailgun-go"
|
||||
"github.com/russross/blackfriday"
|
||||
"go4.org/writerutil"
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -63,37 +53,21 @@ const (
|
|||
var h1TitlePattern = regexp.MustCompile(`<h1[^>]*>([^<]+)</h1>`)
|
||||
|
||||
var (
|
||||
httpAddr = flag.String("http", defaultAddr, "HTTP address. If using Let's Encrypt, this server needs to be able to answer the http-01 challenge on port 80.")
|
||||
httpsAddr = flag.String("https", "", "HTTPS address")
|
||||
root = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl)")
|
||||
logDir = flag.String("logdir", "", "Directory to write log files to (one per hour), or empty to not log.")
|
||||
logStdout = flag.Bool("logstdout", true, "Whether to log to stdout")
|
||||
tlsCertFile = flag.String("tlscert", "", "TLS cert file")
|
||||
tlsKeyFile = flag.String("tlskey", "", "TLS private key file")
|
||||
alsoRun = flag.String("also_run", "", "[optiona] Path to run as a child process. (Used to run perkeep.org's ./scripts/run-blob-server)")
|
||||
devMode = flag.Bool("dev", false, "in dev mode")
|
||||
flagVersion = flag.Bool("version", false, "show version")
|
||||
|
||||
gceProjectID = flag.String("gce_project_id", "", "GCE project ID; required if not running on GCE and gce_log_name is specified.")
|
||||
gceLogName = flag.String("gce_log_name", "", "GCE Cloud Logging log name; if non-empty, logs go to Cloud Logging instead of Apache-style local disk log files")
|
||||
gceJWTFile = flag.String("gce_jwt_file", "", "If non-empty, a filename to the GCE Service Account's JWT (JSON) config file.")
|
||||
gitContainer = flag.Bool("git_container", false, "Use git from the `camlistore/git` Docker container; if false, the system `git` is used.")
|
||||
|
||||
adminEmail = flag.String("email", "", "Address that Let's Encrypt will notify about problems with issued certificates")
|
||||
)
|
||||
|
||||
const (
|
||||
stagingInstName = "camweb-staging" // name of the GCE instance when testing
|
||||
stagingHostname = "staging.camlistore.net"
|
||||
httpAddr = flag.String("http", defaultAddr, "HTTP address. If using Let's Encrypt, this server needs to be able to answer the http-01 challenge on port 80.")
|
||||
httpsAddr = flag.String("https", "", "HTTPS address")
|
||||
root = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl)")
|
||||
logDir = flag.String("logdir", "", "Directory to write log files to (one per hour), or empty to not log.")
|
||||
logStdout = flag.Bool("logstdout", true, "Whether to log to stdout")
|
||||
tlsCertFile = flag.String("tlscert", "", "TLS cert file")
|
||||
tlsKeyFile = flag.String("tlskey", "", "TLS private key file")
|
||||
alsoRun = flag.String("also_run", "", "[optiona] Path to run as a child process. (Used to run perkeep.org's ./scripts/run-blob-server)")
|
||||
flagVersion = flag.Bool("version", false, "show version")
|
||||
adminEmail = flag.String("email", "", "Address that Let's Encrypt will notify about problems with issued certificates")
|
||||
shortLogFile = flag.String("gitlog-file", "", "If non-empty, the path to the `git log | git shortlog -sen output` to use. If empty, it's run as needed.")
|
||||
)
|
||||
|
||||
var (
|
||||
inProd bool
|
||||
// inStaging is whether this instance is the staging server. This should only be true
|
||||
// if inProd is also true - they are not mutually exclusive; staging is still prod -
|
||||
// because we want to test the same code paths as in production. The code then runs
|
||||
// on another GCE instance, and on the stagingHostname host.
|
||||
inStaging bool
|
||||
|
||||
pageHTML, errorHTML, camliErrorHTML *template.Template
|
||||
packageHTML *txttemplate.Template
|
||||
|
|
@ -462,10 +436,6 @@ func (h *redirectRootHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request)
|
|||
host := strings.ToLower(r.Host)
|
||||
if host == "www.camlistore.org" || host == "camlistore.org" ||
|
||||
host == "www."+prodDomain || (inProd && r.TLS == nil) {
|
||||
if inStaging {
|
||||
http.Redirect(rw, r, "https://"+stagingHostname+r.URL.RequestURI(), http.StatusFound)
|
||||
return
|
||||
}
|
||||
http.Redirect(rw, r, "https://"+prodDomain+r.URL.RequestURI(), http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
|
@ -493,126 +463,28 @@ func runAsChild(res string) {
|
|||
}()
|
||||
}
|
||||
|
||||
func checkInProduction() bool {
|
||||
if !metadata.OnGCE() {
|
||||
return false
|
||||
}
|
||||
proj, _ := metadata.ProjectID()
|
||||
inst, _ := metadata.InstanceName()
|
||||
log.Printf("Running on GCE: %v / %v", proj, inst)
|
||||
prod := proj == "camlistore-website" && inst == "camweb" || inst == stagingInstName
|
||||
inStaging = prod && inst == stagingInstName
|
||||
return prod
|
||||
}
|
||||
|
||||
const (
|
||||
prodSrcDir = "/var/camweb/src/" + prodDomain
|
||||
prodLECacheDir = "/var/le/letsencrypt.cache"
|
||||
)
|
||||
|
||||
func setProdFlags() {
|
||||
inProd = checkInProduction()
|
||||
if !inProd {
|
||||
return
|
||||
}
|
||||
if *devMode {
|
||||
log.Fatal("can't use dev mode in production")
|
||||
}
|
||||
log.Printf("Running in production; configuring prod flags & containers")
|
||||
*httpAddr = ":80"
|
||||
*httpsAddr = ":443"
|
||||
// TODO(mpl): investigate why this proxying does not seem to be working (we end up on https://camlistore.org).
|
||||
buildbotBackend = "https://travis-ci.org/perkeep/perkeep"
|
||||
buildbotHost = "build.perkeep.org"
|
||||
*gceLogName = "camweb-access-log"
|
||||
if inStaging {
|
||||
*gceLogName += "-staging"
|
||||
}
|
||||
*root = filepath.Join(prodSrcDir, "website")
|
||||
*gitContainer = true
|
||||
|
||||
*adminEmail = "mathieu.lonjaret@gmail.com" // for let's encrypt
|
||||
*emailsTo = "camlistore-commits@googlegroups.com"
|
||||
*mailgunCfgFile = "mailgun-config.json"
|
||||
if inStaging {
|
||||
// in staging, keep emailsTo so we get in the loop that does the
|
||||
// git pull and refreshes the content, but no mailgunCfgFile so
|
||||
// we don't actually try to send any e-mail.
|
||||
*mailgunCfgFile = ""
|
||||
}
|
||||
|
||||
os.RemoveAll(prodSrcDir)
|
||||
if err := os.MkdirAll(prodSrcDir, 0755); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("fetching git docker image...")
|
||||
getDockerImage("camlistore/git", "docker-git.tar.gz")
|
||||
getDockerImage("camlistore/demoblobserver", "docker-demoblobserver.tar.gz")
|
||||
|
||||
log.Printf("cloning perkeep git tree...")
|
||||
branch := "master"
|
||||
if inStaging {
|
||||
// We work off the staging branch, so we stay in control of the
|
||||
// website contents, regardless of which commits are landing on the
|
||||
// master branch in the meantime.
|
||||
branch = "staging"
|
||||
}
|
||||
cloneArgs := []string{
|
||||
"run",
|
||||
"--rm",
|
||||
"-v", "/var/camweb:/var/camweb",
|
||||
"camlistore/git",
|
||||
"git", "clone", "-b", branch, "https://github.com/perkeep/perkeep.git", prodSrcDir,
|
||||
}
|
||||
out, err := exec.Command("docker", cloneArgs...).CombinedOutput()
|
||||
if err != nil {
|
||||
log.Fatalf("git clone: %v, %s", err, out)
|
||||
}
|
||||
os.Chdir(*root)
|
||||
log.Printf("Starting.")
|
||||
sendStartingEmail()
|
||||
}
|
||||
|
||||
func randHex(n int) string {
|
||||
buf := make([]byte, n/2+1)
|
||||
rand.Read(buf)
|
||||
return fmt.Sprintf("%x", buf)[:n]
|
||||
}
|
||||
|
||||
func removeContainer(name string) {
|
||||
if err := exec.Command("docker", "kill", name).Run(); err == nil {
|
||||
// It was actually running.
|
||||
log.Printf("Killed old %q container.", name)
|
||||
}
|
||||
if err := exec.Command("docker", "rm", name).Run(); err == nil {
|
||||
// Always try to remove, in case we end up with a stale,
|
||||
// non-running one (which has happened in the past).
|
||||
log.Printf("Removed old %q container.", name)
|
||||
}
|
||||
}
|
||||
|
||||
// runDemoBlobServerContainer runs the demo blobserver as name in a docker
|
||||
// container. It is not run in daemon mode, so it never returns if successful.
|
||||
func runDemoBlobServerContainer(name string) error {
|
||||
removeContainer(name)
|
||||
cmd := exec.Command("docker", "run",
|
||||
"--rm",
|
||||
"--name="+name,
|
||||
"-e", "CAMLI_ROOT="+prodSrcDir+"/website/blobserver-example/root",
|
||||
"-e", "CAMLI_PASSWORD="+randHex(20),
|
||||
"-v", pkSrcDir()+":"+prodSrcDir,
|
||||
"--net=host",
|
||||
"--workdir="+prodSrcDir,
|
||||
"camlistore/demoblobserver",
|
||||
"camlistored",
|
||||
"--openbrowser=false",
|
||||
"--listen=:3179",
|
||||
"--configfile="+prodSrcDir+"/website/blobserver-example/example-blobserver-config.json")
|
||||
stderr := &writerutil.PrefixSuffixSaver{N: 32 << 10}
|
||||
cmd.Stderr = stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to run demo blob server: %v, stderr: %v", err, string(stderr.Bytes()))
|
||||
}
|
||||
// removeContainer(name)
|
||||
// cmd := exec.Command("docker", "run",
|
||||
// "--rm",
|
||||
// "--name="+name,
|
||||
// "-e", "CAMLI_ROOT="+prodSrcDir+"/website/blobserver-example/root",
|
||||
// "-e", "CAMLI_PASSWORD="+randHex(20),
|
||||
// "-v", pkSrcDir()+":"+prodSrcDir,
|
||||
// "--net=host",
|
||||
// "--workdir="+prodSrcDir,
|
||||
// "camlistore/demoblobserver",
|
||||
// "camlistored",
|
||||
// "--openbrowser=false",
|
||||
// "--listen=:3179",
|
||||
// "--configfile="+prodSrcDir+"/website/blobserver-example/example-blobserver-config.json")
|
||||
// stderr := &writerutil.PrefixSuffixSaver{N: 32 << 10}
|
||||
// cmd.Stderr = stderr
|
||||
// if err := cmd.Run(); err != nil {
|
||||
// return fmt.Errorf("failed to run demo blob server: %v, stderr: %v", err, string(stderr.Bytes()))
|
||||
// }
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -635,79 +507,6 @@ func runDemoBlobserverLoop() {
|
|||
}
|
||||
}
|
||||
|
||||
func sendStartingEmail() {
|
||||
if *mailgunCfgFile == "" {
|
||||
return
|
||||
}
|
||||
contentRev, err := exec.Command("docker", "run",
|
||||
"--rm",
|
||||
"-v", "/var/camweb:/var/camweb",
|
||||
"-w", prodSrcDir,
|
||||
"camlistore/git",
|
||||
"/bin/bash", "-c",
|
||||
"git show --pretty=format:'%ad-%h' --abbrev-commit --date=short | head -1").Output()
|
||||
|
||||
cfg, err := mailgunCfgFromGCS()
|
||||
if err != nil {
|
||||
log.Printf("Failed to get mailgun config: %v", err)
|
||||
return
|
||||
}
|
||||
mailGun = mailgun.NewMailgun(cfg.Domain, cfg.APIKey, cfg.PublicAPIKey)
|
||||
contents := `Perkeep website starting with binary ` + buildinfo.Summary() + ` and content at git rev ` + string(contentRev)
|
||||
m := mailGun.NewMessage(
|
||||
"noreply@perkeep.org (Perkeep Website)",
|
||||
"Perkeep camweb restarting",
|
||||
contents,
|
||||
"brad@danga.com",
|
||||
"mathieu.lonjaret@gmail.com",
|
||||
)
|
||||
if _, _, err := mailGun.Send(m); err != nil {
|
||||
log.Printf("Failed to send camweb startup e-mail: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func getDockerImage(tag, file string) {
|
||||
have, err := exec.Command("docker", "inspect", tag).Output()
|
||||
if err == nil && len(have) > 0 {
|
||||
return // we have it.
|
||||
}
|
||||
url := "https://storage.googleapis.com/" + prodBucket + "/" + file
|
||||
err = exec.Command("/bin/bash", "-c", "curl --silent "+url+" | docker load").Run()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// httpClient returns an http Client suitable for Google Cloud Storage or Google Cloud
|
||||
// Logging calls with the projID project ID.
|
||||
func httpClient(projID string) *http.Client {
|
||||
if *gceJWTFile == "" {
|
||||
log.Fatal("Cannot initialize an authorized http Client without --gce_jwt_file")
|
||||
}
|
||||
jsonSlurp, err := os.ReadFile(*gceJWTFile)
|
||||
if err != nil {
|
||||
log.Fatalf("Error reading --gce_jwt_file value: %v", err)
|
||||
}
|
||||
jwtConf, err := google.JWTConfigFromJSON(jsonSlurp, logging.WriteScope)
|
||||
if err != nil {
|
||||
log.Fatalf("Error reading --gce_jwt_file value: %v", err)
|
||||
}
|
||||
return jwtConf.Client(context.Background())
|
||||
}
|
||||
|
||||
// projectID returns the GCE project ID used for running this camweb on GCE
|
||||
// and/or for logging on Google Cloud Logging, if any.
|
||||
func projectID() string {
|
||||
if *gceProjectID != "" {
|
||||
return *gceProjectID
|
||||
}
|
||||
projID, err := metadata.ProjectID()
|
||||
if projID == "" || err != nil {
|
||||
log.Fatalf("GCE project ID needed but --gce_project_id not specified (and not running on GCE); metadata error: %v", err)
|
||||
}
|
||||
return projID
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if *flagVersion {
|
||||
|
|
@ -715,7 +514,6 @@ func main() {
|
|||
buildinfo.Summary(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
|
||||
return
|
||||
}
|
||||
setProdFlags()
|
||||
|
||||
if *root == "" {
|
||||
var err error
|
||||
|
|
@ -778,30 +576,6 @@ func main() {
|
|||
if *logDir != "" || *logStdout {
|
||||
handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout))
|
||||
}
|
||||
if *gceLogName != "" {
|
||||
projID := projectID()
|
||||
var hc *http.Client
|
||||
if !metadata.OnGCE() {
|
||||
hc = httpClient(projID)
|
||||
}
|
||||
ctx := context.Background()
|
||||
var logc *logging.Client
|
||||
if metadata.OnGCE() {
|
||||
logc, err = logging.NewClient(ctx, projID)
|
||||
} else {
|
||||
logc, err = logging.NewClient(ctx, projID, option.WithHTTPClient(hc))
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := logc.Ping(ctx); err != nil {
|
||||
log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
|
||||
}
|
||||
handler = NewLoggingHandler(handler, gceLogger{logc.Logger(*gceLogName)})
|
||||
}
|
||||
|
||||
emailErr := make(chan error)
|
||||
startEmailCommitLoop(emailErr)
|
||||
|
||||
if *alsoRun != "" {
|
||||
runAsChild(*alsoRun)
|
||||
|
|
@ -822,8 +596,6 @@ func main() {
|
|||
}()
|
||||
|
||||
select {
|
||||
case err := <-emailErr:
|
||||
log.Fatalf("Error sending emails: %v", err)
|
||||
case err := <-httpsErr:
|
||||
log.Fatalf("Error serving HTTPS: %v", err)
|
||||
}
|
||||
|
|
@ -874,13 +646,9 @@ func serve(httpServer *http.Server, onHTTPError func(error)) error {
|
|||
}
|
||||
hostPolicy = autocert.HostWhitelist(host)
|
||||
} else {
|
||||
if inStaging {
|
||||
hostPolicy = autocert.HostWhitelist(stagingHostname)
|
||||
} else {
|
||||
hostPolicy = autocert.HostWhitelist(prodDomain, "www."+prodDomain,
|
||||
"www.camlistore.org", "camlistore.org")
|
||||
}
|
||||
cacheDir = autocert.DirCache(prodLECacheDir)
|
||||
hostPolicy = autocert.HostWhitelist(prodDomain, "www."+prodDomain,
|
||||
"www.camlistore.org", "camlistore.org")
|
||||
cacheDir = autocert.DirCache("/var/le/letsencrypt.cache")
|
||||
}
|
||||
m := autocert.Manager{
|
||||
Prompt: autocert.AcceptTOS,
|
||||
|
|
@ -917,23 +685,6 @@ func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
|
|||
return tc, nil
|
||||
}
|
||||
|
||||
func fromGCS(filename string) ([]byte, error) {
|
||||
ctx := context.Background()
|
||||
sc, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slurp := func(key string) ([]byte, error) {
|
||||
rc, err := sc.Bucket(prodBucket).Object(key).NewReader(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error fetching GCS object %q in bucket %q: %v", key, prodBucket, err)
|
||||
}
|
||||
defer rc.Close()
|
||||
return io.ReadAll(rc)
|
||||
}
|
||||
return slurp(filename)
|
||||
}
|
||||
|
||||
var issueNum = regexp.MustCompile(`^/(?:issue|bug)s?(/\d*)?$`)
|
||||
|
||||
// issueRedirect returns whether the request should be redirected to the
|
||||
|
|
@ -1047,14 +798,3 @@ func errHandler(w http.ResponseWriter, r *http.Request) {
|
|||
content: contents,
|
||||
})
|
||||
}
|
||||
|
||||
func pkSrcDir() string {
|
||||
if inProd {
|
||||
return prodSrcDir
|
||||
}
|
||||
dir, err := osutil.GoPackagePath(prodDomain)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to find the root of the %s source code via osutil.GoPackagePath: %v", prodDomain, err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue