From dd996d05f51fe1de31c631523d255df5abab6a42 Mon Sep 17 00:00:00 2001 From: Alexandre Viau Date: Thu, 29 Jul 2021 14:22:27 -0400 Subject: [PATCH] doc: document all environement variables (#1383) envvardoc: repair the script so that it does not fail on very long lines. envvardoc: look for env vars with the `PERKEEP` prefix too. envvardoc: adapt the `docVar` regex to the markdown format. envvardoc: also look for vars in internal/. CI: run envvardoc so that we notice when we add new undocumented environement variables. doc: List all remaining undocumented env variables. This "shitlist" gives us a green build in the CI and allows us to ensure all new environment variables are documented in the future. doc: Link to the environment variables page. --- .github/workflows/tests-lint.yaml | 1 + dev/envvardoc/envvardoc.go | 19 +++-- doc/README.md | 1 + doc/environment-vars.md | 114 +++++++++++++++++++++--------- 4 files changed, 98 insertions(+), 37 deletions(-) diff --git a/.github/workflows/tests-lint.yaml b/.github/workflows/tests-lint.yaml index 001eab0a3..e17f04c9e 100644 --- a/.github/workflows/tests-lint.yaml +++ b/.github/workflows/tests-lint.yaml @@ -30,3 +30,4 @@ jobs: go fmt ./... test -z "$(git status --porcelain)" go vet ./... + go run dev/envvardoc/envvardoc.go diff --git a/dev/envvardoc/envvardoc.go b/dev/envvardoc/envvardoc.go index 9ee7b0647..823a0f96d 100644 --- a/dev/envvardoc/envvardoc.go +++ b/dev/envvardoc/envvardoc.go @@ -17,15 +17,17 @@ import ( ) var ( - srcDirs = flag.String("srcDirs", "cmd,dev,pkg,server", + srcDirs = flag.String("srcDirs", "cmd,dev,pkg,server,internal", "comma separated source directories") - doc = flag.String("doc", "doc/environment-vars.txt", + doc = flag.String("doc", "doc/environment-vars.md", "file containing environment variable documentation") all = flag.Bool("all", false, "show all environment vars found") - prefixes = flag.String("prefixes", "CAM,DEV,AWS", + prefixes = flag.String("prefixes", "PERKEEP,CAM,DEV,AWS", "comma-separated list of env var prefixes we care about. Empty implies all") - docVar = regexp.MustCompile(`^(\w+) \(.+?\):$`) + // ^`(\w+)`\s+\(\w+\)\s*$ + docVar = regexp.MustCompile(`^` + "`" + `(\w+)` + "`" + `\s+\(\w+\)\s*$`) + literalEnvVar = regexp.MustCompile(`os.Getenv\("(\w+)"\)`) variableEnvVar = regexp.MustCompile(`os.Getenv\((\w+)\)`) ) @@ -82,7 +84,14 @@ func (ec *envCollector) findEnvVars(path string, r io.Reader) error { } line++ } - return scanner.Err() + + err := scanner.Err() + if err == bufio.ErrTooLong { + // Happens only for unreasonably long lines. + // In our case the webui's embeded stuff. + return nil + } + return err } func (ec *envCollector) findDocVars(r io.Reader) error { diff --git a/doc/README.md b/doc/README.md index d18ffc48c..222c1396b 100644 --- a/doc/README.md +++ b/doc/README.md @@ -48,6 +48,7 @@ on the [mailing list](https://groups.google.com/group/camlistore). * [Protocol](/doc/protocol/): HTTP APIs (discovery, blob storage, JSON signing, ...) * [JSON Signing](/doc/json-signing/) * [Sharing](/doc/sharing.md) +* [Environement Variables](/doc/environment-vars.md) ## Presentations {#presentations} diff --git a/doc/environment-vars.md b/doc/environment-vars.md index df6661342..98f4b94a1 100644 --- a/doc/environment-vars.md +++ b/doc/environment-vars.md @@ -10,7 +10,11 @@ valid. [strconv.ParseBool]: https://golang.org/pkg/strconv/#ParseBool [strconv.Atoi]: https://golang.org/pkg/strconv/#Atoi -`AWS_ACCESS_KEY_ID` (string) and `AWS_ACCESS_KEY_SECRET` (string) +## General + +`AWS_ACCESS_KEY_ID` (string) +and +`AWS_ACCESS_KEY_SECRET` (string) : See http://docs.aws.amazon.com/fws/1.1/GettingStartedGuide/index.html?AWSCredentials.html. Used in s3 tests. If not set some tests are skip. If set, queries will be sent to Amazon's S3 service. @@ -29,6 +33,8 @@ valid. `CAMLI_BASEURL` (string) : URL set in devcam to act as a baseURL in the devcam launched perkeepd. +`PERKEEP_CACHE_DIR` (string) +or `CAMLI_CACHE_DIR` (string) : Path used by [pkg/osutil](/pkg/osutil) to override operating system specific cache directory. @@ -42,20 +48,6 @@ valid. Set in devcam to pass database configuration for the indexer to the devcam launched perkeepd. -`CAMLI_DEBUG` (bool) -: Used by perkeepd and pk put to enable additional commandline options. - Used in pkg/schema to enable additional logging. - -`CAMLI_DEBUG_CONFIG` (bool) -: Causes pkg/serverconfig to dump low-level configuration derived from - high-level configuation on load. - -`CAMLI_DEBUG_X` (string) -: String containing magic substring(s) to enable debuggging in code. - -`CAMLI_DEBUG_UPLOADS` (bool) -: Used by [pkg/client](/pkg/client) to enable additional logging. - `CAMLI_DEFAULT_SERVER` (string) : The server alias to use by default. The string is the server's alias key in the client-config.json "servers" object. If set, the `CAMLI_DEFAULT_SERVER` @@ -75,10 +67,6 @@ valid. : If true, importers are disabled (at least automatic background importing, e.g. at start-up). Mostly for debugging. -`CAMLI_FAST_DEV` (bool) -: Used by dev/demo.sh for giving presentations with devcam server/put/etc - for faster pre-built builds, without calling make.go. - `CAMLI_FORCE_OSARCH` (bool) Used by make.go to force building an unrecommended OS/ARCH pair. @@ -125,12 +113,9 @@ valid. CAMLI_GOPHERJS_GOROOT should be set to a Go 1.10 root so that gopherjs can be built with Go 1.10. Otherwise it defaults to $HOME/go1.10. -`CAMLI_HELLO_ENABLED (bool) +`CAMLI_HELLO_ENABLED` (bool) : Whether to start the hello world app as well. Variable used only by devcam server. -`CAMLI_HTTP_DEBUG` (bool) -: Enable per-request logging in [pkg/webserver](/pkg/webserver). - `CAMLI_HTTP_EXPVAR` (bool) : Enable json export of expvars at /debug/vars @@ -191,11 +176,6 @@ files to be ignored by [pkg/client](/pkg/client) when uploading. : Path to the GPG secret keyring, which is otherwise set by identitySecretRing in the server config, and secretRing in the client config. -`CAMLI_SHA1_ENABLED (bool) -: Whether to enable the use of legacy sha1 blobs. Only used for development, for - creating new blobs with the legacy SHA-1 hash, instead of with the current one. - It does not affect the ability of Perkeep to read SHA-1 blobs. - `CAMLI_DISABLE_CLIENT_CONFIG_FILE` (bool) : If set, the [pkg/client](/pkg/client) code will never use the on-disk config file. @@ -210,10 +190,6 @@ files to be ignored by [pkg/client](/pkg/client) when uploading. `CAMPUT_ANDROID_OUTPUT` (bool) : Enable pkg/client status messages to print to stdout. Used in android client. -`CAMLI_DEBUG_IMAGES` (bool) -: Enable extra debugging in [pkg/images](/pkg/images) when decoding images. - Used by indexers. - `CAMLI_DISABLE_DJPEG` (bool) : Disable use of djpeg(1) to down-sample JPEG images by a factor of 2, 4 or 8. Only has an effect when djpeg is found in the PATH. @@ -236,9 +212,83 @@ files to be ignored by [pkg/client](/pkg/client) when uploading. : Number from 0-100 of what percentage of the time to fail receiving blobs for the S3 handler. +## Development + +`CAMLI_DEBUG` (bool) +: Used by perkeepd and pk put to enable additional commandline options. + Used in pkg/schema to enable additional logging. + +`CAMLI_HTTP_DEBUG` (bool) +: Enable per-request logging in [pkg/webserver](/pkg/webserver). + +`CAMLI_DEBUG_CONFIG` (bool) +: Causes pkg/serverconfig to dump low-level configuration derived from + high-level configuation on load. + +`CAMLI_DEBUG_X` (string) +: String containing magic substring(s) to enable debuggging in code. + +`CAMLI_DEBUG_UPLOADS` (bool) +: Used by [pkg/client](/pkg/client) to enable additional logging. + +`CAMLI_DEBUG_IMAGES` (bool) +: Enable extra debugging in [pkg/images](/pkg/images) when decoding images. + Used by indexers. + +`CAMLI_SHA1_ENABLED` (bool) +: Whether to enable the use of legacy sha1 blobs. Only used for development, for + creating new blobs with the legacy SHA-1 hash, instead of with the current one. + It does not affect the ability of Perkeep to read SHA-1 blobs. + +`CAMLI_FAST_DEV` (bool) +: Used by dev/demo.sh for giving presentations with devcam server/put/etc + for faster pre-built builds, without calling make.go. + `DEV_THROTTLE_KBPS` (integer) and `DEV_THROTTLE_LATENCY_MS` (integer) : Rate limit and/or inject latency in pkg/webserver responses. A value of 0 disables traffic-shaping. `RUN_BROKEN_TESTS` (bool) : Run known-broken tests. + +## Undocumented + +These variables are yet to be documented: + +`CAMLI_API_HOST` (todo) + +`CAMLI_APP_LISTEN` (todo) + +`CAMLI_DEBUG_QUERY_SPEED` (todo) + +`CAMLI_DEV_MAP_CLUSTERING` (todo) + +`CAMLI_FAKE_STATUS_ERROR` (todo) + +`CAMLI_GPHOTOS_FULL_IMPORT` (todo) + +`CAMLI_MORE_FLAGS` (todo) + +`CAMLI_MORE_FLAGS` (todo) + +`CAMLI_PICASA_FULL_IMPORT` (todo) + +`CAMLI_QUIET` (todo) + +`CAMLI_REINDEX_START` (todo) + +`CAMLI_SERVER` (todo) + +`CAMLI_SET_BASE_URL_AND_SEND_ADDR_TO` (todo) + +`CAMLI_SYNC_VALIDATE` (todo) + +`CAMLI_TWITTER_FULL_IMPORT` (todo) + +`CAMLI_TWITTER_SKIP_API_IMPORT` (todo) + +`DEV_THROTTLE_KBPS` (todo) + +`DEV_THROTTLE_LATENCY_MS` (todo) + +`PERKEEP_MASTODON_FULL_IMPORT` (todo)