2018-02-06 01:50:26 +00:00
|
|
|
# Gopkg for Perkeepa, requires dep v0.4.0 or later
|
2018-01-07 20:49:03 +00:00
|
|
|
#
|
|
|
|
|
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
|
|
|
|
|
# for detailed Gopkg.toml documentation.
|
|
|
|
|
|
|
|
|
|
# The required section includes packages that are indirectly used and
|
|
|
|
|
# tools used during build like gopherjs. Note that you have to
|
|
|
|
|
# reference a package with go code included here, which may not be the
|
|
|
|
|
# root level of the repo.
|
|
|
|
|
|
|
|
|
|
required = [
|
|
|
|
|
"github.com/fsnotify/fsnotify",
|
|
|
|
|
"github.com/spf13/cobra", # for gopherjs
|
|
|
|
|
"github.com/spf13/pflag", # for gopherjs
|
|
|
|
|
"github.com/cznic/internal/buffer",
|
|
|
|
|
"github.com/shurcooL/sanitized_anchor_name",
|
|
|
|
|
"github.com/cznic/internal/file",
|
|
|
|
|
"github.com/fsnotify/fsnotify",
|
|
|
|
|
"github.com/golang/snappy",
|
|
|
|
|
"github.com/googleapis/gax-go",
|
|
|
|
|
"github.com/gopherjs/gopherjs",
|
|
|
|
|
"github.com/gopherjs/gopherjs/nosync",
|
|
|
|
|
"github.com/kardianos/osext", # for gopherjs
|
|
|
|
|
"github.com/kisielk/gotool",
|
|
|
|
|
"google.golang.org/grpc", # fork by bradfitz
|
|
|
|
|
"golang.org/x/text",
|
|
|
|
|
"github.com/neelance/sourcemap",
|
|
|
|
|
"golang.org/x/sys/unix",
|
|
|
|
|
"golang.org/x/tools/go/types/typeutil", # for gopherjs
|
|
|
|
|
"golang.org/x/tools/go/gcimporter15", # for gopherjs
|
|
|
|
|
"google.golang.org/genproto/googleapis/datastore/v1",
|
2018-04-11 15:12:00 +00:00
|
|
|
"myitcv.io/react/cmd/reactGen",
|
|
|
|
|
"myitcv.io/gogenerate"
|
2018-01-07 20:49:03 +00:00
|
|
|
]
|
|
|
|
|
|
2018-01-17 01:30:20 +00:00
|
|
|
|
|
|
|
|
# Try to eliminate as much cruft as possible to keep the vendor
|
|
|
|
|
# directory small. Each entry is a directory that you want to eliminate.
|
|
|
|
|
# To omit individual files add to the top-level .gitignore
|
|
|
|
|
|
|
|
|
|
ignored = [
|
|
|
|
|
"cloud.google.com/go/datastore/testdata",
|
|
|
|
|
"github.com/cznic/kv/_testdata",
|
|
|
|
|
"github.com/cznic/lldb/testdata",
|
|
|
|
|
"github.com/cznic/zappy/testdata",
|
|
|
|
|
"github.com/garyburd/go-oauth/examples",
|
|
|
|
|
"github.com/golang/protobuf/proto/testdata",
|
|
|
|
|
"github.com/gopherjs/gopherjs/tests",
|
|
|
|
|
"github.com/gopherjs/gopherjs/tests/*",
|
|
|
|
|
"github.com/gorilla/websocket/examples",
|
|
|
|
|
"github.com/rwcarlsen/goexif/exif/corrupt",
|
|
|
|
|
"github.com/rwcarlsen/goexif/exif/samples",
|
|
|
|
|
"github.com/russross/blackfriday/testdata",
|
|
|
|
|
"github.com/shurcooL/go/importgraphutil",
|
|
|
|
|
"github.com/syndtr/goleveldb/leveldb/testutil",
|
|
|
|
|
"github.com/syndtr/goleveldb/manualtest",
|
|
|
|
|
"github.com/tgulacsi/picago/testdata",
|
|
|
|
|
"github.com/tgulacsi/picago/testdata",
|
|
|
|
|
"go4.org/jsonconfig/testdata",
|
|
|
|
|
"golang.org/x/image/testdata",
|
|
|
|
|
"golang.org/x/image/font/testdata",
|
|
|
|
|
"golang.org/x/net/html/charset/testdata",
|
|
|
|
|
"golang.org/x/net/html/testdata",
|
|
|
|
|
"golang.org/x/net/http2/testdata",
|
|
|
|
|
"golang.org/x/oauth2/google/testdata",
|
|
|
|
|
"golang.org/x/text/encoding/testdata",
|
|
|
|
|
"golang.org/x/text/language/testdata",
|
|
|
|
|
"golang.org/x/tools/go/gcimporter15/testdata",
|
|
|
|
|
"golang.org/x/tools/refactor/importgraph",
|
|
|
|
|
"google.golang.org/grpc/benchmark",
|
|
|
|
|
"google.golang.org/grpc/examples",
|
|
|
|
|
"google.golang.org/grpc/interop/client/testdata",
|
|
|
|
|
"google.golang.org/grpc/interop/server/testdata",
|
|
|
|
|
"google.golang.org/grpc/reflection",
|
|
|
|
|
"google.golang.org/grpc/stress",
|
|
|
|
|
"google.golang.org/grpc/test",
|
|
|
|
|
"google.golang.org/grpc/testdata",
|
|
|
|
|
"google.golang.org/grpc/transport/testdata",
|
|
|
|
|
"myitcv.io/gogenerate/_testFiles",
|
|
|
|
|
"myitcv.io/react/examples",
|
|
|
|
|
"myitcv.io/react/_scripts",
|
|
|
|
|
"myitcv.io/react/_talks",
|
|
|
|
|
"myitcv.io/react/_vendor"
|
|
|
|
|
]
|
|
|
|
|
|
2018-02-06 01:50:26 +00:00
|
|
|
[prune]
|
|
|
|
|
unused-packages = true
|
|
|
|
|
go-tests = true
|
|
|
|
|
non-go = true
|
|
|
|
|
|
2018-02-12 23:25:35 +00:00
|
|
|
# To avoid deletion of jsbuiltin.inc.js
|
|
|
|
|
[[prune.project]]
|
|
|
|
|
name = "github.com/gopherjs/jsbuiltin"
|
|
|
|
|
non-go = false
|
|
|
|
|
|
2018-01-07 20:49:03 +00:00
|
|
|
# Constraints / Overrides follow
|
|
|
|
|
# Semantic versions (like v1.0.0) are preferred
|
|
|
|
|
# Please keep alphabetized
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "bazil.org/fuse"
|
|
|
|
|
revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "cloud.google.com/go"
|
|
|
|
|
revision = "b70ccc799b9d019708c3eb9395acef6e3f6b7bc8"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# github.com
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/FiloSottile/b2"
|
|
|
|
|
revision = "b197f7a2c317098d18afe80f0d6789782d52a090"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/bradfitz/latlong"
|
|
|
|
|
revision = "b745505085619d2818bc4b32b5a9fcc539e551fd"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/cznic/internal"
|
|
|
|
|
revision = "4747030f7cf2f4c0a01512b00cd68734b167ac3b"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/cznic/kv"
|
|
|
|
|
revision = "892ccf731fb7aa5e9aa300eb24456d1519afcfc7"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/fsnotify/fsnotify"
|
|
|
|
|
revision = "629574ca2a5df945712d3079857300b5e4da0236"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/garyburd/go-oauth"
|
|
|
|
|
revision = "7d749055310d0d9fed966cd500664dbca484967e"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/go-sql-driver/mysql"
|
2018-01-11 22:34:16 +00:00
|
|
|
version = "1.3.0"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/golang/snappy"
|
|
|
|
|
revision = "553a641470496b2327abcac10b36396bd98e45c9"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/googleapis/gax-go"
|
|
|
|
|
revision = "da06d194a00e19ce00d9011a13931c3f6f6887c7"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/gopherjs/gopherjs"
|
2018-02-23 19:57:12 +00:00
|
|
|
# temporarily switching to embed-core-pkgs branch, until Dmitri merges it to master.
|
|
|
|
|
revision = "2ad4a5b9a30e5f1f31c643c2a2ac27543a5ddb58"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/gopherjs/jquery"
|
|
|
|
|
revision = "fbbfc4bbe29a29cb05788b66be44e0ac7f43cac7"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/kardianos/osext"
|
|
|
|
|
revision = "29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/gorilla/websocket"
|
2018-01-11 17:12:32 +00:00
|
|
|
version = "1.2.0"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/hjfreyer/taglib-go"
|
|
|
|
|
revision = "0ef8bba9c41b66c12f60ce9833786838d2c2d3d8"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/kisielk/gotool"
|
|
|
|
|
revision = "0de1eaf82fa3f583ce21fde859f1e7e0c5e9b220"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/lib/pq"
|
|
|
|
|
revision = "9afcd9aa793101bd0536da34e74ae0123345bab1"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/mailgun/mailgun-go"
|
|
|
|
|
revision = "17e8bd11e87cb660ba5da8d635bbeae44b9443ac"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/mattn/go-sqlite3"
|
2018-01-21 18:14:40 +00:00
|
|
|
version = "v1.6.0"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/miekg/dns"
|
|
|
|
|
revision = "3f1f7c8ec9ead89493df11f2c3d8bec353a2c2c0"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/neelance/sourcemap"
|
|
|
|
|
revision = "8c68805598ab8d5637b1a72b5f7d381ea0f39c31"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/nf/cr2"
|
|
|
|
|
revision = "05d46fef4f2f903240751e25871cd318d001deba"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/plaid/plaid-go"
|
|
|
|
|
revision = "02b6af68061bf89a293eaf15dc6c955ce02dd22b"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/russross/blackfriday"
|
2018-01-13 02:07:42 +00:00
|
|
|
version = "v2.0.0"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/rwcarlsen/goexif"
|
2018-04-10 21:58:43 +00:00
|
|
|
branch = "go1"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/shurcooL/sanitized_anchor_name"
|
|
|
|
|
revision = "11a20b799bf22a02808c862eb6ca09f7fb38f84a"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/spf13/cobra"
|
|
|
|
|
revision = "c678ff029ee250b65714e518f4f5c5cb934955de"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/spf13/pflag"
|
|
|
|
|
revision = "7f60f83a2c81bc3c3c0d5297f61ddfa68da9d3b7"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/syndtr/goleveldb"
|
cmd/camput: compact LevelDB on HaveCache setup
This CL is about levelDB as the HaveCache for camput, and there are
several aspects to it. To describe it, I'll take the particular example
where you want to add many permanodes (~33k) to a given set, with
camput. Something like:
for _, blob := range blobs {
do("camput attr -add sha1-foobar camliMember " + blob)
}
In a "normal" levelDB use case, everytime the number of level-0 .ldb
files goes over 4 (by default), a background compaction task is
started to transform these SST into level-1 ones, and remove the level-0
ones.
However, since our particular camput call is very short lived
(especially on a local Perkeep), not only might there be not enough time
for the compaction to be triggered, but even if it is, when the DB is
flushed (on a Close call), any ongoing compactions are cancelled. This
makes level-0 compactions very unlikely to happen on short-lived camput
calls. As a result, the number of level-0 files keeps growing until
levelDB fails while trying to open them all, because it hits the current
process ulimit.
Now, in this CL, what we propose is to systematically force a compaction
as soon as the HaveCache is opened. It is not scheduled concurrently, so
we are sure that the compaction happens before the DB actually gets used
by camput. This seems to make sure that the number of level-0 tables
never grows too much. With this change, I was able to run the above
example on 33K blobs without hitting the ulimit error.
However, it should be noted that potential problems might remain. The
compaction for levels above 0 is triggered based only on the total size
of the level (e.g. at 100MB by default for level-1), and not on the
number of files. Since we're creating many tiny tables (basically 1
entry per table), the number of files grows very fast while the total
size does not, and the compaction does not get triggered, even if forced
with CompactRange. This does not seem to be a problem for our use case,
as levelDB does not seem to need to open many of the level-1 files at
the same time, so we're not hitting the ulimit problem because of that.
If needed, there's at least one way this problem (if it is one?) could
be fixed: make the compaction trigger on other conditions, such as
number of files per level. I've experimented with it (forcing the
level-1 compaction to trigger at the 100 files limit), and it seems to
be working. But I had to do change the goleveldb code itself, and I
don't think levelDB implementations are supposed to do that.
For information, at the end of the run on the 33K blobs:
$ du -sch *.ldb
...
83M total
$ ll | wc -l
20988
And indeed, when asking for leveldb.stats on the table:
Level | Tables | Size(MB) |
-------+------------+---------------+
0 | 1 | 0.00015 |
1 | 20981 | 3.47307 |
Also, update github.com/syndtr/goleveldb to
34011bf325bce385408353a30b101fe5e923eb6e
And remove github.com/syndtr/gosnappy as goleveldb does not use it
anymore.
Also apply this change to StatCache.
Fixes #1008
Change-Id: If9f790a003e67f3c075881470e52e5f2174afa73
2018-01-12 19:39:44 +00:00
|
|
|
revision = "34011bf325bce385408353a30b101fe5e923eb6e"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "github.com/tgulacsi/picago"
|
|
|
|
|
revision = "9e1ac2306c701ca7477a169b2b49902b7b4c58bf"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "go4.org"
|
2018-04-12 22:07:32 +00:00
|
|
|
branch = "master"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
# golang/x
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "golang.org/x/crypto"
|
2018-01-16 00:12:08 +00:00
|
|
|
revision = "13931e22f9e72ea58bb73048bc752b48c6d4d4ac"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "golang.org/x/image"
|
|
|
|
|
#branch = "master"
|
|
|
|
|
#revision = "b3d8467d91f6ab3c9240e1c1e98309baf9ba6343"
|
|
|
|
|
revision = "12117c17ca67ffa1ce22e9409f3b0b0a93ac08c7"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "golang.org/x/net"
|
|
|
|
|
revision = "d866cfc389cec985d6fda2859936a575a55a3ab6"
|
|
|
|
|
|
|
|
|
|
# picago uses branch="master", override for now to match our explicit version
|
|
|
|
|
[[override]]
|
|
|
|
|
revision = "197281d4e0ecd78c33865daf9c6e51626feefcb2"
|
|
|
|
|
name = "golang.org/x/oauth2"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "golang.org/x/sync"
|
|
|
|
|
revision = "fd80eb99c"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "golang.org/x/sys"
|
2018-01-16 00:12:08 +00:00
|
|
|
revision = "fff93fa7cd278d84afc205751523809c464168ab"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "golang.org/x/text"
|
|
|
|
|
revision = "88f656faf3f37f690df1a32515b479415e1a6769"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "golang.org/x/tools"
|
|
|
|
|
revision ="e531a2a1c15f94033f6fa87666caeb19a688175f"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
revision = "a4bde12657593d5e90d0533a3e4fd95e635124cb"
|
|
|
|
|
name = "golang.org/x/time"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# google.golang.org
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
revision = "48e49d1645e228d1c50c3d54fb476b2224477303"
|
|
|
|
|
name = "google.golang.org/api"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "google.golang.org/genproto"
|
|
|
|
|
revision = "08f135d1a31b6ba454287638a3ce23a55adace6f"
|
|
|
|
|
|
|
|
|
|
# Brad's fork of grpc
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "google.golang.org/grpc"
|
|
|
|
|
source = "https://github.com/bradfitz/grpc-go.git"
|
|
|
|
|
revision = "188a132adcfba339f1f2d5da52498451341f9ee8"
|
|
|
|
|
|
|
|
|
|
|
2018-01-13 02:07:42 +00:00
|
|
|
# gopkg.in
|
|
|
|
|
|
2018-01-07 20:49:03 +00:00
|
|
|
[[constraint]]
|
|
|
|
|
name = "gopkg.in/mgo.v2"
|
|
|
|
|
branch = "v2"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "honnef.co/go/js/dom"
|
|
|
|
|
revision = "24aa052bc5c63cfb9383bf59493ee48621ca788c"
|
|
|
|
|
|
2018-04-11 15:12:00 +00:00
|
|
|
[[constraint]]
|
|
|
|
|
name = "myitcv.io/gogenerate"
|
|
|
|
|
revision ="bd69a94c96953d20e106734856b69d71c8fa122b"
|
2018-01-07 20:49:03 +00:00
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "myitcv.io/react"
|
|
|
|
|
revision = "bca7c66b77ed8a5b86fb77cff70914c4a7cc3ce5"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "rsc.io/pdf"
|
|
|
|
|
revision = "1d34785eb915fd1ea1c437ad41621c9066642030"
|
|
|
|
|
|
|
|
|
|
[[constraint]]
|
|
|
|
|
name = "rsc.io/qr"
|
|
|
|
|
revision = "48b2ede4844e13f1a2b7ce4d2529c9af7e359fc5"
|