From 59534f0ad66e39b1165eb5648e443d4948c6f65d Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 12 Sep 2013 16:04:10 +0100 Subject: [PATCH] Move pkg/gate to pkg/syncutil, to house more stuff. Change-Id: I737458641a9de03da55ed627bfac10f3377a13aa --- cmd/camput/camput.go | 4 ++-- cmd/camput/uploader.go | 4 ++-- pkg/blobserver/localdisk/stat.go | 4 ++-- pkg/blobserver/s3/stat.go | 4 ++-- pkg/schema/filewriter.go | 4 ++-- pkg/{gate => syncutil}/gate.go | 9 ++++----- 6 files changed, 14 insertions(+), 15 deletions(-) rename pkg/{gate => syncutil}/gate.go (81%) diff --git a/cmd/camput/camput.go b/cmd/camput/camput.go index e82381398..61b09f425 100644 --- a/cmd/camput/camput.go +++ b/cmd/camput/camput.go @@ -28,8 +28,8 @@ import ( "camlistore.org/pkg/client" "camlistore.org/pkg/cmdmain" - "camlistore.org/pkg/gate" "camlistore.org/pkg/httputil" + "camlistore.org/pkg/syncutil" ) const buffered = 16 // arbitrary @@ -128,7 +128,7 @@ func newUploader() *Uploader { Client: cc, transport: httpStats, pwd: pwd, - fdGate: gate.New(100), // gate things that waste fds, assuming a low system limit + fdGate: syncutil.NewGate(100), // gate things that waste fds, assuming a low system limit } } diff --git a/cmd/camput/uploader.go b/cmd/camput/uploader.go index 8b23fdfba..0ccbaaa09 100644 --- a/cmd/camput/uploader.go +++ b/cmd/camput/uploader.go @@ -22,15 +22,15 @@ import ( "camlistore.org/pkg/blobserver" "camlistore.org/pkg/client" - "camlistore.org/pkg/gate" "camlistore.org/pkg/httputil" + "camlistore.org/pkg/syncutil" ) type Uploader struct { *client.Client // fdGate guards gates the creation of file descriptors. - fdGate *gate.Gate + fdGate *syncutil.Gate fileOpts *fileOptions // per-file options; may be nil diff --git a/pkg/blobserver/localdisk/stat.go b/pkg/blobserver/localdisk/stat.go index 24c069956..690819273 100644 --- a/pkg/blobserver/localdisk/stat.go +++ b/pkg/blobserver/localdisk/stat.go @@ -20,7 +20,7 @@ import ( "os" "camlistore.org/pkg/blob" - "camlistore.org/pkg/gate" + "camlistore.org/pkg/syncutil" ) const maxParallelStats = 20 @@ -48,7 +48,7 @@ func (ds *DiskStorage) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) er errc := make(chan error, len(blobs)) - gt := gate.New(maxParallelStats) + gt := syncutil.NewGate(maxParallelStats) for _, ref := range blobs { gt.Start() go func(ref blob.Ref) { diff --git a/pkg/blobserver/s3/stat.go b/pkg/blobserver/s3/stat.go index 3ba46dce2..3510665ea 100644 --- a/pkg/blobserver/s3/stat.go +++ b/pkg/blobserver/s3/stat.go @@ -21,10 +21,10 @@ import ( "os" "camlistore.org/pkg/blob" - "camlistore.org/pkg/gate" + "camlistore.org/pkg/syncutil" ) -var statGate = gate.New(20) // arbitrary +var statGate = syncutil.NewGate(20) // arbitrary func (sto *s3Storage) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error { errc := make(chan error, len(blobs)) diff --git a/pkg/schema/filewriter.go b/pkg/schema/filewriter.go index 17b5c9f21..78b543b89 100644 --- a/pkg/schema/filewriter.go +++ b/pkg/schema/filewriter.go @@ -26,8 +26,8 @@ import ( "camlistore.org/pkg/blob" "camlistore.org/pkg/blobserver" - "camlistore.org/pkg/gate" "camlistore.org/pkg/rollsum" + "camlistore.org/pkg/syncutil" ) const ( @@ -340,7 +340,7 @@ func writeFileChunks(bs blobserver.StatReceiver, file *Builder, r io.Reader) (n blobSize := 0 // of the next blob being built, should be same as buf.Len() const chunksInFlight = 32 // at ~64 KB chunks, this is ~2MB memory per file - gatec := gate.New(chunksInFlight) + gatec := syncutil.NewGate(chunksInFlight) firsterrc := make(chan error, 1) // uploadLastSpan runs in the same goroutine as the loop below and is responsible for diff --git a/pkg/gate/gate.go b/pkg/syncutil/gate.go similarity index 81% rename from pkg/gate/gate.go rename to pkg/syncutil/gate.go index 3172a4b5d..497c7a5a5 100644 --- a/pkg/gate/gate.go +++ b/pkg/syncutil/gate.go @@ -14,17 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package gate provides a mechanism to control a maximum number of -// operations happening at once. -package gate +// Package syncutil provides various concurrency mechanisms. +package syncutil // A Gate limits concurrency. type Gate struct { c chan struct{} } -// New returns a new gate that will only permit max operations at once. -func New(max int) *Gate { +// NewGate returns a new gate that will only permit max operations at once. +func NewGate(max int) *Gate { return &Gate{make(chan struct{}, max)} }