From 52ed51b80b73412a89789c6a38fc1a19522cf912 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 11 Jan 2014 21:41:53 -0800 Subject: [PATCH] New 'gc' package, implementing a generic garbage collector. Not yet used by Camlistore. Change-Id: I81d04ef0c2cbf7457a3ee734bb3710662ff8db34 --- pkg/gc/gc.go | 198 ++++++++++++++++++++++++++++++++++++++++++++++ pkg/gc/gc_test.go | 183 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 pkg/gc/gc.go create mode 100644 pkg/gc/gc_test.go diff --git a/pkg/gc/gc.go b/pkg/gc/gc.go new file mode 100644 index 000000000..06a1298a0 --- /dev/null +++ b/pkg/gc/gc.go @@ -0,0 +1,198 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package gc defines a generic garbage collector. +package gc + +import ( + "errors" + "fmt" + + "camlistore.org/pkg/context" + "camlistore.org/pkg/syncutil" +) + +const buffered = 32 // arbitrary + +// Item is something that exists that may or may not survive a GC collection. +type Item interface{} + +// A Collector performs a garbage collection. +type Collector struct { + // World specifies a World that should be stopped before a + // collection and started again after. + World World + + Marker Marker + Roots Enumerator + Sweeper Enumerator + ItemEnumerator ItemEnumerator + Deleter Deleter +} + +type Marker interface { + // Mark marks that an item should exist. + // It must be safe for calls from concurrent goroutines. + Mark(Item) error + + // IsMarked returns whether the item is marked. + // It must be safe for calls from concurrent goroutines. + IsMarked(Item) (bool, error) +} + +// World defines the thing that should be stopped before GC and started after. +type World interface { + Stop() error + Start() error +} + +type Deleter interface { + // Delete deletes an item that was deemed unreachable via + // the garbage collector. + // It must be safe for calls from concurrent goroutines. + Delete(Item) error +} + +// Enumerator enumerates items. +type Enumerator interface { + // Enumerate enumerates items (which items depends on usage) + // and sends them to the provided channel. Regardless of return + // value, the channel should be closed. + // + // If the provided context is closed, Enumerate should return + // with an error (typically context.ErrCanceled) + Enumerate(*context.Context, chan<- Item) error +} + +// ItemEnumerator enumerates all the edges out from an item. +type ItemEnumerator interface { + // EnumerateItme is like Enuerator's Enumerate, but specific + // to the provided item. + EnumerateItem(*context.Context, Item, chan<- Item) error +} + +// ctx will be canceled on failure +func (c *Collector) markItem(ctx *context.Context, it Item, isRoot bool) error { + if !isRoot { + marked, err := c.Marker.IsMarked(it) + if err != nil { + return err + } + if marked { + return nil + } + } + if err := c.Marker.Mark(it); err != nil { + return err + } + + ch := make(chan Item, buffered) + var grp syncutil.Group + grp.Go(func() error { + return c.ItemEnumerator.EnumerateItem(ctx, it, ch) + }) + grp.Go(func() error { + for it := range ch { + if err := c.markItem(ctx, it, false); err != nil { + return err + } + } + return nil + }) + if err := grp.Err(); err != nil { + ctx.Cancel() + return err + } + return nil +} + +// Collect performs a garbage collection. +func (c *Collector) Collect(ctx *context.Context) (err error) { + if c.World == nil { + return errors.New("no World") + } + if c.Marker == nil { + return errors.New("no Marker") + } + if c.Roots == nil { + return errors.New("no Roots") + } + if c.Sweeper == nil { + return errors.New("no Sweeper") + } + if c.ItemEnumerator == nil { + return errors.New("no ItemEnumerator") + } + if c.Deleter == nil { + return errors.New("no Deleter") + } + if err := c.World.Stop(); err != nil { + return err + } + defer func() { + startErr := c.World.Start() + if err == nil { + err = startErr + } + }() + + // Mark. + roots := make(chan Item, buffered) + markCtx := ctx.New() + var marker syncutil.Group + marker.Go(func() error { + defer markCtx.Cancel() + for it := range roots { + if err := c.markItem(markCtx, it, true); err != nil { + return err + } + } + return nil + }) + marker.Go(func() error { + return c.Roots.Enumerate(markCtx, roots) + }) + if err := marker.Err(); err != nil { + return fmt.Errorf("Mark failure: %v", err) + } + + // Sweep. + all := make(chan Item, buffered) + sweepCtx := ctx.New() + var sweeper syncutil.Group + sweeper.Go(func() error { + return c.Sweeper.Enumerate(sweepCtx, all) + }) + sweeper.Go(func() error { + defer sweepCtx.Done() + for it := range all { + ok, err := c.Marker.IsMarked(it) + if err != nil { + return err + } + if !ok { + if err := c.Deleter.Delete(it); err != nil { + return err + } + } + } + return nil + }) + if err := sweeper.Err(); err != nil { + return fmt.Errorf("Sweep failure: %v", err) + } + return nil +} diff --git a/pkg/gc/gc_test.go b/pkg/gc/gc_test.go new file mode 100644 index 000000000..82edef3e4 --- /dev/null +++ b/pkg/gc/gc_test.go @@ -0,0 +1,183 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gc + +import ( + "reflect" + "sort" + "testing" + + "camlistore.org/pkg/context" +) + +func sl(v ...string) []string { + if len(v) == 0 { + return nil + } + return v +} + +var collectTests = []struct { + name string + world []string + roots []string + graph map[string][]string + wantWorld []string +}{ + { + name: "delete everything", + world: sl("a", "b", "c"), + wantWorld: sl(), + }, + + { + name: "keep everything", + world: sl("a", "b", "c"), + roots: sl("a", "b", "c"), + wantWorld: sl("a", "b", "c"), + }, + + { + name: "keep all via chain", + world: sl("a", "b", "c", "d", "e"), + roots: sl("a"), + graph: map[string][]string{ + "a": sl("b"), + "b": sl("c"), + "c": sl("d"), + "d": sl("e"), + }, + wantWorld: sl("a", "b", "c", "d", "e"), + }, + + { + name: "keep all via fan", + world: sl("a", "b", "c", "d", "e"), + roots: sl("a"), + graph: map[string][]string{ + "a": sl("b", "c", "d", "e"), + }, + wantWorld: sl("a", "b", "c", "d", "e"), + }, + + { + name: "c dies, two roots", + world: sl("a", "b", "c", "d", "e"), + roots: sl("a", "d"), + graph: map[string][]string{ + "a": sl("b"), + "d": sl("e"), + }, + wantWorld: sl("a", "b", "d", "e"), + }, +} + +type worldSet map[string]bool + +func newWorldSet(start []string) worldSet { + s := make(worldSet) + for _, v := range start { + s[v] = true + } + return s +} + +func (s worldSet) Delete(it Item) error { + delete(s, it.(string)) + return nil +} + +func (s worldSet) items() []string { + if len(s) == 0 { + return nil + } + ret := make([]string, 0, len(s)) + for it := range s { + ret = append(ret, it) + } + sort.Strings(ret) + return ret +} + +func TestCollector(t *testing.T) { + for _, tt := range collectTests { + if tt.name == "" { + panic("no name in test") + } + w := newWorldSet(tt.world) + c := &Collector{ + World: testWorld{}, + Marker: testMarker(map[Item]bool{}), + Roots: testEnum(tt.roots), + Sweeper: testEnum(tt.world), + ItemEnumerator: testItemEnum(tt.graph), + Deleter: w, + } + if err := c.Collect(context.New()); err != nil { + t.Errorf("%s: Collect = %v", tt.name, err) + } + got := w.items() + if !reflect.DeepEqual(tt.wantWorld, got) { + t.Errorf("%s: world = %q; want %q", tt.name, got, tt.wantWorld) + } + } +} + +type testEnum []string + +func (s testEnum) Enumerate(ctx *context.Context, dest chan<- Item) error { + defer close(dest) + for _, v := range s { + select { + case dest <- v: + case <-ctx.Done(): + return context.ErrCanceled + } + } + return nil +} + +type testItemEnum map[string][]string + +func (m testItemEnum) EnumerateItem(ctx *context.Context, it Item, dest chan<- Item) error { + defer close(dest) + for _, v := range m[it.(string)] { + select { + case dest <- v: + case <-ctx.Done(): + return context.ErrCanceled + } + } + return nil +} + +type testMarker map[Item]bool + +func (m testMarker) Mark(it Item) error { + m[it] = true + return nil +} + +func (m testMarker) IsMarked(it Item) (v bool, err error) { + v = m[it] + return +} + +type testWorld struct{} + +func (testWorld) Start() error { return nil } +func (testWorld) Stop() error { return nil }