mirror of https://github.com/perkeep/perkeep.git
New 'gc' package, implementing a generic garbage collector.
Not yet used by Camlistore. Change-Id: I81d04ef0c2cbf7457a3ee734bb3710662ff8db34
This commit is contained in:
Brad Fitzpatrick
parent
d3b0e933b9
commit
52ed51b80b
|
|
@ -0,0 +1,198 @@
|
|||
/*
|
||||
Copyright 2014 The Camlistore Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package gc defines a generic garbage collector.
|
||||
package gc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"camlistore.org/pkg/context"
|
||||
"camlistore.org/pkg/syncutil"
|
||||
)
|
||||
|
||||
const buffered = 32 // arbitrary
|
||||
|
||||
// Item is something that exists that may or may not survive a GC collection.
|
||||
type Item interface{}
|
||||
|
||||
// A Collector performs a garbage collection.
|
||||
type Collector struct {
|
||||
// World specifies a World that should be stopped before a
|
||||
// collection and started again after.
|
||||
World World
|
||||
|
||||
Marker Marker
|
||||
Roots Enumerator
|
||||
Sweeper Enumerator
|
||||
ItemEnumerator ItemEnumerator
|
||||
Deleter Deleter
|
||||
}
|
||||
|
||||
type Marker interface {
|
||||
// Mark marks that an item should exist.
|
||||
// It must be safe for calls from concurrent goroutines.
|
||||
Mark(Item) error
|
||||
|
||||
// IsMarked returns whether the item is marked.
|
||||
// It must be safe for calls from concurrent goroutines.
|
||||
IsMarked(Item) (bool, error)
|
||||
}
|
||||
|
||||
// World defines the thing that should be stopped before GC and started after.
|
||||
type World interface {
|
||||
Stop() error
|
||||
Start() error
|
||||
}
|
||||
|
||||
type Deleter interface {
|
||||
// Delete deletes an item that was deemed unreachable via
|
||||
// the garbage collector.
|
||||
// It must be safe for calls from concurrent goroutines.
|
||||
Delete(Item) error
|
||||
}
|
||||
|
||||
// Enumerator enumerates items.
|
||||
type Enumerator interface {
|
||||
// Enumerate enumerates items (which items depends on usage)
|
||||
// and sends them to the provided channel. Regardless of return
|
||||
// value, the channel should be closed.
|
||||
//
|
||||
// If the provided context is closed, Enumerate should return
|
||||
// with an error (typically context.ErrCanceled)
|
||||
Enumerate(*context.Context, chan<- Item) error
|
||||
}
|
||||
|
||||
// ItemEnumerator enumerates all the edges out from an item.
|
||||
type ItemEnumerator interface {
|
||||
// EnumerateItme is like Enuerator's Enumerate, but specific
|
||||
// to the provided item.
|
||||
EnumerateItem(*context.Context, Item, chan<- Item) error
|
||||
}
|
||||
|
||||
// ctx will be canceled on failure
|
||||
func (c *Collector) markItem(ctx *context.Context, it Item, isRoot bool) error {
|
||||
if !isRoot {
|
||||
marked, err := c.Marker.IsMarked(it)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if marked {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err := c.Marker.Mark(it); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ch := make(chan Item, buffered)
|
||||
var grp syncutil.Group
|
||||
grp.Go(func() error {
|
||||
return c.ItemEnumerator.EnumerateItem(ctx, it, ch)
|
||||
})
|
||||
grp.Go(func() error {
|
||||
for it := range ch {
|
||||
if err := c.markItem(ctx, it, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err := grp.Err(); err != nil {
|
||||
ctx.Cancel()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collect performs a garbage collection.
|
||||
func (c *Collector) Collect(ctx *context.Context) (err error) {
|
||||
if c.World == nil {
|
||||
return errors.New("no World")
|
||||
}
|
||||
if c.Marker == nil {
|
||||
return errors.New("no Marker")
|
||||
}
|
||||
if c.Roots == nil {
|
||||
return errors.New("no Roots")
|
||||
}
|
||||
if c.Sweeper == nil {
|
||||
return errors.New("no Sweeper")
|
||||
}
|
||||
if c.ItemEnumerator == nil {
|
||||
return errors.New("no ItemEnumerator")
|
||||
}
|
||||
if c.Deleter == nil {
|
||||
return errors.New("no Deleter")
|
||||
}
|
||||
if err := c.World.Stop(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
startErr := c.World.Start()
|
||||
if err == nil {
|
||||
err = startErr
|
||||
}
|
||||
}()
|
||||
|
||||
// Mark.
|
||||
roots := make(chan Item, buffered)
|
||||
markCtx := ctx.New()
|
||||
var marker syncutil.Group
|
||||
marker.Go(func() error {
|
||||
defer markCtx.Cancel()
|
||||
for it := range roots {
|
||||
if err := c.markItem(markCtx, it, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
marker.Go(func() error {
|
||||
return c.Roots.Enumerate(markCtx, roots)
|
||||
})
|
||||
if err := marker.Err(); err != nil {
|
||||
return fmt.Errorf("Mark failure: %v", err)
|
||||
}
|
||||
|
||||
// Sweep.
|
||||
all := make(chan Item, buffered)
|
||||
sweepCtx := ctx.New()
|
||||
var sweeper syncutil.Group
|
||||
sweeper.Go(func() error {
|
||||
return c.Sweeper.Enumerate(sweepCtx, all)
|
||||
})
|
||||
sweeper.Go(func() error {
|
||||
defer sweepCtx.Done()
|
||||
for it := range all {
|
||||
ok, err := c.Marker.IsMarked(it)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
if err := c.Deleter.Delete(it); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err := sweeper.Err(); err != nil {
|
||||
return fmt.Errorf("Sweep failure: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
/*
|
||||
Copyright 2014 The Camlistore Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gc
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"camlistore.org/pkg/context"
|
||||
)
|
||||
|
||||
func sl(v ...string) []string {
|
||||
if len(v) == 0 {
|
||||
return nil
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
var collectTests = []struct {
|
||||
name string
|
||||
world []string
|
||||
roots []string
|
||||
graph map[string][]string
|
||||
wantWorld []string
|
||||
}{
|
||||
{
|
||||
name: "delete everything",
|
||||
world: sl("a", "b", "c"),
|
||||
wantWorld: sl(),
|
||||
},
|
||||
|
||||
{
|
||||
name: "keep everything",
|
||||
world: sl("a", "b", "c"),
|
||||
roots: sl("a", "b", "c"),
|
||||
wantWorld: sl("a", "b", "c"),
|
||||
},
|
||||
|
||||
{
|
||||
name: "keep all via chain",
|
||||
world: sl("a", "b", "c", "d", "e"),
|
||||
roots: sl("a"),
|
||||
graph: map[string][]string{
|
||||
"a": sl("b"),
|
||||
"b": sl("c"),
|
||||
"c": sl("d"),
|
||||
"d": sl("e"),
|
||||
},
|
||||
wantWorld: sl("a", "b", "c", "d", "e"),
|
||||
},
|
||||
|
||||
{
|
||||
name: "keep all via fan",
|
||||
world: sl("a", "b", "c", "d", "e"),
|
||||
roots: sl("a"),
|
||||
graph: map[string][]string{
|
||||
"a": sl("b", "c", "d", "e"),
|
||||
},
|
||||
wantWorld: sl("a", "b", "c", "d", "e"),
|
||||
},
|
||||
|
||||
{
|
||||
name: "c dies, two roots",
|
||||
world: sl("a", "b", "c", "d", "e"),
|
||||
roots: sl("a", "d"),
|
||||
graph: map[string][]string{
|
||||
"a": sl("b"),
|
||||
"d": sl("e"),
|
||||
},
|
||||
wantWorld: sl("a", "b", "d", "e"),
|
||||
},
|
||||
}
|
||||
|
||||
type worldSet map[string]bool
|
||||
|
||||
func newWorldSet(start []string) worldSet {
|
||||
s := make(worldSet)
|
||||
for _, v := range start {
|
||||
s[v] = true
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s worldSet) Delete(it Item) error {
|
||||
delete(s, it.(string))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s worldSet) items() []string {
|
||||
if len(s) == 0 {
|
||||
return nil
|
||||
}
|
||||
ret := make([]string, 0, len(s))
|
||||
for it := range s {
|
||||
ret = append(ret, it)
|
||||
}
|
||||
sort.Strings(ret)
|
||||
return ret
|
||||
}
|
||||
|
||||
func TestCollector(t *testing.T) {
|
||||
for _, tt := range collectTests {
|
||||
if tt.name == "" {
|
||||
panic("no name in test")
|
||||
}
|
||||
w := newWorldSet(tt.world)
|
||||
c := &Collector{
|
||||
World: testWorld{},
|
||||
Marker: testMarker(map[Item]bool{}),
|
||||
Roots: testEnum(tt.roots),
|
||||
Sweeper: testEnum(tt.world),
|
||||
ItemEnumerator: testItemEnum(tt.graph),
|
||||
Deleter: w,
|
||||
}
|
||||
if err := c.Collect(context.New()); err != nil {
|
||||
t.Errorf("%s: Collect = %v", tt.name, err)
|
||||
}
|
||||
got := w.items()
|
||||
if !reflect.DeepEqual(tt.wantWorld, got) {
|
||||
t.Errorf("%s: world = %q; want %q", tt.name, got, tt.wantWorld)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type testEnum []string
|
||||
|
||||
func (s testEnum) Enumerate(ctx *context.Context, dest chan<- Item) error {
|
||||
defer close(dest)
|
||||
for _, v := range s {
|
||||
select {
|
||||
case dest <- v:
|
||||
case <-ctx.Done():
|
||||
return context.ErrCanceled
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type testItemEnum map[string][]string
|
||||
|
||||
func (m testItemEnum) EnumerateItem(ctx *context.Context, it Item, dest chan<- Item) error {
|
||||
defer close(dest)
|
||||
for _, v := range m[it.(string)] {
|
||||
select {
|
||||
case dest <- v:
|
||||
case <-ctx.Done():
|
||||
return context.ErrCanceled
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type testMarker map[Item]bool
|
||||
|
||||
func (m testMarker) Mark(it Item) error {
|
||||
m[it] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m testMarker) IsMarked(it Item) (v bool, err error) {
|
||||
v = m[it]
|
||||
return
|
||||
}
|
||||
|
||||
type testWorld struct{}
|
||||
|
||||
func (testWorld) Start() error { return nil }
|
||||
func (testWorld) Stop() error { return nil }
|
||||
Loading…
Reference in New Issue