From 8fb6c4f764dd5bddab8eeefc18c0515a340989cd Mon Sep 17 00:00:00 2001 From: Ben Harper Date: Wed, 1 Apr 2015 16:39:53 +0200 Subject: [PATCH 01/10] Add byte slice accessor to Go code --- go/table.go | 7 ++++++- src/idl_gen_go.cpp | 16 ++++++++++++++++ tests/MyGame/Example/Monster.go | 18 ++++++++++++++++++ tests/go_test.go | 8 ++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/go/table.go b/go/table.go index dc5c39908..695b92da8 100644 --- a/go/table.go +++ b/go/table.go @@ -26,10 +26,15 @@ func (t *Table) Indirect(off UOffsetT) UOffsetT { // String gets a string from data stored inside the flatbuffer. func (t *Table) String(off UOffsetT) string { + return string(t.ByteVector(off)) +} + +// ByteVector gets a byte slice from data stored inside the flatbuffer. +func (t *Table) ByteVector(off UOffsetT) []byte { off += GetUOffsetT(t.Bytes[off:]) start := off + UOffsetT(SizeUOffsetT) length := GetUOffsetT(t.Bytes[off:]) - return string(t.Bytes[start : start+length]) + return t.Bytes[start : start+length] } // VectorLen retrieves the length of the vector whose offset is stored at diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp index 24b7d0380..630fd09d4 100644 --- a/src/idl_gen_go.cpp +++ b/src/idl_gen_go.cpp @@ -144,6 +144,19 @@ static void GetVectorLen(const StructDef &struct_def, code += "\treturn 0\n}\n\n"; } +// Get a [ubyte] vector as a byte slice. +static void GetUByteSlice(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name) + "Bytes("; + code += ") []byte " + OffsetPrefix(field); + code += "\t\treturn rcv._tab.ByteVector(o + rcv._tab.Pos)\n\t}\n"; + code += "\treturn nil\n}\n\n"; +} + // Get the value of a struct's scalar. static void GetScalarFieldOfStruct(const StructDef &struct_def, const FieldDef &field, @@ -480,6 +493,9 @@ static void GenStructAccessor(const StructDef &struct_def, } if (field.value.type.base_type == BASE_TYPE_VECTOR) { GetVectorLen(struct_def, field, code_ptr); + if (field.value.type.element == BASE_TYPE_UCHAR) { + GetUByteSlice(struct_def, field, code_ptr); + } } } diff --git a/tests/MyGame/Example/Monster.go b/tests/MyGame/Example/Monster.go index 1abb902e6..0ca0c8097 100644 --- a/tests/MyGame/Example/Monster.go +++ b/tests/MyGame/Example/Monster.go @@ -75,6 +75,14 @@ func (rcv *Monster) InventoryLength() int { return 0 } +func (rcv *Monster) InventoryBytes() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + func (rcv *Monster) Color() int8 { o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) if o != 0 { @@ -140,7 +148,9 @@ func (rcv *Monster) TestarrayofstringLength() int { } /// an example documentation comment: this will end up in the generated code + /// multiline too + func (rcv *Monster) Testarrayoftables(obj *Monster, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(26)) if o != 0 { @@ -194,6 +204,14 @@ func (rcv *Monster) TestnestedflatbufferLength() int { return 0 } +func (rcv *Monster) TestnestedflatbufferBytes() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(30)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + func (rcv *Monster) Testempty(obj *Stat) *Stat { o := flatbuffers.UOffsetT(rcv._tab.Offset(32)) if o != 0 { diff --git a/tests/go_test.go b/tests/go_test.go index 03eaddfad..02311eb58 100644 --- a/tests/go_test.go +++ b/tests/go_test.go @@ -213,6 +213,11 @@ func CheckReadBuffer(buf []byte, offset flatbuffers.UOffsetT, fail func(string, fail(FailString("monster2.Name()", "Fred", got)) } + inventorySlice := monster.InventoryBytes() + if len(inventorySlice) != monster.InventoryLength() { + fail(FailString("len(monster.InventoryBytes) != monster.InventoryLength", len(inventorySlice), monster.InventoryLength())) + } + if got := monster.InventoryLength(); 5 != got { fail(FailString("monster.InventoryLength", 5, got)) } @@ -221,6 +226,9 @@ func CheckReadBuffer(buf []byte, offset flatbuffers.UOffsetT, fail func(string, l := monster.InventoryLength() for i := 0; i < l; i++ { v := monster.Inventory(i) + if v != inventorySlice[i] { + fail(FailString("monster inventory slice[i] != Inventory(i)", v, inventorySlice[i])) + } invsum += int(v) } if invsum != 10 { From 468124fb9bec295541e21b68a705447980f4825f Mon Sep 17 00:00:00 2001 From: rw Date: Thu, 2 Apr 2015 11:25:48 -0700 Subject: [PATCH 02/10] chmod GoTest.sh +x --- tests/GoTest.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/GoTest.sh diff --git a/tests/GoTest.sh b/tests/GoTest.sh old mode 100644 new mode 100755 From 796be3282c05c59a0919595d9a80c6fdf3b15753 Mon Sep 17 00:00:00 2001 From: rw Date: Thu, 2 Apr 2015 11:26:00 -0700 Subject: [PATCH 03/10] Benchmarks for building and parsing 'gold' data. Identifies alloc-heavy codepaths. --- tests/go_test.go | 122 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/tests/go_test.go b/tests/go_test.go index 03eaddfad..1cd11d7a2 100644 --- a/tests/go_test.go +++ b/tests/go_test.go @@ -1162,3 +1162,125 @@ func BenchmarkVtableDeduplication(b *testing.B) { builder.EndObject() } } + +// BenchmarkParseGold measures the speed of parsing the 'gold' data +// used throughout this test suite. +func BenchmarkParseGold(b *testing.B) { + buf, offset := CheckGeneratedBuild(b.Fatalf) + monster := example.GetRootAsMonster(buf, offset) + + // use these to prevent allocations: + reuse_pos := example.Vec3{} + reuse_test3 := example.Test{} + reuse_table2 := flatbuffers.Table{} + reuse_monster2 := example.Monster{} + reuse_test4_0 := example.Test{} + reuse_test4_1 := example.Test{} + + b.SetBytes(int64(len(buf[offset:]))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + monster.Hp() + monster.Mana() + name := monster.Name() + _ = name[0] + _ = name[len(name)-1] + + monster.Pos(&reuse_pos) + reuse_pos.X() + reuse_pos.Y() + reuse_pos.Z() + reuse_pos.Test1() + reuse_pos.Test2() + reuse_pos.Test3(&reuse_test3) + reuse_test3.A() + reuse_test3.B() + monster.TestType() + monster.Test(&reuse_table2) + reuse_monster2.Init(reuse_table2.Bytes, reuse_table2.Pos) + name2 := reuse_monster2.Name() + _ = name2[0] + _ = name2[len(name2)-1] + monster.InventoryLength() + l := monster.InventoryLength() + for i := 0; i < l; i++ { + monster.Inventory(i) + } + monster.Test4Length() + monster.Test4(&reuse_test4_0, 0) + monster.Test4(&reuse_test4_1, 1) + + reuse_test4_0.A() + reuse_test4_0.B() + reuse_test4_1.A() + reuse_test4_1.B() + + monster.TestarrayofstringLength() + str0 := monster.Testarrayofstring(0) + _ = str0[0] + _ = str0[len(str0)-1] + str1 := monster.Testarrayofstring(1) + _ = str1[0] + _ = str1[len(str1)-1] + } +} + +// BenchmarkBuildGold uses generated code to build the example Monster. +func BenchmarkBuildGold(b *testing.B) { + buf, offset := CheckGeneratedBuild(b.Fatalf) + bytes_length := int64(len(buf[offset:])) + + reuse_str := "MyMonster" + reuse_test1 := "test1" + reuse_test2 := "test2" + reuse_fred := "Fred" + + b.SetBytes(bytes_length) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + bldr := flatbuffers.NewBuilder(0) + str := bldr.CreateString(reuse_str) + test1 := bldr.CreateString(reuse_test1) + test2 := bldr.CreateString(reuse_test2) + fred := bldr.CreateString(reuse_fred) + + example.MonsterStartInventoryVector(bldr, 5) + bldr.PrependByte(4) + bldr.PrependByte(3) + bldr.PrependByte(2) + bldr.PrependByte(1) + bldr.PrependByte(0) + inv := bldr.EndVector(5) + + example.MonsterStart(bldr) + example.MonsterAddName(bldr, fred) + mon2 := example.MonsterEnd(bldr) + + example.MonsterStartTest4Vector(bldr, 2) + example.CreateTest(bldr, 10, 20) + example.CreateTest(bldr, 30, 40) + test4 := bldr.EndVector(2) + + example.MonsterStartTestarrayofstringVector(bldr, 2) + bldr.PrependUOffsetT(test2) + bldr.PrependUOffsetT(test1) + testArrayOfString := bldr.EndVector(2) + + example.MonsterStart(bldr) + + pos := example.CreateVec3(bldr, 1.0, 2.0, 3.0, 3.0, 2, 5, 6) + example.MonsterAddPos(bldr, pos) + + example.MonsterAddHp(bldr, 80) + example.MonsterAddName(bldr, str) + example.MonsterAddInventory(bldr, inv) + example.MonsterAddTestType(bldr, 1) + example.MonsterAddTest(bldr, mon2) + example.MonsterAddTest4(bldr, test4) + example.MonsterAddTestarrayofstring(bldr, testArrayOfString) + mon := example.MonsterEnd(bldr) + + bldr.Finish(mon) + } +} From f02646e35757d80ee23d8fe6dde26756d1d86c8c Mon Sep 17 00:00:00 2001 From: rw Date: Thu, 2 Apr 2015 11:56:55 -0700 Subject: [PATCH 04/10] Remove all string allocations during parsing. Change the signature for 'string' getters and settings to use byte slices instead of strings. --- src/idl_gen_go.cpp | 8 ++++---- tests/GoTest.sh | 2 ++ tests/MyGame/Example/Monster.go | 14 ++++++-------- tests/MyGame/Example/Stat.go | 6 +++--- tests/go_test.go | 8 ++++---- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp index 630fd09d4..08be8105b 100644 --- a/src/idl_gen_go.cpp +++ b/src/idl_gen_go.cpp @@ -238,7 +238,7 @@ static void GetStringField(const StructDef &struct_def, code += " " + MakeCamel(field.name); code += "() " + TypeName(field) + " "; code += OffsetPrefix(field) + "\t\treturn " + GenGetter(field.value.type); - code += "(o + rcv._tab.Pos)\n\t}\n\treturn \"\"\n"; + code += "(o + rcv._tab.Pos)\n\t}\n\treturn nil\n"; code += "}\n\n"; } @@ -301,7 +301,7 @@ static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, code += NumToString(InlineSize(vectortype)) + "))\n"; code += "\t}\n"; if (vectortype.base_type == BASE_TYPE_STRING) { - code += "\treturn \"\"\n"; + code += "\treturn nil\n"; } else { code += "\treturn 0\n"; } @@ -573,7 +573,7 @@ static void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { // Returns the function name that is able to read a value of the given type. static std::string GenGetter(const Type &type) { switch (type.base_type) { - case BASE_TYPE_STRING: return "rcv._tab.String"; + case BASE_TYPE_STRING: return "rcv._tab.ByteVector"; case BASE_TYPE_UNION: return "rcv._tab.Union"; case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); default: @@ -626,7 +626,7 @@ static std::string GenTypeBasic(const Type &type) { static std::string GenTypePointer(const Type &type) { switch (type.base_type) { case BASE_TYPE_STRING: - return "string"; + return "[]byte"; case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); case BASE_TYPE_STRUCT: diff --git a/tests/GoTest.sh b/tests/GoTest.sh index ba4835a20..877a3ca9f 100755 --- a/tests/GoTest.sh +++ b/tests/GoTest.sh @@ -43,6 +43,8 @@ GOPATH=${go_path} go test flatbuffers_test \ --test.coverpkg=github.com/google/flatbuffers/go \ --cpp_data=${test_dir}/monsterdata_test.mon \ --out_data=${test_dir}/monsterdata_go_wire.mon \ + --test.bench=. \ + --test.benchtime=5s \ --fuzz=true \ --fuzz_fields=4 \ --fuzz_objects=10000 diff --git a/tests/MyGame/Example/Monster.go b/tests/MyGame/Example/Monster.go index 0ca0c8097..a51e42966 100644 --- a/tests/MyGame/Example/Monster.go +++ b/tests/MyGame/Example/Monster.go @@ -50,12 +50,12 @@ func (rcv *Monster) Hp() int16 { return 100 } -func (rcv *Monster) Name() string { +func (rcv *Monster) Name() []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { - return rcv._tab.String(o + rcv._tab.Pos) + return rcv._tab.ByteVector(o + rcv._tab.Pos) } - return "" + return nil } func (rcv *Monster) Inventory(j int) byte { @@ -130,13 +130,13 @@ func (rcv *Monster) Test4Length() int { return 0 } -func (rcv *Monster) Testarrayofstring(j int) string { +func (rcv *Monster) Testarrayofstring(j int) []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(24)) if o != 0 { a := rcv._tab.Vector(o) - return rcv._tab.String(a + flatbuffers.UOffsetT(j * 4)) + return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j * 4)) } - return "" + return nil } func (rcv *Monster) TestarrayofstringLength() int { @@ -148,9 +148,7 @@ func (rcv *Monster) TestarrayofstringLength() int { } /// an example documentation comment: this will end up in the generated code - /// multiline too - func (rcv *Monster) Testarrayoftables(obj *Monster, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(26)) if o != 0 { diff --git a/tests/MyGame/Example/Stat.go b/tests/MyGame/Example/Stat.go index b2c8e3e18..936d5ec9c 100644 --- a/tests/MyGame/Example/Stat.go +++ b/tests/MyGame/Example/Stat.go @@ -14,12 +14,12 @@ func (rcv *Stat) Init(buf []byte, i flatbuffers.UOffsetT) { rcv._tab.Pos = i } -func (rcv *Stat) Id() string { +func (rcv *Stat) Id() []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { - return rcv._tab.String(o + rcv._tab.Pos) + return rcv._tab.ByteVector(o + rcv._tab.Pos) } - return "" + return nil } func (rcv *Stat) Val() int64 { diff --git a/tests/go_test.go b/tests/go_test.go index ec48d2f41..c1ca1658e 100644 --- a/tests/go_test.go +++ b/tests/go_test.go @@ -131,7 +131,7 @@ func CheckReadBuffer(buf []byte, offset flatbuffers.UOffsetT, fail func(string, fail(FailString("mana", 150, got)) } - if got := monster.Name(); "MyMonster" != got { + if got := monster.Name(); !bytes.Equal([]byte("MyMonster"), got) { fail(FailString("name", "MyMonster", got)) } @@ -209,7 +209,7 @@ func CheckReadBuffer(buf []byte, offset flatbuffers.UOffsetT, fail func(string, var monster2 example.Monster monster2.Init(table2.Bytes, table2.Pos) - if got := monster2.Name(); "Fred" != got { + if got := monster2.Name(); !bytes.Equal([]byte("Fred"), got) { fail(FailString("monster2.Name()", "Fred", got)) } @@ -267,11 +267,11 @@ func CheckReadBuffer(buf []byte, offset flatbuffers.UOffsetT, fail func(string, fail(FailString("Testarrayofstring length", 2, got)) } - if got := monster.Testarrayofstring(0); "test1" != got { + if got := monster.Testarrayofstring(0); !bytes.Equal([]byte("test1"), got) { fail(FailString("Testarrayofstring(0)", "test1", got)) } - if got := monster.Testarrayofstring(1); "test2" != got { + if got := monster.Testarrayofstring(1); !bytes.Equal([]byte("test2"), got) { fail(FailString("Testarrayofstring(1)", "test2", got)) } } From ace7fa8094de5ba92a74dce63903f8386d30e8b8 Mon Sep 17 00:00:00 2001 From: rw Date: Thu, 2 Apr 2015 18:22:13 -0700 Subject: [PATCH 05/10] Reduce allocations when building strings. Builder has a new CreateByteString function that writes a null-terimnated byte slice to the buffer. This results in zero allocations for writing strings. --- go/builder.go | 12 ++++++++---- tests/GoTest.sh | 4 ++-- tests/go_test.go | 25 +++++++++++++++++-------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/go/builder.go b/go/builder.go index e1b1f4308..cdae69598 100644 --- a/go/builder.go +++ b/go/builder.go @@ -247,16 +247,20 @@ func (b *Builder) EndVector(vectorNumElems int) UOffsetT { // CreateString writes a null-terminated string as a vector. func (b *Builder) CreateString(s string) UOffsetT { + return b.CreateByteString([]byte(s)) +} + +// CreateByteString writes a null-terminated byteslice as a vector. +func (b *Builder) CreateByteString(s []byte) UOffsetT { b.Prep(int(SizeUOffsetT), (len(s)+1)*SizeByte) b.PlaceByte(0) - x := []byte(s) - l := UOffsetT(len(x)) + l := UOffsetT(len(s)) b.head -= l - copy(b.Bytes[b.head:b.head+l], x) + copy(b.Bytes[b.head:b.head+l], s) - return b.EndVector(len(x)) + return b.EndVector(len(s)) } // CreateByteVector writes a ubyte vector diff --git a/tests/GoTest.sh b/tests/GoTest.sh index 877a3ca9f..2011434a3 100755 --- a/tests/GoTest.sh +++ b/tests/GoTest.sh @@ -43,8 +43,8 @@ GOPATH=${go_path} go test flatbuffers_test \ --test.coverpkg=github.com/google/flatbuffers/go \ --cpp_data=${test_dir}/monsterdata_test.mon \ --out_data=${test_dir}/monsterdata_go_wire.mon \ - --test.bench=. \ - --test.benchtime=5s \ + --test.bench=Build \ + --test.benchtime=3s \ --fuzz=true \ --fuzz_fields=4 \ --fuzz_objects=10000 diff --git a/tests/go_test.go b/tests/go_test.go index c1ca1658e..87a81c3f5 100644 --- a/tests/go_test.go +++ b/tests/go_test.go @@ -518,6 +518,15 @@ func CheckByteLayout(fail func(string, ...interface{})) { check([]byte{4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, // 0-terminated, 3-byte pad 3, 0, 0, 0, 'f', 'o', 'o', 0}) + // test 6b: CreateByteString + + b = flatbuffers.NewBuilder(0) + b.CreateByteString([]byte("foo")) + check([]byte{3, 0, 0, 0, 'f', 'o', 'o', 0}) // 0-terminated, no pad + b.CreateByteString([]byte("moop")) + check([]byte{4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, // 0-terminated, 3-byte pad + 3, 0, 0, 0, 'f', 'o', 'o', 0}) + // test 7: empty vtable b = flatbuffers.NewBuilder(0) b.StartObject(0) @@ -1239,19 +1248,19 @@ func BenchmarkBuildGold(b *testing.B) { buf, offset := CheckGeneratedBuild(b.Fatalf) bytes_length := int64(len(buf[offset:])) - reuse_str := "MyMonster" - reuse_test1 := "test1" - reuse_test2 := "test2" - reuse_fred := "Fred" + reuse_str := []byte("MyMonster") + reuse_test1 := []byte("test1") + reuse_test2 := []byte("test2") + reuse_fred := []byte("Fred") b.SetBytes(bytes_length) b.ReportAllocs() for i := 0; i < b.N; i++ { bldr := flatbuffers.NewBuilder(0) - str := bldr.CreateString(reuse_str) - test1 := bldr.CreateString(reuse_test1) - test2 := bldr.CreateString(reuse_test2) - fred := bldr.CreateString(reuse_fred) + str := bldr.CreateByteString(reuse_str) + test1 := bldr.CreateByteString(reuse_test1) + test2 := bldr.CreateByteString(reuse_test2) + fred := bldr.CreateByteString(reuse_fred) example.MonsterStartInventoryVector(bldr, 5) bldr.PrependByte(4) From d756efbf764af942ef0563b2a1141349fbc4e72b Mon Sep 17 00:00:00 2001 From: rw Date: Thu, 2 Apr 2015 19:33:00 -0700 Subject: [PATCH 06/10] Reduce allocations when reusing a Builder. Add the function `Reset` to the Builder, which facilitates reuse of the underlying byte slice. --- go/builder.go | 31 +++++++++++++++++++++++++------ tests/GoTest.sh | 2 +- tests/go_test.go | 4 +++- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/go/builder.go b/go/builder.go index cdae69598..d8be20486 100644 --- a/go/builder.go +++ b/go/builder.go @@ -31,6 +31,15 @@ func NewBuilder(initialSize int) *Builder { return b } +// Reset truncates the underlying Builder buffer, facilitating alloc-free +// reuse of a Builder. +func (b *Builder) Reset() { + b.Bytes = b.Bytes[:0] + b.head = UOffsetT(0) + b.minalign = 1 + b.vtables = b.vtables[:0] +} + // StartObject initializes bookkeeping for writing a new object. func (b *Builder) StartObject(numfields int) { b.notNested() @@ -155,13 +164,23 @@ func (b *Builder) growByteBuffer() { if (int64(len(b.Bytes)) & int64(0xC0000000)) != 0 { panic("cannot grow buffer beyond 2 gigabytes") } - newSize := len(b.Bytes) * 2 - if newSize == 0 { - newSize = 1 + newLen := len(b.Bytes) * 2 + if newLen == 0 { + newLen = 1 + } + + if cap(b.Bytes) >= newLen { + b.Bytes = b.Bytes[:newLen] + } else { + extension := make([]byte, newLen-len(b.Bytes)) + b.Bytes = append(b.Bytes, extension...) + } + + middle := newLen / 2 + copy(b.Bytes[middle:], b.Bytes[:middle]) + for i := 0 ; i < middle; i++ { + b.Bytes[i] = 0 } - bytes2 := make([]byte, newSize) - copy(bytes2[newSize-len(b.Bytes):], b.Bytes) - b.Bytes = bytes2 } // Head gives the start of useful data in the underlying byte buffer. diff --git a/tests/GoTest.sh b/tests/GoTest.sh index 2011434a3..f9cc41040 100755 --- a/tests/GoTest.sh +++ b/tests/GoTest.sh @@ -43,7 +43,7 @@ GOPATH=${go_path} go test flatbuffers_test \ --test.coverpkg=github.com/google/flatbuffers/go \ --cpp_data=${test_dir}/monsterdata_test.mon \ --out_data=${test_dir}/monsterdata_go_wire.mon \ - --test.bench=Build \ + --test.bench=. \ --test.benchtime=3s \ --fuzz=true \ --fuzz_fields=4 \ diff --git a/tests/go_test.go b/tests/go_test.go index 87a81c3f5..df887053d 100644 --- a/tests/go_test.go +++ b/tests/go_test.go @@ -1254,9 +1254,11 @@ func BenchmarkBuildGold(b *testing.B) { reuse_fred := []byte("Fred") b.SetBytes(bytes_length) + bldr := flatbuffers.NewBuilder(512) b.ReportAllocs() for i := 0; i < b.N; i++ { - bldr := flatbuffers.NewBuilder(0) + bldr.Reset() + str := bldr.CreateByteString(reuse_str) test1 := bldr.CreateByteString(reuse_test1) test2 := bldr.CreateByteString(reuse_test2) From 5d68493df4848e46a58df1baaa562e520dc7ae39 Mon Sep 17 00:00:00 2001 From: rw Date: Sat, 9 May 2015 15:37:13 -0700 Subject: [PATCH 07/10] update CheckClash for string accesses --- src/idl_parser.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index f64fb7955..9a4f32062 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -947,6 +947,8 @@ void Parser::ParseDecl() { CheckClash("Type", BASE_TYPE_UNION); CheckClash("_length", BASE_TYPE_VECTOR); CheckClash("Length", BASE_TYPE_VECTOR); + CheckClash("_byte_vector", BASE_TYPE_STRING); + CheckClash("ByteVector", BASE_TYPE_STRING); Expect('}'); } From 3dd54424c031ab91050760c0816f779bd4ccc76c Mon Sep 17 00:00:00 2001 From: rw Date: Sat, 9 May 2015 16:07:11 -0700 Subject: [PATCH 08/10] remove remaining allocs during build --- go/builder.go | 30 ++++++++++++++++++++++++------ tests/go_test.go | 3 ++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/go/builder.go b/go/builder.go index d8be20486..a61448d56 100644 --- a/go/builder.go +++ b/go/builder.go @@ -34,17 +34,35 @@ func NewBuilder(initialSize int) *Builder { // Reset truncates the underlying Builder buffer, facilitating alloc-free // reuse of a Builder. func (b *Builder) Reset() { - b.Bytes = b.Bytes[:0] b.head = UOffsetT(0) b.minalign = 1 - b.vtables = b.vtables[:0] + + if b.Bytes != nil { + b.Bytes = b.Bytes[:0] + } + + if b.vtables != nil { + b.vtables = b.vtables[:0] + } + + if b.vtable != nil { + b.vtable = b.vtable[:0] + } } // StartObject initializes bookkeeping for writing a new object. func (b *Builder) StartObject(numfields int) { b.notNested() // use 32-bit offsets so that arithmetic doesn't overflow. - b.vtable = make([]UOffsetT, numfields) + if cap(b.vtable) < numfields || b.vtable == nil { + b.vtable = make([]UOffsetT, numfields) + } else { + b.vtable = b.vtable[:numfields] + for i := 0 ; i < len(b.vtable) ; i++ { + b.vtable[i] = 0 + } + } + b.objectEnd = b.Offset() b.minalign = 1 } @@ -146,7 +164,7 @@ func (b *Builder) WriteVtable() (n UOffsetT) { SOffsetT(existingVtable)-SOffsetT(objectOffset)) } - b.vtable = nil + b.vtable = b.vtable[:0] return objectOffset } @@ -269,7 +287,7 @@ func (b *Builder) CreateString(s string) UOffsetT { return b.CreateByteString([]byte(s)) } -// CreateByteString writes a null-terminated byteslice as a vector. +// CreateByteString writes a byte slice as a string (null-terminated). func (b *Builder) CreateByteString(s []byte) UOffsetT { b.Prep(int(SizeUOffsetT), (len(s)+1)*SizeByte) b.PlaceByte(0) @@ -297,7 +315,7 @@ func (b *Builder) CreateByteVector(v []byte) UOffsetT { func (b *Builder) notNested() { // Check that no other objects are being built while making this // object. If not, panic: - if b.vtable != nil { + if len(b.vtable) > 0 { panic("non-inline data write inside of object") } } diff --git a/tests/go_test.go b/tests/go_test.go index df887053d..36c30cbe2 100644 --- a/tests/go_test.go +++ b/tests/go_test.go @@ -1254,7 +1254,8 @@ func BenchmarkBuildGold(b *testing.B) { reuse_fred := []byte("Fred") b.SetBytes(bytes_length) - bldr := flatbuffers.NewBuilder(512) + bldr := flatbuffers.NewBuilder(0) + b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { bldr.Reset() From e11da87a24f51d651e54bb5769c892b9c183516b Mon Sep 17 00:00:00 2001 From: rw Date: Sat, 9 May 2015 16:10:03 -0700 Subject: [PATCH 09/10] gofmt --- go/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/builder.go b/go/builder.go index a61448d56..0484af7b6 100644 --- a/go/builder.go +++ b/go/builder.go @@ -58,7 +58,7 @@ func (b *Builder) StartObject(numfields int) { b.vtable = make([]UOffsetT, numfields) } else { b.vtable = b.vtable[:numfields] - for i := 0 ; i < len(b.vtable) ; i++ { + for i := 0; i < len(b.vtable); i++ { b.vtable[i] = 0 } } @@ -196,7 +196,7 @@ func (b *Builder) growByteBuffer() { middle := newLen / 2 copy(b.Bytes[middle:], b.Bytes[:middle]) - for i := 0 ; i < middle; i++ { + for i := 0; i < middle; i++ { b.Bytes[i] = 0 } } From e5c21ec66685df0b3e113a9e708a2d4458e9d859 Mon Sep 17 00:00:00 2001 From: rw Date: Sat, 9 May 2015 16:32:26 -0700 Subject: [PATCH 10/10] invoke many fewer growth events --- go/builder.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/go/builder.go b/go/builder.go index 0484af7b6..81905f1a4 100644 --- a/go/builder.go +++ b/go/builder.go @@ -34,11 +34,8 @@ func NewBuilder(initialSize int) *Builder { // Reset truncates the underlying Builder buffer, facilitating alloc-free // reuse of a Builder. func (b *Builder) Reset() { - b.head = UOffsetT(0) - b.minalign = 1 - if b.Bytes != nil { - b.Bytes = b.Bytes[:0] + b.Bytes = b.Bytes[:cap(b.Bytes)] } if b.vtables != nil { @@ -48,6 +45,9 @@ func (b *Builder) Reset() { if b.vtable != nil { b.vtable = b.vtable[:0] } + + b.head = UOffsetT(len(b.Bytes)) + b.minalign = 1 } // StartObject initializes bookkeeping for writing a new object. @@ -196,9 +196,6 @@ func (b *Builder) growByteBuffer() { middle := newLen / 2 copy(b.Bytes[middle:], b.Bytes[:middle]) - for i := 0; i < middle; i++ { - b.Bytes[i] = 0 - } } // Head gives the start of useful data in the underlying byte buffer.