From 48dfc69ee613a176f13b04c2310adb7a08fe6737 Mon Sep 17 00:00:00 2001 From: rw Date: Tue, 16 Dec 2014 00:32:11 -0800 Subject: [PATCH] Port FlatBuffers to Python. Implement code generation and self-contained runtime library for Python. The test suite verifies: - Correctness of generated Python code by comparing output to that of the other language ports. - The exact bytes in the Builder buffer during many scenarios. - Vtable deduplication correctness. - Edge cases for table construction, via a fuzzer derived from the Go implementation. - All code is simultaneously valid in Python 2.6, 2.7, and 3.4. The test suite includes benchmarks for: - Building 'gold' data. - Parsing 'gold' data. - Deduplicating vtables. All tests pass on this author's system for the following Python implementations: - CPython 2.6.7 - CPython 2.7.8 - CPython 3.4.2 - PyPy 2.5.0 (CPython 2.7.8 compatible) --- .gitignore | 1 + CMakeLists.txt | 1 + docs/source/PythonUsage.md | 115 +++ docs/source/doxyfile | 1 + include/flatbuffers/idl.h | 48 +- python/__init__.py | 0 python/flatbuffers/__init__.py | 17 + python/flatbuffers/builder.py | 549 ++++++++++++ python/flatbuffers/compat.py | 27 + python/flatbuffers/encode.py | 29 + python/flatbuffers/number_types.py | 174 ++++ python/flatbuffers/packer.py | 28 + python/flatbuffers/table.py | 117 +++ python/setup.py | 17 + samples/monster_generated.h | 0 src/flatc.cpp | 4 + src/idl_gen_cpp.cpp | 3 +- src/idl_gen_general.cpp | 2 +- src/idl_gen_go.cpp | 3 +- src/idl_gen_python.cpp | 664 ++++++++++++++ src/idl_gen_text.cpp | 9 +- src/idl_parser.cpp | 21 +- tests/MyGame/Example/Any.py | 8 + tests/MyGame/Example/Color.py | 9 + tests/MyGame/Example/Monster.py | 278 ++++++ tests/MyGame/Example/Stat.py | 39 + tests/MyGame/Example/Test.py | 24 + tests/MyGame/Example/Vec3.py | 44 + tests/MyGame/Example/__init__.py | 0 tests/MyGame/__init__.py | 0 tests/PythonTest.sh | 78 ++ tests/monster_test_generated.h | 0 tests/monsterdata_python_wire.mon | Bin 0 -> 288 bytes tests/py_test.py | 1331 ++++++++++++++++++++++++++++ tests/test.cpp | 2 +- 35 files changed, 3608 insertions(+), 35 deletions(-) create mode 100755 docs/source/PythonUsage.md create mode 100644 python/__init__.py create mode 100644 python/flatbuffers/__init__.py create mode 100644 python/flatbuffers/builder.py create mode 100644 python/flatbuffers/compat.py create mode 100644 python/flatbuffers/encode.py create mode 100644 python/flatbuffers/number_types.py create mode 100644 python/flatbuffers/packer.py create mode 100644 python/flatbuffers/table.py create mode 100644 python/setup.py mode change 100755 => 100644 samples/monster_generated.h create mode 100644 src/idl_gen_python.cpp create mode 100644 tests/MyGame/Example/Any.py create mode 100644 tests/MyGame/Example/Color.py create mode 100644 tests/MyGame/Example/Monster.py create mode 100644 tests/MyGame/Example/Stat.py create mode 100644 tests/MyGame/Example/Test.py create mode 100644 tests/MyGame/Example/Vec3.py create mode 100644 tests/MyGame/Example/__init__.py create mode 100644 tests/MyGame/__init__.py create mode 100755 tests/PythonTest.sh mode change 100755 => 100644 tests/monster_test_generated.h create mode 100644 tests/monsterdata_python_wire.mon create mode 100644 tests/py_test.py diff --git a/.gitignore b/.gitignore index c95336eb0..46d1ae419 100755 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,4 @@ FlatBuffers.xcodeproj/ java/.idea java/*.iml java/target +**/*.pyc diff --git a/CMakeLists.txt b/CMakeLists.txt index 39e65fae1..5f3da26d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ set(FlatBuffers_Compiler_SRCS src/idl_gen_cpp.cpp src/idl_gen_general.cpp src/idl_gen_go.cpp + src/idl_gen_python.cpp src/idl_gen_text.cpp src/idl_gen_fbs.cpp src/flatc.cpp diff --git a/docs/source/PythonUsage.md b/docs/source/PythonUsage.md new file mode 100755 index 000000000..e20b464d6 --- /dev/null +++ b/docs/source/PythonUsage.md @@ -0,0 +1,115 @@ +# Use in Python + +There's experimental support for reading FlatBuffers in Python. Generate +code for Python with the `-p` option to `flatc`. + +See `py_test.py` for an example. You import the generated code, read a +FlatBuffer binary file into a `bytearray`, which you pass to the +`GetRootAsMonster` function: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + import MyGame.Example as example + import flatbuffers + + buf = open('monster.dat', 'rb').read() + buf = bytearray(buf) + monster = example.GetRootAsMonster(buf, 0) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access values like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + hp = monster.Hp() + pos = monster.Pos() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To access vectors you pass an extra index to the +vector field accessor. Then a second method with the same name suffixed +by `Length` let's you know the number of elements you can access: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + for i in xrange(monster.InventoryLength()): + monster.Inventory(i) # do something here +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also construct these buffers in Python using the functions found +in the generated code, and the FlatBufferBuilder class: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + builder = flatbuffers.NewBuilder(0) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create strings: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + s = builder.CreateString("MyMonster") +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create a table with a struct contained therein: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + example.MonsterStart(builder) + example.MonsterAddPos(builder, example.CreateVec3(builder, 1.0, 2.0, 3.0, 3.0, 4, 5, 6)) + example.MonsterAddHp(builder, 80) + example.MonsterAddName(builder, str) + example.MonsterAddInventory(builder, inv) + example.MonsterAddTest_Type(builder, 1) + example.MonsterAddTest(builder, mon2) + example.MonsterAddTest4(builder, test4s) + mon = example.MonsterEnd(builder) + + final_flatbuffer = bulder.Output() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unlike C++, Python does not support table creation functions like 'createMonster()'. +This is to create the buffer without +using temporary object allocation (since the `Vec3` is an inline component of +`Monster`, it has to be created right where it is added, whereas the name and +the inventory are not inline, and **must** be created outside of the table +creation sequence). +Structs do have convenient methods that allow you to construct them in one call. +These also have arguments for nested structs, e.g. if a struct has a field `a` +and a nested struct field `b` (which has fields `c` and `d`), then the arguments +will be `a`, `c` and `d`. + +Vectors also use this start/end pattern to allow vectors of both scalar types +and structs: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + example.MonsterStartInventoryVector(builder, 5) + i = 4 + while i >= 0: + builder.PrependByte(byte(i)) + i -= 1 + + inv = builder.EndVector(5) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The generated method 'StartInventoryVector' is provided as a convenience +function which calls 'StartVector' with the correct element size of the vector +type which in this case is 'ubyte' or 1 byte per vector element. +You pass the number of elements you want to write. +You write the elements backwards since the buffer +is being constructed back to front. Use the correct `Prepend` call for the type, +or `PrependUOffsetT` for offsets. You then pass `inv` to the corresponding +`Add` call when you construct the table containing it afterwards. + +There are `Prepend` functions for all the scalar types. You use +`PrependUOffset` for any previously constructed objects (such as other tables, +strings, vectors). For structs, you use the appropriate `create` function +in-line, as shown above in the `Monster` example. + +Once you're done constructing a buffer, you call `Finish` with the root object +offset (`mon` in the example above). Your data now resides in Builder.Bytes. +Important to note is that the real data starts at the index indicated by Head(), +for Offset() bytes (this is because the buffer is constructed backwards). +If you wanted to read the buffer right after creating it (using +`GetRootAsMonster` above), the second argument, instead of `0` would thus +also be `Head()`. + +## Text Parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from Python, though you could use the C++ parser through SWIG or ctypes. Please +see the C++ documentation for more on text parsing. + diff --git a/docs/source/doxyfile b/docs/source/doxyfile index e0ece1d22..202f47dd0 100755 --- a/docs/source/doxyfile +++ b/docs/source/doxyfile @@ -750,6 +750,7 @@ INPUT = "FlatBuffers.md" \ "CppUsage.md" \ "GoUsage.md" \ "JavaUsage.md" \ + "PythonUsage.md" \ "Benchmarks.md" \ "WhitePaper.md" \ "Internals.md" \ diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 46f6540cd..ac0e20ec8 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -35,24 +35,24 @@ namespace flatbuffers { // Additionally, Parser::ParseType assumes bool..string is a contiguous range // of type tokens. #define FLATBUFFERS_GEN_TYPES_SCALAR(TD) \ - TD(NONE, "", uint8_t, byte, byte, byte) \ - TD(UTYPE, "", uint8_t, byte, byte, byte) /* begin scalar/int */ \ - TD(BOOL, "bool", uint8_t, boolean,byte, bool) \ - TD(CHAR, "byte", int8_t, byte, int8, sbyte) \ - TD(UCHAR, "ubyte", uint8_t, byte, byte, byte) \ - TD(SHORT, "short", int16_t, short, int16, short) \ - TD(USHORT, "ushort", uint16_t, short, uint16, ushort) \ - TD(INT, "int", int32_t, int, int32, int) \ - TD(UINT, "uint", uint32_t, int, uint32, uint) \ - TD(LONG, "long", int64_t, long, int64, long) \ - TD(ULONG, "ulong", uint64_t, long, uint64, ulong) /* end int */ \ - TD(FLOAT, "float", float, float, float32, float) /* begin float */ \ - TD(DOUBLE, "double", double, double, float64, double) /* end float/scalar */ + TD(NONE, "", uint8_t, byte, byte, byte, uint8) \ + TD(UTYPE, "", uint8_t, byte, byte, byte, uint8) /* begin scalar/int */ \ + TD(BOOL, "bool", uint8_t, boolean,byte, bool, bool) \ + TD(CHAR, "byte", int8_t, byte, int8, sbyte, int8) \ + TD(UCHAR, "ubyte", uint8_t, byte, byte, byte, uint8) \ + TD(SHORT, "short", int16_t, short, int16, short, int16) \ + TD(USHORT, "ushort", uint16_t, short, uint16, ushort, uint16) \ + TD(INT, "int", int32_t, int, int32, int, int32) \ + TD(UINT, "uint", uint32_t, int, uint32, uint, uint32) \ + TD(LONG, "long", int64_t, long, int64, long, int64) \ + TD(ULONG, "ulong", uint64_t, long, uint64, ulong, uint64) /* end int */ \ + TD(FLOAT, "float", float, float, float32, float, float32) /* begin float */ \ + TD(DOUBLE, "double", double, double, float64, double, float64) /* end float/scalar */ #define FLATBUFFERS_GEN_TYPES_POINTER(TD) \ - TD(STRING, "string", Offset, int, int, int) \ - TD(VECTOR, "", Offset, int, int, int) \ - TD(STRUCT, "", Offset, int, int, int) \ - TD(UNION, "", Offset, int, int, int) + TD(STRING, "string", Offset, int, int, int, int) \ + TD(VECTOR, "", Offset, int, int, int, int) \ + TD(STRUCT, "", Offset, int, int, int, int) \ + TD(UNION, "", Offset, int, int, int, int) // The fields are: // - enum @@ -61,12 +61,13 @@ namespace flatbuffers { // - Java type. // - Go type. // - C# / .Net type. +// - Python type. // using these macros, we can now write code dealing with types just once, e.g. /* switch (type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ case BASE_TYPE_ ## ENUM: \ // do something specific to CTYPE here FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -83,13 +84,13 @@ switch (type) { __extension__ // Stop GCC complaining about trailing comma with -Wpendantic. #endif enum BaseType { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ BASE_TYPE_ ## ENUM, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; -#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ +#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ static_assert(sizeof(CTYPE) <= sizeof(largest_scalar_t), \ "define largest_scalar_t as " #CTYPE); FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -464,6 +465,13 @@ extern bool GenerateJava(const Parser &parser, const std::string &file_name, const GeneratorOptions &opts); +// Generate Python files from the definitions in the Parser object. +// See idl_gen_python.cpp. +extern bool GeneratePython(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); + // Generate C# files from the definitions in the Parser object. // See idl_gen_csharp.cpp. extern bool GenerateCSharp(const Parser &parser, diff --git a/python/__init__.py b/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/flatbuffers/__init__.py b/python/flatbuffers/__init__.py new file mode 100644 index 000000000..d14872ae1 --- /dev/null +++ b/python/flatbuffers/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .builder import Builder +from .table import Table +from .compat import range_func as compat_range diff --git a/python/flatbuffers/builder.py b/python/flatbuffers/builder.py new file mode 100644 index 000000000..068c413f7 --- /dev/null +++ b/python/flatbuffers/builder.py @@ -0,0 +1,549 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import number_types as N +from .number_types import (UOffsetTFlags, SOffsetTFlags, VOffsetTFlags) + +from . import encode +from . import packer + +from . import compat +from .compat import range_func +from .compat import memoryview_type + + +class OffsetArithmeticError(RuntimeError): + """ + Error caused by an Offset arithmetic error. Probably caused by bad + writing of fields. This is considered an unreachable situation in + normal circumstances. + """ + pass + + +class NotInObjectError(RuntimeError): + """ + Error caused by using a Builder to write Object data when not inside + an Object. + """ + pass + + +class ObjectIsNestedError(RuntimeError): + """ + Error caused by using a Builder to begin an Object when an Object is + already being built. + """ + pass + + +class StructIsNotInlineError(RuntimeError): + """ + Error caused by using a Builder to write a Struct at a location that + is not the current Offset. + """ + pass + + +class BuilderSizeError(RuntimeError): + """ + Error caused by causing a Builder to exceed the hardcoded limit of 2 + gigabytes. + """ + pass + + +# VtableMetadataFields is the count of metadata fields in each vtable. +VtableMetadataFields = 2 + + +class Builder(object): + """ + A Builder is used to construct one or more FlatBuffers. Typically, Builder + objects will be used from code generated by the `flatc` compiler. + + A Builder constructs byte buffers in a last-first manner for simplicity and + performance during reading. + + Internally, a Builder is a state machine for creating FlatBuffer objects. + + It holds the following internal state: + Bytes: an array of bytes. + current_vtable: a list of integers. + vtables: a list of vtable entries (i.e. a list of list of integers). + """ + + __slots__ = ("Bytes", "current_vtable", "head", "minalign", "objectEnd", + "vtables") + + def __init__(self, initialSize): + """ + Initializes a Builder of size `initial_size`. + The internal buffer is grown as needed. + """ + + if not (0 <= initialSize < (2**UOffsetTFlags.bytewidth - 1)): + msg = "flatbuffers: Cannot create Builder larger than 2 gigabytes." + raise BuilderSizeError(msg) + + self.Bytes = bytearray(initialSize) + self.current_vtable = None + self.head = UOffsetTFlags.py_type(initialSize) + self.minalign = 1 + self.objectEnd = None + self.vtables = [] + + def Output(self): + """ + Output returns the portion of the buffer that has been used for + writing data. + """ + + return self.Bytes[self.Head():] + + def StartObject(self, numfields): + """StartObject initializes bookkeeping for writing a new object.""" + + self.assertNotNested() + + # use 32-bit offsets so that arithmetic doesn't overflow. + self.current_vtable = [0 for _ in range_func(numfields)] + self.objectEnd = self.Offset() + self.minalign = 1 + + def WriteVtable(self): + """ + WriteVtable serializes the vtable for the current object, if needed. + + Before writing out the vtable, this checks pre-existing vtables for + equality to this one. If an equal vtable is found, point the object to + the existing vtable and return. + + Because vtable values are sensitive to alignment of object data, not + all logically-equal vtables will be deduplicated. + + A vtable has the following format: + + + * N, where N is the number of fields + in the schema for this type. Includes deprecated fields. + Thus, a vtable is made of 2 + N elements, each VOffsetT bytes wide. + + An object has the following format: + + + + """ + + # Prepend a zero scalar to the object. Later in this function we'll + # write an offset here that points to the object's vtable: + self.PrependSOffsetTRelative(0) + + objectOffset = self.Offset() + existingVtable = None + + # Search backwards through existing vtables, because similar vtables + # are likely to have been recently appended. See + # BenchmarkVtableDeduplication for a case in which this heuristic + # saves about 30% of the time used in writing objects with duplicate + # tables. + + i = len(self.vtables) - 1 + while i >= 0: + # Find the other vtable, which is associated with `i`: + vt2Offset = self.vtables[i] + vt2Start = len(self.Bytes) - vt2Offset + vt2Len = encode.Get(packer.voffset, self.Bytes, vt2Start) + + metadata = VtableMetadataFields * N.VOffsetTFlags.bytewidth + vt2End = vt2Start + vt2Len + vt2 = self.Bytes[vt2Start+metadata:vt2End] + + # Compare the other vtable to the one under consideration. + # If they are equal, store the offset and break: + if vtableEqual(self.current_vtable, objectOffset, vt2): + existingVtable = vt2Offset + break + + i -= 1 + + if existingVtable is None: + # Did not find a vtable, so write this one to the buffer. + + # Write out the current vtable in reverse , because + # serialization occurs in last-first order: + i = len(self.current_vtable) - 1 + while i >= 0: + off = 0 + if self.current_vtable[i] != 0: + # Forward reference to field; + # use 32bit number to ensure no overflow: + off = objectOffset - self.current_vtable[i] + + self.PrependVOffsetT(off) + i -= 1 + + # The two metadata fields are written last. + + # First, store the object bytesize: + objectSize = UOffsetTFlags.py_type(objectOffset - self.objectEnd) + self.PrependVOffsetT(VOffsetTFlags.py_type(objectSize)) + + # Second, store the vtable bytesize: + vBytes = len(self.current_vtable) + VtableMetadataFields + vBytes *= N.VOffsetTFlags.bytewidth + self.PrependVOffsetT(VOffsetTFlags.py_type(vBytes)) + + # Next, write the offset to the new vtable in the + # already-allocated SOffsetT at the beginning of this object: + objectStart = SOffsetTFlags.py_type(len(self.Bytes) - objectOffset) + encode.Write(packer.soffset, self.Bytes, objectStart, + SOffsetTFlags.py_type(self.Offset() - objectOffset)) + + # Finally, store this vtable in memory for future + # deduplication: + self.vtables.append(self.Offset()) + else: + # Found a duplicate vtable. + + objectStart = SOffsetTFlags.py_type(len(self.Bytes) - objectOffset) + self.head = UOffsetTFlags.py_type(objectStart) + + # Write the offset to the found vtable in the + # already-allocated SOffsetT at the beginning of this object: + encode.Write(packer.soffset, self.Bytes, self.Head(), + SOffsetTFlags.py_type(existingVtable - objectOffset)) + + self.current_vtable = None + return objectOffset + + def EndObject(self): + """EndObject writes data necessary to finish object construction.""" + if self.current_vtable is None: + msg = ("flatbuffers: Tried to write the end of an Object when " + "the Builder was not currently writing an Object.") + raise NotInObjectError(msg) + return self.WriteVtable() + + def growByteBuffer(self): + """Doubles the size of the byteslice, and copies the old data towards + the end of the new buffer (since we build the buffer backwards).""" + if not len(self.Bytes) <= 2**20: + msg = "flatbuffers: cannot grow buffer beyond 2 gigabytes" + raise BuilderSizeError(msg) + + newSize = len(self.Bytes) * 2 + if newSize == 0: + newSize = 1 + bytes2 = bytearray(newSize) + bytes2[newSize-len(self.Bytes):] = self.Bytes + self.Bytes = bytes2 + + def Head(self): + """ + Head gives the start of useful data in the underlying byte buffer. + Note: unlike other functions, this value is interpreted as from the left. + """ + return self.head + + def Offset(self): + """Offset relative to the end of the buffer.""" + return UOffsetTFlags.py_type(len(self.Bytes) - self.Head()) + + def Pad(self, n): + """Pad places zeros at the current offset.""" + for i in range_func(n): + self.Place(0, N.Uint8Flags) + + def Prep(self, size, additionalBytes): + """ + Prep prepares to write an element of `size` after `additional_bytes` + have been written, e.g. if you write a string, you need to align + such the int length field is aligned to SizeInt32, and the string + data follows it directly. + If all you need to do is align, `additionalBytes` will be 0. + """ + + # Track the biggest thing we've ever aligned to. + if size > self.minalign: + self.minalign = size + + # Find the amount of alignment needed such that `size` is properly + # aligned after `additionalBytes`: + alignSize = (~(len(self.Bytes) - self.Head() + additionalBytes)) + 1 + alignSize &= (size - 1) + + # Reallocate the buffer if needed: + while self.Head() < alignSize+size+additionalBytes: + oldBufSize = len(self.Bytes) + self.growByteBuffer() + updated_head = self.head + len(self.Bytes) - oldBufSize + self.head = UOffsetTFlags.py_type(updated_head) + self.Pad(alignSize) + + def PrependSOffsetTRelative(self, off): + """ + PrependSOffsetTRelative prepends an SOffsetT, relative to where it + will be written. + """ + + # Ensure alignment is already done: + self.Prep(N.SOffsetTFlags.bytewidth, 0) + if not (off <= self.Offset()): + msg = "flatbuffers: Offset arithmetic error." + raise OffsetArithmeticError(msg) + off2 = self.Offset() - off + N.SOffsetTFlags.bytewidth + self.PlaceSOffsetT(off2) + + def PrependUOffsetTRelative(self, off): + """ + PrependUOffsetTRelative prepends an UOffsetT, relative to where it + will be written. + """ + + # Ensure alignment is already done: + self.Prep(N.UOffsetTFlags.bytewidth, 0) + if not (off <= self.Offset()): + msg = "flatbuffers: Offset arithmetic error." + raise OffsetArithmeticError(msg) + off2 = self.Offset() - off + N.UOffsetTFlags.bytewidth + self.PlaceUOffsetT(off2) + + def StartVector(self, elemSize, numElems, alignment): + """ + StartVector initializes bookkeeping for writing a new vector. + + A vector has the following format: + + +, where T is the type of elements of this vector. + """ + + self.assertNotNested() + self.Prep(N.Uint32Flags.bytewidth, elemSize*numElems) + self.Prep(alignment, elemSize*numElems) # In case alignment > int. + return self.Offset() + + def EndVector(self, vectorNumElems): + """EndVector writes data necessary to finish vector construction.""" + + # we already made space for this, so write without PrependUint32 + self.PlaceUOffsetT(vectorNumElems) + return self.Offset() + + def CreateString(self, s): + """CreateString writes a null-terminated byte string as a vector.""" + + if isinstance(s, compat.string_types): + x = s.encode() + elif isinstance(s, compat.binary_type): + x = s + else: + raise TypeError("non-string passed to CreateString") + + self.Prep(N.UOffsetTFlags.bytewidth, (len(x)+1)*N.Uint8Flags.bytewidth) + self.Place(0, N.Uint8Flags) + + l = UOffsetTFlags.py_type(len(s)) + + self.head = UOffsetTFlags.py_type(self.Head() - l) + self.Bytes[self.Head():self.Head()+l] = x + + return self.EndVector(len(x)) + + def assertNotNested(self): + """ + Check that no other objects are being built while making this + object. If not, raise an exception. + """ + + if self.current_vtable is not None: + msg = ("flatbuffers: Tried to write a new Object when the " + "Builder was already writing an Object.") + raise ObjectIsNestedError(msg) + + def assertNested(self, obj): + """ + Structs are always stored inline, so need to be created right + where they are used. You'll get this error if you created it + elsewhere. + """ + + N.enforce_number(obj, N.UOffsetTFlags) + if obj != self.Offset(): + msg = ("flatbuffers: Tried to write a Struct at an Offset that " + "is different from the current Offset of the Builder.") + raise StructIsNotInlineError(msg) + + def Slot(self, slotnum): + """ + Slot sets the vtable key `voffset` to the current location in the + buffer. + + """ + if self.current_vtable is None: + msg = ("flatbuffers: Tried to write an Object field when " + "the Builder was not currently writing an Object.") + raise NotInObjectError(msg) + + self.current_vtable[slotnum] = self.Offset() + + def Finish(self, rootTable): + """Finish finalizes a buffer, pointing to the given `rootTable`.""" + N.enforce_number(rootTable, N.UOffsetTFlags) + self.Prep(self.minalign, N.UOffsetTFlags.bytewidth) + self.PrependUOffsetTRelative(rootTable) + return self.Head() + + def Prepend(self, flags, off): + self.Prep(flags.bytewidth, 0) + self.Place(off, flags) + + def PrependSlot(self, flags, o, x, d): + N.enforce_number(x, flags) + N.enforce_number(d, flags) + if x != d: + self.Prepend(flags, x) + self.Slot(o) + + def PrependBoolSlot(self, *args): self.PrependSlot(N.BoolFlags, *args) + + def PrependByteSlot(self, *args): self.PrependSlot(N.Uint8Flags, *args) + + def PrependUint8Slot(self, *args): self.PrependSlot(N.Uint8Flags, *args) + + def PrependUint16Slot(self, *args): self.PrependSlot(N.Uint16Flags, *args) + + def PrependUint32Slot(self, *args): self.PrependSlot(N.Uint32Flags, *args) + + def PrependUint64Slot(self, *args): self.PrependSlot(N.Uint64Flags, *args) + + def PrependInt8Slot(self, *args): self.PrependSlot(N.Int8Flags, *args) + + def PrependInt16Slot(self, *args): self.PrependSlot(N.Int16Flags, *args) + + def PrependInt32Slot(self, *args): self.PrependSlot(N.Int32Flags, *args) + + def PrependInt64Slot(self, *args): self.PrependSlot(N.Int64Flags, *args) + + def PrependFloat32Slot(self, *args): self.PrependSlot(N.Float32Flags, + *args) + + def PrependFloat64Slot(self, *args): self.PrependSlot(N.Float64Flags, + *args) + + def PrependUOffsetTRelativeSlot(self, o, x, d): + """ + PrependUOffsetTRelativeSlot prepends an UOffsetT onto the object at + vtable slot `o`. If value `x` equals default `d`, then the slot will + be set to zero and no other data will be written. + """ + + if x != d: + self.PrependUOffsetTRelative(x) + self.Slot(o) + + def PrependStructSlot(self, v, x, d): + """ + PrependStructSlot prepends a struct onto the object at vtable slot `o`. + Structs are stored inline, so nothing additional is being added. + In generated code, `d` is always 0. + """ + + N.enforce_number(d, N.UOffsetTFlags) + if x != d: + self.assertNested(x) + self.Slot(v) + + def PrependBool(self, x): self.Prepend(N.BoolFlags, x) + + def PrependByte(self, x): self.Prepend(N.Uint8Flags, x) + + def PrependUint8(self, x): self.Prepend(N.Uint8Flags, x) + + def PrependUint16(self, x): self.Prepend(N.Uint16Flags, x) + + def PrependUint32(self, x): self.Prepend(N.Uint32Flags, x) + + def PrependUint64(self, x): self.Prepend(N.Uint64Flags, x) + + def PrependInt8(self, x): self.Prepend(N.Int8Flags, x) + + def PrependInt16(self, x): self.Prepend(N.Int16Flags, x) + + def PrependInt32(self, x): self.Prepend(N.Int32Flags, x) + + def PrependInt64(self, x): self.Prepend(N.Int64Flags, x) + + def PrependFloat32(self, x): self.Prepend(N.Float32Flags, x) + + def PrependFloat64(self, x): self.Prepend(N.Float64Flags, x) + + def PrependVOffsetT(self, x): self.Prepend(N.VOffsetTFlags, x) + + def Place(self, x, flags): + """ + Place prepends a value specified by `flags` to the Builder, + without checking for available space. + """ + + N.enforce_number(x, flags) + self.head = self.head - flags.bytewidth + encode.Write(flags.packer_type, self.Bytes, self.Head(), x) + + def PlaceVOffsetT(self, x): + """ + PlaceVOffsetT prepends a VOffsetT to the Builder, without checking for + space. + """ + N.enforce_number(x, N.VOffsetTFlags) + self.head = self.head - N.VOffsetTFlags.bytewidth + encode.Write(packer.voffset, self.Bytes, self.Head(), x) + + def PlaceSOffsetT(self, x): + """ + PlaceSOffsetT prepends a SOffsetT to the Builder, without checking for + space. + """ + N.enforce_number(x, N.SOffsetTFlags) + self.head = self.head - N.SOffsetTFlags.bytewidth + encode.Write(packer.soffset, self.Bytes, self.Head(), x) + + def PlaceUOffsetT(self, x): + """ + PlaceUOffsetT prepends a UOffsetT to the Builder, without checking for + space. + """ + N.enforce_number(x, N.UOffsetTFlags) + self.head = self.head - N.UOffsetTFlags.bytewidth + encode.Write(packer.uoffset, self.Bytes, self.Head(), x) + + +def vtableEqual(a, objectStart, b): + """vtableEqual compares an unwritten vtable to a written vtable.""" + + N.enforce_number(objectStart, N.UOffsetTFlags) + + if len(a) * N.VOffsetTFlags.bytewidth != len(b): + return False + + for i, elem in enumerate(a): + x = encode.Get(packer.voffset, b, i * N.VOffsetTFlags.bytewidth) + + # Skip vtable entries that indicate a default value. + if x == 0 and elem == 0: + pass + else: + y = objectStart - elem + if x != y: + return False + return True diff --git a/python/flatbuffers/compat.py b/python/flatbuffers/compat.py new file mode 100644 index 000000000..30c504d5e --- /dev/null +++ b/python/flatbuffers/compat.py @@ -0,0 +1,27 @@ +""" A tiny version of `six` to help with backwards compability. """ + +import sys + +PY2 = sys.version_info[0] == 2 +PY26 = sys.version_info[0:2] == (2, 6) +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = (str,) + binary_type = bytes + range_func = range + memoryview_type = memoryview + struct_bool_decl = "?" +else: + string_types = (basestring,) + binary_type = str + range_func = xrange + if PY26: + memoryview_type = buffer + struct_bool_decl = " + +#include "flatbuffers/flatbuffers.h" +#include "flatbuffers/idl.h" +#include "flatbuffers/util.h" + +namespace flatbuffers { +namespace python { + +static std::string GenGetter(const Type &type); +static std::string GenMethod(const FieldDef &field); +static void GenStructBuilder(const StructDef &struct_def, + std::string *code_ptr); +static void GenReceiver(const StructDef &struct_def, std::string *code_ptr); +static std::string GenTypeBasic(const Type &type); +static std::string GenTypeGet(const Type &type); +static std::string TypeName(const FieldDef &field); + + +// Hardcode spaces per indentation. +const std::string Indent = " "; + +// Most field accessors need to retrieve and test the field offset first, +// this is the prefix code for that. +std::string OffsetPrefix(const FieldDef &field) { + return "\n" + Indent + Indent + + "o = flatbuffers.number_types.UOffsetTFlags.py_type" + + "(self._tab.Offset(" + + NumToString(field.value.offset) + + "))\n" + Indent + Indent + "if o != 0:\n"; +} + +// Begin by declaring namespace and imports. +static void BeginFile(const std::string name_space_name, + const bool needs_imports, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "# automatically generated, do not modify\n\n"; + code += "# namespace: " + name_space_name + "\n\n"; + if (needs_imports) { + code += "import flatbuffers\n\n"; + } +} + +// Begin a class declaration. +static void BeginClass(const StructDef &struct_def, std::string *code_ptr) { + std::string &code = *code_ptr; + code += "class " + struct_def.name + "(object):\n"; + code += Indent + "__slots__ = ['_tab']"; + code += "\n\n"; +} + +// Begin enum code with a class declaration. +static void BeginEnum(const std::string class_name, std::string *code_ptr) { + std::string &code = *code_ptr; + code += "class " + class_name + "(object):\n"; +} + +// A single enum member. +static void EnumMember(const EnumVal ev, std::string *code_ptr) { + std::string &code = *code_ptr; + code += Indent; + code += ev.name; + code += " = "; + code += NumToString(ev.value) + "\n"; +} + +// End enum code. +static void EndEnum(std::string *code_ptr) { + std::string &code = *code_ptr; + code += "\n"; +} + +// Initialize a new struct or table from existing data. +static void NewRootTypeFromBuffer(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + code += Indent + "@classmethod\n"; + code += Indent + "def GetRootAs"; + code += struct_def.name; + code += "(cls, buf, offset):"; + code += "\n"; + code += Indent + Indent; + code += "n = flatbuffers.encode.Get"; + code += "(flatbuffers.packer.uoffset, buf, offset)\n"; + code += Indent + Indent + "x = " + struct_def.name + "()\n"; + code += Indent + Indent + "x.Init(buf, n + offset)\n"; + code += Indent + Indent + "return x\n"; + code += "\n\n"; +} + +// Initialize an existing object with other data, to avoid an allocation. +static void InitializeExisting(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += "Init(self, buf, pos):\n"; + code += Indent + Indent + "self._tab = flatbuffers.table.Table(buf, pos)\n"; + code += "\n"; +} + +// Get the length of a vector. +static void GetVectorLen(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name) + "Length(self"; + code += "):" + OffsetPrefix(field); + code += Indent + Indent + Indent + "return self._tab.VectorLen(o)\n"; + code += Indent + Indent + "return 0\n\n"; +} + +// Get the value of a struct's scalar. +static void GetScalarFieldOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + std::string getter = GenGetter(field.value.type); + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name); + code += "(self): return " + getter; + code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; + code += NumToString(field.value.offset) + "))\n"; +} + +// Get the value of a table's scalar. +static void GetScalarFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + std::string getter = GenGetter(field.value.type); + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name); + code += "(self):"; + code += OffsetPrefix(field); + code += Indent + Indent + Indent + "return " + getter; + code += "o + self._tab.Pos)\n"; + code += Indent + Indent + "return " + field.value.constant + "\n\n"; +} + +// Get a struct by initializing an existing struct. +// Specific to Struct. +static void GetStructFieldOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name); + code += "(self, obj):\n"; + code += Indent + Indent + "obj.Init(self._tab.Bytes, self._tab.Pos + "; + code += NumToString(field.value.offset) + ")"; + code += "\n" + Indent + Indent + "return obj\n\n"; +} + +// Get a struct by initializing an existing struct. +// Specific to Table. +static void GetStructFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name); + code += "(self):"; + code += OffsetPrefix(field); + if (field.value.type.struct_def->fixed) { + code += Indent + Indent + Indent + "x = o + self._tab.Pos\n"; + } else { + code += Indent + Indent + Indent; + code += "x = self._tab.Indirect(o + self._tab.Pos)\n"; + } + code += Indent + Indent + Indent; + code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; + code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; + code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; + code += Indent + Indent + Indent + "return obj\n"; + code += Indent + Indent + "return None\n\n"; +} + +// Get the value of a string. +static void GetStringField(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name); + code += "(self):"; + code += OffsetPrefix(field); + code += Indent + Indent + Indent + "return " + GenGetter(field.value.type); + code += "o + self._tab.Pos)\n"; + code += Indent + Indent + "return \"\"\n\n"; +} + +// Get the value of a union from an object. +static void GetUnionField(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name) + "(self):"; + code += OffsetPrefix(field); + + // TODO(rw): this works and is not the good way to it: + bool is_native_table = TypeName(field) == "*flatbuffers.Table"; + if (is_native_table) { + code += Indent + Indent + Indent + "from flatbuffers.table import Table\n"; + } else { + code += Indent + Indent + Indent; + code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; + } + code += Indent + Indent + Indent + "obj = Table(bytearray(), 0)\n"; + code += Indent + Indent + Indent + GenGetter(field.value.type); + code += "obj, o)\n" + Indent + Indent + Indent + "return obj\n"; + code += Indent + Indent + "return None\n\n"; +} + +// Get the value of a vector's struct member. +static void GetMemberOfVectorOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name); + code += "(self, j):" + OffsetPrefix(field); + code += Indent + Indent + Indent + "x = self._tab.Vector(o)\n"; + code += Indent + Indent + Indent; + code += "x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * "; + code += NumToString(InlineSize(vectortype)) + "\n"; + if (!(vectortype.struct_def->fixed)) { + code += Indent + Indent + Indent + "x = self._tab.Indirect(x)\n"; + } + code += Indent + Indent + Indent; + code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; + code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; + code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; + code += Indent + Indent + Indent + "return obj\n"; + code += Indent + Indent + "return None\n\n"; +} + +// Get the value of a vector's non-struct member. Uses a named return +// argument to conveniently set the zero value for the result. +static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + + GenReceiver(struct_def, code_ptr); + code += MakeCamel(field.name); + code += "(self, j):"; + code += OffsetPrefix(field); + code += Indent + Indent + Indent + "a = self._tab.Vector(o)\n"; + code += Indent + Indent + Indent; + code += "return " + GenGetter(field.value.type); + code += "a + flatbuffers.number_types.UOffsetTFlags.py_type(j * "; + code += NumToString(InlineSize(vectortype)) + "))\n"; + if (vectortype.base_type == BASE_TYPE_STRING) { + code += Indent + Indent + "return \"\"\n"; + } else { + code += Indent + Indent + "return 0\n"; + } + code += "\n"; +} + +// Begin the creator function signature. +static void BeginBuilderArgs(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + code += "\n"; + code += "def Create" + struct_def.name; + code += "(builder"; +} + +// Recursively generate arguments for a constructor, to deal with nested +// structs. +static void StructBuilderArgs(const StructDef &struct_def, + const char *nameprefix, + std::string *code_ptr) { + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); + ++it) { + auto &field = **it; + if (IsStruct(field.value.type)) { + // Generate arguments for a struct inside a struct. To ensure names + // don't clash, and to make it obvious these arguments are constructing + // a nested struct, prefix the name with the struct name. + StructBuilderArgs(*field.value.type.struct_def, + (field.value.type.struct_def->name + "_").c_str(), + code_ptr); + } else { + std::string &code = *code_ptr; + code += (std::string)", " + nameprefix; + code += MakeCamel(field.name, false); + } + } +} + +// End the creator function signature. +static void EndBuilderArgs(std::string *code_ptr) { + std::string &code = *code_ptr; + code += "):\n"; +} + +// Recursively generate struct construction statements and instert manual +// padding. +static void StructBuilderBody(const StructDef &struct_def, + const char *nameprefix, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += " builder.Prep(" + NumToString(struct_def.minalign) + ", "; + code += NumToString(struct_def.bytesize) + ")\n"; + for (auto it = struct_def.fields.vec.rbegin(); + it != struct_def.fields.vec.rend(); + ++it) { + auto &field = **it; + if (field.padding) + code += " builder.Pad(" + NumToString(field.padding) + ")\n"; + if (IsStruct(field.value.type)) { + StructBuilderBody(*field.value.type.struct_def, + (field.value.type.struct_def->name + "_").c_str(), + code_ptr); + } else { + code += " builder.Prepend" + GenMethod(field) + "("; + code += nameprefix + MakeCamel(field.name, false) + ")\n"; + } + } +} + +static void EndBuilderBody(std::string *code_ptr) { + std::string &code = *code_ptr; + code += " return builder.Offset()\n"; +} + +// Get the value of a table's starting offset. +static void GetStartOfTable(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + struct_def.name + "Start"; + code += "(builder): "; + code += "builder.StartObject("; + code += NumToString(struct_def.fields.vec.size()); + code += ")\n"; +} + +// Set the value of a table's field. +static void BuildFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + const size_t offset, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + struct_def.name + "Add" + MakeCamel(field.name); + code += "(builder, "; + code += MakeCamel(field.name, false); + code += "): "; + code += "builder.Prepend"; + code += GenMethod(field) + "Slot("; + code += NumToString(offset) + ", "; + if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { + code += "flatbuffers.number_types.UOffsetTFlags.py_type"; + code += "("; + code += MakeCamel(field.name, false) + ")"; + } else { + code += MakeCamel(field.name, false); + } + code += ", " + field.value.constant; + code += ")\n"; +} + +// Set the value of one of the members of a table's vector. +static void BuildVectorOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + struct_def.name + "Start"; + code += MakeCamel(field.name); + code += "Vector(builder, numElems): return builder.StartVector("; + auto vector_type = field.value.type.VectorType(); + auto alignment = InlineAlignment(vector_type); + auto elem_size = InlineSize(vector_type); + code += NumToString(elem_size); + code += ", numElems, " + NumToString(alignment); + code += ")\n"; +} + +// Get the offset of the end of a table. +static void GetEndOffsetOnTable(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + struct_def.name + "End"; + code += "(builder): "; + code += "return builder.EndObject()\n"; +} + +// Generate the receiver for function signatures. +static void GenReceiver(const StructDef &struct_def, std::string *code_ptr) { + std::string &code = *code_ptr; + code += Indent + "# " + struct_def.name + "\n"; + code += Indent + "def "; +} + +// Generate a struct field, conditioned on its child type(s). +static void GenStructAccessor(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + GenComment(field.doc_comment, code_ptr, nullptr, "# "); + if (IsScalar(field.value.type.base_type)) { + if (struct_def.fixed) { + GetScalarFieldOfStruct(struct_def, field, code_ptr); + } else { + GetScalarFieldOfTable(struct_def, field, code_ptr); + } + } else { + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: + if (struct_def.fixed) { + GetStructFieldOfStruct(struct_def, field, code_ptr); + } else { + GetStructFieldOfTable(struct_def, field, code_ptr); + } + break; + case BASE_TYPE_STRING: + GetStringField(struct_def, field, code_ptr); + break; + case BASE_TYPE_VECTOR: { + auto vectortype = field.value.type.VectorType(); + if (vectortype.base_type == BASE_TYPE_STRUCT) { + GetMemberOfVectorOfStruct(struct_def, field, code_ptr); + } else { + GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr); + } + break; + } + case BASE_TYPE_UNION: + GetUnionField(struct_def, field, code_ptr); + break; + default: + assert(0); + } + } + if (field.value.type.base_type == BASE_TYPE_VECTOR) { + GetVectorLen(struct_def, field, code_ptr); + } +} + +// Generate table constructors, conditioned on its members' types. +static void GenTableBuilders(const StructDef &struct_def, + std::string *code_ptr) { + GetStartOfTable(struct_def, code_ptr); + + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); + ++it) { + auto &field = **it; + if (field.deprecated) continue; + + auto offset = it - struct_def.fields.vec.begin(); + BuildFieldOfTable(struct_def, field, offset, code_ptr); + if (field.value.type.base_type == BASE_TYPE_VECTOR) { + BuildVectorOfTable(struct_def, field, code_ptr); + } + } + + GetEndOffsetOnTable(struct_def, code_ptr); +} + +// Generate struct or table methods. +static void GenStruct(const StructDef &struct_def, + std::string *code_ptr, + StructDef *root_struct_def) { + if (struct_def.generated) return; + + GenComment(struct_def.doc_comment, code_ptr, nullptr); + BeginClass(struct_def, code_ptr); + if (&struct_def == root_struct_def) { + // Generate a special accessor for the table that has been declared as + // the root type. + NewRootTypeFromBuffer(struct_def, code_ptr); + } + // Generate the Init method that sets the field in a pre-existing + // accessor object. This is to allow object reuse. + InitializeExisting(struct_def, code_ptr); + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); + ++it) { + auto &field = **it; + if (field.deprecated) continue; + + GenStructAccessor(struct_def, field, code_ptr); + } + + if (struct_def.fixed) { + // create a struct constructor function + GenStructBuilder(struct_def, code_ptr); + } else { + // Create a set of functions that allow table construction. + GenTableBuilders(struct_def, code_ptr); + } +} + +// Generate enum declarations. +static void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { + if (enum_def.generated) return; + + GenComment(enum_def.doc_comment, code_ptr, nullptr, "# "); + BeginEnum(enum_def.name, code_ptr); + for (auto it = enum_def.vals.vec.begin(); + it != enum_def.vals.vec.end(); + ++it) { + auto &ev = **it; + GenComment(ev.doc_comment, code_ptr, nullptr, "# "); + EnumMember(ev, code_ptr); + } + EndEnum(code_ptr); +} + +// Returns the function name that is able to read a value of the given type. +static std::string GenGetter(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_STRING: return "self._tab.String("; + case BASE_TYPE_UNION: return "self._tab.Union("; + case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); + default: + return "self._tab.Get(flatbuffers.number_types." + \ + MakeCamel(GenTypeGet(type)) + \ + "Flags, "; + } +} + +// Returns the method name for use with add/put calls. +static std::string GenMethod(const FieldDef &field) { + return IsScalar(field.value.type.base_type) + ? MakeCamel(GenTypeBasic(field.value.type)) + : (IsStruct(field.value.type) ? "Struct" : "UOffsetTRelative"); +} + + +// Save out the generated code for a Python Table type. +static bool SaveType(const Parser &parser, const Definition &def, + const std::string &classcode, const std::string &path, + bool needs_imports) { + if (!classcode.length()) return true; + + std::string namespace_name; + std::string namespace_dir = path; + auto &namespaces = parser.namespaces_.back()->components; + for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { + if (namespace_name.length()) { + namespace_name += "."; + namespace_dir += kPathSeparator; + } + namespace_name = *it; + namespace_dir += *it; + mkdir(namespace_dir.c_str(), S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH); + + std::string init_py_filename = namespace_dir + "/__init__.py"; + SaveFile(init_py_filename.c_str(), "", false); + } + + + std::string code = ""; + BeginFile(namespace_name, needs_imports, &code); + code += classcode; + std::string filename = namespace_dir + kPathSeparator + def.name + ".py"; + return SaveFile(filename.c_str(), code, false); +} + +static std::string GenTypeBasic(const Type &type) { + static const char *ctypename[] = { + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ + #PTYPE, + FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) + #undef FLATBUFFERS_TD + }; + return ctypename[type.base_type]; +} + +static std::string GenTypePointer(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_STRING: + return "string"; + case BASE_TYPE_VECTOR: + return GenTypeGet(type.VectorType()); + case BASE_TYPE_STRUCT: + return type.struct_def->name; + case BASE_TYPE_UNION: + // fall through + default: + return "*flatbuffers.Table"; + } +} + +static std::string GenTypeGet(const Type &type) { + return IsScalar(type.base_type) + ? GenTypeBasic(type) + : GenTypePointer(type); +} + +static std::string TypeName(const FieldDef &field) { + return GenTypeGet(field.value.type); +} + +// Create a struct with a builder and the struct's arguments. +static void GenStructBuilder(const StructDef &struct_def, + std::string *code_ptr) { + BeginBuilderArgs(struct_def, code_ptr); + StructBuilderArgs(struct_def, "", code_ptr); + EndBuilderArgs(code_ptr); + + StructBuilderBody(struct_def, "", code_ptr); + EndBuilderBody(code_ptr); +} + +} // namespace python + +bool GeneratePython(const Parser &parser, + const std::string &path, + const std::string & /*file_name*/, + const GeneratorOptions & /*opts*/) { + for (auto it = parser.enums_.vec.begin(); + it != parser.enums_.vec.end(); ++it) { + std::string enumcode; + python::GenEnum(**it, &enumcode); + if (!python::SaveType(parser, **it, enumcode, path, false)) + return false; + } + + for (auto it = parser.structs_.vec.begin(); + it != parser.structs_.vec.end(); ++it) { + std::string declcode; + python::GenStruct(**it, &declcode, parser.root_struct_def); + if (!python::SaveType(parser, **it, declcode, path, true)) + return false; + } + + return true; +} + +} // namespace flatbuffers + + diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index 71be2be93..4fb8f0923 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -159,7 +159,8 @@ template<> void Print(const void *val, type = type.VectorType(); // Call PrintVector above specifically for each element type: switch (type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ + PTYPE) \ case BASE_TYPE_ ## ENUM: \ PrintVector( \ *reinterpret_cast *>(val), \ @@ -225,7 +226,8 @@ static void GenStruct(const StructDef &struct_def, const Table *table, OutputIdentifier(fd.name, opts, _text); text += ": "; switch (fd.value.type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ + PTYPE) \ case BASE_TYPE_ ## ENUM: \ GenField(fd, table, struct_def.fixed, \ opts, indent + Indent(opts), _text); \ @@ -233,7 +235,8 @@ static void GenStruct(const StructDef &struct_def, const Table *table, FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) #undef FLATBUFFERS_TD // Generate drop-thru case statements for all pointer types: - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ + PTYPE) \ case BASE_TYPE_ ## ENUM: FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 158ffcaf0..1e2e952ac 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -25,14 +25,15 @@ namespace flatbuffers { const char *const kTypeNames[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) IDLTYPE, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ + IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD nullptr }; const char kTypeSizes[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ sizeof(CTYPE), FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -96,7 +97,7 @@ enum { #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE, FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) #undef FLATBUFFERS_TOKEN - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ kToken ## ENUM, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -107,7 +108,8 @@ static std::string TokenToString(int t) { #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING, FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) #undef FLATBUFFERS_TOKEN - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) IDLTYPE, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ + IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; @@ -205,7 +207,8 @@ void Parser::Next() { attribute_.clear(); attribute_.append(start, cursor_); // First, see if it is a type keyword from the table of types: - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ + PTYPE) \ if (attribute_ == IDLTYPE) { \ token_ = kToken ## ENUM; \ return; \ @@ -580,7 +583,8 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def) { auto field = it->second; if (!struct_def.sortbysize || size == SizeOf(value.type.base_type)) { switch (value.type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ + PTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ if (struct_def.fixed) { \ @@ -593,7 +597,8 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def) { break; FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); #undef FLATBUFFERS_TD - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ + PTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ if (IsStruct(field->value.type)) { \ @@ -648,7 +653,7 @@ uoffset_t Parser::ParseVector(const Type &type) { // start at the back, since we're building the data backwards. auto &val = field_stack_.back().first; switch (val.type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ case BASE_TYPE_ ## ENUM: \ if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ else builder_.PushElement(atot(val.constant.c_str())); \ diff --git a/tests/MyGame/Example/Any.py b/tests/MyGame/Example/Any.py new file mode 100644 index 000000000..c88362ece --- /dev/null +++ b/tests/MyGame/Example/Any.py @@ -0,0 +1,8 @@ +# automatically generated, do not modify + +# namespace: Example + +class Any(object): + NONE = 0 + Monster = 1 + diff --git a/tests/MyGame/Example/Color.py b/tests/MyGame/Example/Color.py new file mode 100644 index 000000000..18b147dce --- /dev/null +++ b/tests/MyGame/Example/Color.py @@ -0,0 +1,9 @@ +# automatically generated, do not modify + +# namespace: Example + +class Color(object): + Red = 1 + Green = 2 + Blue = 8 + diff --git a/tests/MyGame/Example/Monster.py b/tests/MyGame/Example/Monster.py new file mode 100644 index 000000000..2f0dc35ed --- /dev/null +++ b/tests/MyGame/Example/Monster.py @@ -0,0 +1,278 @@ +# automatically generated, do not modify + +# namespace: Example + +import flatbuffers + +class Monster(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsMonster(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Monster() + x.Init(buf, n + offset) + return x + + + # Monster + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Monster + def Pos(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + x = o + self._tab.Pos + from .Vec3 import Vec3 + obj = Vec3() + obj.Init(self._tab.Bytes, x) + return obj + return None + + # Monster + def Mana(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int16Flags, o + self._tab.Pos) + return 150 + + # Monster + def Hp(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int16Flags, o + self._tab.Pos) + return 100 + + # Monster + def Name(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) + if o != 0: + return self._tab.String(o + self._tab.Pos) + return "" + + # Monster + def Inventory(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) + return 0 + + # Monster + def InventoryLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # Monster + def Color(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int8Flags, o + self._tab.Pos) + return 8 + + # Monster + def TestType(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(18)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return 0 + + # Monster + def Test(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(20)) + if o != 0: + from flatbuffers.table import Table + obj = Table(bytearray(), 0) + self._tab.Union(obj, o) + return obj + return None + + # Monster + def Test4(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(22)) + if o != 0: + x = self._tab.Vector(o) + x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 + from .Test import Test + obj = Test() + obj.Init(self._tab.Bytes, x) + return obj + return None + + # Monster + def Test4Length(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(22)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # Monster + def Testarrayofstring(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(24)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return "" + + # Monster + def TestarrayofstringLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(24)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +# /// an example documentation comment: this will end up in the generated code +# /// multiline too + # Monster + def Testarrayoftables(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(26)) + if o != 0: + x = self._tab.Vector(o) + x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 + x = self._tab.Indirect(x) + from .Monster import Monster + obj = Monster() + obj.Init(self._tab.Bytes, x) + return obj + return None + + # Monster + def TestarrayoftablesLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(26)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # Monster + def Enemy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(28)) + if o != 0: + x = self._tab.Indirect(o + self._tab.Pos) + from .Monster import Monster + obj = Monster() + obj.Init(self._tab.Bytes, x) + return obj + return None + + # Monster + def Testnestedflatbuffer(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) + return 0 + + # Monster + def TestnestedflatbufferLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # Monster + def Testempty(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(32)) + if o != 0: + x = self._tab.Indirect(o + self._tab.Pos) + from .Stat import Stat + obj = Stat() + obj.Init(self._tab.Bytes, x) + return obj + return None + + # Monster + def Testbool(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(34)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.BoolFlags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashs32Fnv1(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(36)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int32Flags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashu32Fnv1(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(38)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashs64Fnv1(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(40)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashu64Fnv1(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(42)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint64Flags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashs32Fnv1a(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(44)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int32Flags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashu32Fnv1a(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(46)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashs64Fnv1a(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(48)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos) + return 0 + + # Monster + def Testhashu64Fnv1a(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(50)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint64Flags, o + self._tab.Pos) + return 0 + +def MonsterStart(builder): builder.StartObject(24) +def MonsterAddPos(builder, pos): builder.PrependStructSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(pos), 0) +def MonsterAddMana(builder, mana): builder.PrependInt16Slot(1, mana, 150) +def MonsterAddHp(builder, hp): builder.PrependInt16Slot(2, hp, 100) +def MonsterAddName(builder, name): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) +def MonsterAddInventory(builder, inventory): builder.PrependUOffsetTRelativeSlot(5, flatbuffers.number_types.UOffsetTFlags.py_type(inventory), 0) +def MonsterStartInventoryVector(builder, numElems): return builder.StartVector(1, numElems, 1) +def MonsterAddColor(builder, color): builder.PrependInt8Slot(6, color, 8) +def MonsterAddTestType(builder, testType): builder.PrependUint8Slot(7, testType, 0) +def MonsterAddTest(builder, test): builder.PrependUOffsetTRelativeSlot(8, flatbuffers.number_types.UOffsetTFlags.py_type(test), 0) +def MonsterAddTest4(builder, test4): builder.PrependUOffsetTRelativeSlot(9, flatbuffers.number_types.UOffsetTFlags.py_type(test4), 0) +def MonsterStartTest4Vector(builder, numElems): return builder.StartVector(4, numElems, 2) +def MonsterAddTestarrayofstring(builder, testarrayofstring): builder.PrependUOffsetTRelativeSlot(10, flatbuffers.number_types.UOffsetTFlags.py_type(testarrayofstring), 0) +def MonsterStartTestarrayofstringVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def MonsterAddTestarrayoftables(builder, testarrayoftables): builder.PrependUOffsetTRelativeSlot(11, flatbuffers.number_types.UOffsetTFlags.py_type(testarrayoftables), 0) +def MonsterStartTestarrayoftablesVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def MonsterAddEnemy(builder, enemy): builder.PrependUOffsetTRelativeSlot(12, flatbuffers.number_types.UOffsetTFlags.py_type(enemy), 0) +def MonsterAddTestnestedflatbuffer(builder, testnestedflatbuffer): builder.PrependUOffsetTRelativeSlot(13, flatbuffers.number_types.UOffsetTFlags.py_type(testnestedflatbuffer), 0) +def MonsterStartTestnestedflatbufferVector(builder, numElems): return builder.StartVector(1, numElems, 1) +def MonsterAddTestempty(builder, testempty): builder.PrependUOffsetTRelativeSlot(14, flatbuffers.number_types.UOffsetTFlags.py_type(testempty), 0) +def MonsterAddTestbool(builder, testbool): builder.PrependBoolSlot(15, testbool, 0) +def MonsterAddTesthashs32Fnv1(builder, testhashs32Fnv1): builder.PrependInt32Slot(16, testhashs32Fnv1, 0) +def MonsterAddTesthashu32Fnv1(builder, testhashu32Fnv1): builder.PrependUint32Slot(17, testhashu32Fnv1, 0) +def MonsterAddTesthashs64Fnv1(builder, testhashs64Fnv1): builder.PrependInt64Slot(18, testhashs64Fnv1, 0) +def MonsterAddTesthashu64Fnv1(builder, testhashu64Fnv1): builder.PrependUint64Slot(19, testhashu64Fnv1, 0) +def MonsterAddTesthashs32Fnv1a(builder, testhashs32Fnv1a): builder.PrependInt32Slot(20, testhashs32Fnv1a, 0) +def MonsterAddTesthashu32Fnv1a(builder, testhashu32Fnv1a): builder.PrependUint32Slot(21, testhashu32Fnv1a, 0) +def MonsterAddTesthashs64Fnv1a(builder, testhashs64Fnv1a): builder.PrependInt64Slot(22, testhashs64Fnv1a, 0) +def MonsterAddTesthashu64Fnv1a(builder, testhashu64Fnv1a): builder.PrependUint64Slot(23, testhashu64Fnv1a, 0) +def MonsterEnd(builder): return builder.EndObject() diff --git a/tests/MyGame/Example/Stat.py b/tests/MyGame/Example/Stat.py new file mode 100644 index 000000000..23bef0db6 --- /dev/null +++ b/tests/MyGame/Example/Stat.py @@ -0,0 +1,39 @@ +# automatically generated, do not modify + +# namespace: Example + +import flatbuffers + +class Stat(object): + __slots__ = ['_tab'] + + # Stat + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Stat + def Id(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.String(o + self._tab.Pos) + return "" + + # Stat + def Val(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos) + return 0 + + # Stat + def Count(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint16Flags, o + self._tab.Pos) + return 0 + +def StatStart(builder): builder.StartObject(3) +def StatAddId(builder, id): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(id), 0) +def StatAddVal(builder, val): builder.PrependInt64Slot(1, val, 0) +def StatAddCount(builder, count): builder.PrependUint16Slot(2, count, 0) +def StatEnd(builder): return builder.EndObject() diff --git a/tests/MyGame/Example/Test.py b/tests/MyGame/Example/Test.py new file mode 100644 index 000000000..4c4912a61 --- /dev/null +++ b/tests/MyGame/Example/Test.py @@ -0,0 +1,24 @@ +# automatically generated, do not modify + +# namespace: Example + +import flatbuffers + +class Test(object): + __slots__ = ['_tab'] + + # Test + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Test + def A(self): return self._tab.Get(flatbuffers.number_types.Int16Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0)) + # Test + def B(self): return self._tab.Get(flatbuffers.number_types.Int8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(2)) + +def CreateTest(builder, a, b): + builder.Prep(2, 4) + builder.Pad(1) + builder.PrependInt8(b) + builder.PrependInt16(a) + return builder.Offset() diff --git a/tests/MyGame/Example/Vec3.py b/tests/MyGame/Example/Vec3.py new file mode 100644 index 000000000..3010d5bc9 --- /dev/null +++ b/tests/MyGame/Example/Vec3.py @@ -0,0 +1,44 @@ +# automatically generated, do not modify + +# namespace: Example + +import flatbuffers + +class Vec3(object): + __slots__ = ['_tab'] + + # Vec3 + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Vec3 + def X(self): return self._tab.Get(flatbuffers.number_types.Float32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0)) + # Vec3 + def Y(self): return self._tab.Get(flatbuffers.number_types.Float32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(4)) + # Vec3 + def Z(self): return self._tab.Get(flatbuffers.number_types.Float32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(8)) + # Vec3 + def Test1(self): return self._tab.Get(flatbuffers.number_types.Float64Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(16)) + # Vec3 + def Test2(self): return self._tab.Get(flatbuffers.number_types.Int8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(24)) + # Vec3 + def Test3(self, obj): + obj.Init(self._tab.Bytes, self._tab.Pos + 26) + return obj + + +def CreateVec3(builder, x, y, z, test1, test2, Test_a, Test_b): + builder.Prep(16, 32) + builder.Pad(2) + builder.Prep(2, 4) + builder.Pad(1) + builder.PrependInt8(Test_b) + builder.PrependInt16(Test_a) + builder.Pad(1) + builder.PrependInt8(test2) + builder.PrependFloat64(test1) + builder.Pad(4) + builder.PrependFloat32(z) + builder.PrependFloat32(y) + builder.PrependFloat32(x) + return builder.Offset() diff --git a/tests/MyGame/Example/__init__.py b/tests/MyGame/Example/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/MyGame/__init__.py b/tests/MyGame/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/PythonTest.sh b/tests/PythonTest.sh new file mode 100755 index 000000000..e59110b70 --- /dev/null +++ b/tests/PythonTest.sh @@ -0,0 +1,78 @@ +#!/bin/bash -eu +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pushd "$(dirname $0)" >/dev/null +test_dir="$(pwd)" +gen_code_path=${test_dir} +runtime_library_dir=${test_dir}/../python + +# Emit Python code for the example schema in the test dir: +${test_dir}/../flatc -p -o ${gen_code_path} monster_test.fbs + +# Syntax: run_tests +# +interpreters_tested=() +function run_tests() { + if $(which ${1} >/dev/null); then + echo "Testing with interpreter: ${1}" + PYTHONDONTWRITEBYTECODE=1 \ + JYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=${runtime_library_dir}:${gen_code_path} \ + JYTHONPATH=${runtime_library_dir}:${gen_code_path} \ + COMPARE_GENERATED_TO_GO=0 \ + COMPARE_GENERATED_TO_JAVA=0 \ + $1 py_test.py $2 $3 $4 + interpreters_tested+=(${1}) + echo + fi +} + +# Run test suite with these interpreters. The arguments are benchmark counts. +run_tests python2.6 100 100 100 +run_tests python2.7 100 100 100 +run_tests python3 100 100 100 +run_tests pypy 100 100 100 + +# NOTE: We'd like to support python2.5 in the future. + +# NOTE: Jython 2.7.0 fails due to a bug in the stdlib `struct` library: +# http://bugs.jython.org/issue2188 + +if [ ${#interpreters_tested[@]} -eq 0 ]; then + echo "No Python interpeters found on this system, could not run tests." + exit 1 +fi + +# Run test suite with default python intereter. +# (If the Python program `coverage` is available, it will be run, too. +# Install `coverage` with `pip install coverage`.) +if $(which coverage >/dev/null); then + echo 'Found coverage utility, running coverage with default Python:' + + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=${runtime_library_dir}:${gen_code_path} \ + coverage run --source=flatbuffers,MyGame py_test.py 0 0 0 > /dev/null + + echo + cov_result=`coverage report --omit="*flatbuffers/vendor*,*py_test*" \ + | tail -n 1 | awk ' { print $4 } '` + echo "Code coverage: ${cov_result}" +else + echo -n "Did not find coverage utility for default Python, skipping. " + echo "Install with 'pip install coverage'." +fi + +echo +echo "OK: all tests passed for ${#interpreters_tested[@]} interpreters: ${interpreters_tested[@]}." diff --git a/tests/monster_test_generated.h b/tests/monster_test_generated.h old mode 100755 new mode 100644 diff --git a/tests/monsterdata_python_wire.mon b/tests/monsterdata_python_wire.mon new file mode 100644 index 0000000000000000000000000000000000000000..ae24c6bdd7a5bdf77850bca6291e7596f62317e0 GIT binary patch literal 288 zcmZ=@KmZd49|i>=SB^mfNQwZlFar+*2LlU`g$tMfB|LyQ0El}aiWouk4lo-e+F%bP z9DoRjVSvMdiGh`Y4aSAiOhCE>h*N+V#Fk^wVBi9}!vyFC2*BqOs1hgza+d%Qvx4}H oOw1siAnaC@3NjunUXof|VuZvt1hP4S*tgO*Kd-nXwFt-t0MD=y6951J literal 0 HcmV?d00001 diff --git a/tests/py_test.py b/tests/py_test.py new file mode 100644 index 000000000..c6861f1e3 --- /dev/null +++ b/tests/py_test.py @@ -0,0 +1,1331 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path +import sys +PY_VERSION = sys.version_info[:2] + +import ctypes +from collections import defaultdict +import timeit +import unittest + + +from flatbuffers import compat +from flatbuffers.compat import range_func as compat_range + +import flatbuffers +from flatbuffers import number_types as N + +import MyGame # refers to generated code +import MyGame.Example # refers to generated code +import MyGame.Example.Any # refers to generated code +import MyGame.Example.Color # refers to generated code +import MyGame.Example.Monster # refers to generated code +import MyGame.Example.Test # refers to generated code +import MyGame.Example.Stat # refers to generated code +import MyGame.Example.Vec3 # refers to generated code + + +def assertRaises(test_case, fn, exception_class): + ''' Backwards-compatible assertion for exceptions raised. ''' + + exc = None + try: + fn() + except Exception as e: + exc = e + test_case.assertTrue(exc is not None) + test_case.assertTrue(isinstance(exc, exception_class)) + + +class TestWireFormat(unittest.TestCase): + def test_wire_format(self): + # Verify that using the generated Python code builds a buffer without + # returning errors, and is interpreted correctly: + gen_buf, gen_off = make_monster_from_generated_code() + CheckReadBuffer(gen_buf, gen_off) + + # Verify that the canonical flatbuffer file is readable by the + # generated Python code. Note that context managers are not part of + # Python 2.5, so we use the simpler open/close methods here: + f = open('monsterdata_test.mon', 'rb') + canonicalWireData = f.read() + f.close() + CheckReadBuffer(bytearray(canonicalWireData), 0) + + # Write the generated buffer out to a file: + f = open('monsterdata_python_wire.mon', 'wb') + f.write(gen_buf[gen_off:]) + f.close() + + +def CheckReadBuffer(buf, offset): + ''' CheckReadBuffer checks that the given buffer is evaluated correctly + as the example Monster. ''' + + def asserter(stmt): + ''' An assertion helper that is separated from TestCase classes. ''' + if not stmt: + raise AssertionError('CheckReadBuffer case failed') + + monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset) + + asserter(monster.Hp() == 80) + asserter(monster.Mana() == 150) + asserter(monster.Name() == b'MyMonster') + + # initialize a Vec3 from Pos() + vec = monster.Pos() + asserter(vec is not None) + + # verify the properties of the Vec3 + asserter(vec.X() == 1.0) + asserter(vec.Y() == 2.0) + asserter(vec.Z() == 3.0) + asserter(vec.Test1() == 3.0) + asserter(vec.Test2() == 2) + + # initialize a Test from Test3(...) + t = MyGame.Example.Test.Test() + t = vec.Test3(t) + asserter(t is not None) + + # verify the properties of the Test + asserter(t.A() == 5) + asserter(t.B() == 6) + + # verify that the enum code matches the enum declaration: + union_type = MyGame.Example.Any.Any + asserter(monster.TestType() == union_type.Monster) + + # initialize a Table from a union field Test(...) + table2 = monster.Test() + asserter(type(table2) is flatbuffers.table.Table) + + # initialize a Monster from the Table from the union + monster2 = MyGame.Example.Monster.Monster() + monster2.Init(table2.Bytes, table2.Pos) + + asserter(monster2.Name() == b"Fred") + + # iterate through the first monster's inventory: + asserter(monster.InventoryLength() == 5) + + invsum = 0 + for i in compat_range(monster.InventoryLength()): + v = monster.Inventory(i) + invsum += int(v) + asserter(invsum == 10) + + asserter(monster.Test4Length() == 2) + + # create a 'Test' object and populate it: + test0 = monster.Test4(0) + asserter(type(test0) is MyGame.Example.Test.Test) + + test1 = monster.Test4(1) + asserter(type(test1) is MyGame.Example.Test.Test) + + # the position of test0 and test1 are swapped in monsterdata_java_wire + # and monsterdata_test_wire, so ignore ordering + v0 = test0.A() + v1 = test0.B() + v2 = test1.A() + v3 = test1.B() + sumtest12 = int(v0) + int(v1) + int(v2) + int(v3) + + asserter(sumtest12 == 100) + + asserter(monster.TestarrayofstringLength() == 2) + asserter(monster.Testarrayofstring(0) == b"test1") + asserter(monster.Testarrayofstring(1) == b"test2") + + asserter(monster.Enemy() is None) + + asserter(monster.TestarrayoftablesLength() == 0) + asserter(monster.TestnestedflatbufferLength() == 0) + asserter(monster.Testempty() is None) + + +class TestFuzz(unittest.TestCase): + ''' Low level stress/fuzz test: serialize/deserialize a variety of + different kinds of data in different combinations ''' + + ofInt32Bytes = compat.binary_type([0x83, 0x33, 0x33, 0x33]) + ofInt64Bytes = compat.binary_type([0x84, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44]) + overflowingInt32Val = flatbuffers.encode.Get(flatbuffers.packer.int32, + ofInt32Bytes, 0) + overflowingInt64Val = flatbuffers.encode.Get(flatbuffers.packer.int64, + ofInt64Bytes, 0) + + # Values we're testing against: chosen to ensure no bits get chopped + # off anywhere, and also be different from eachother. + boolVal = True + int8Val = N.Int8Flags.py_type(-127) # 0x81 + uint8Val = N.Uint8Flags.py_type(0xFF) + int16Val = N.Int16Flags.py_type(-32222) # 0x8222 + uint16Val = N.Uint16Flags.py_type(0xFEEE) + int32Val = N.Int32Flags.py_type(overflowingInt32Val) + uint32Val = N.Uint32Flags.py_type(0xFDDDDDDD) + int64Val = N.Int64Flags.py_type(overflowingInt64Val) + uint64Val = N.Uint64Flags.py_type(0xFCCCCCCCCCCCCCCC) + # Python uses doubles, so force it here + float32Val = N.Float32Flags.py_type(ctypes.c_float(3.14159).value) + float64Val = N.Float64Flags.py_type(3.14159265359) + + def test_fuzz(self): + return self.check_once(11, 100) + + def check_once(self, fuzzFields, fuzzObjects): + testValuesMax = 11 # hardcoded to the number of scalar types + + builder = flatbuffers.Builder(0) + l = LCG() + + objects = [0 for _ in compat_range(fuzzObjects)] + + # Generate fuzzObjects random objects each consisting of + # fuzzFields fields, each of a random type. + for i in compat_range(fuzzObjects): + builder.StartObject(fuzzFields) + + for j in compat_range(fuzzFields): + choice = int(l.Next()) % testValuesMax + if choice == 0: + builder.PrependBoolSlot(int(j), self.boolVal, False) + elif choice == 1: + builder.PrependInt8Slot(int(j), self.int8Val, 0) + elif choice == 2: + builder.PrependUint8Slot(int(j), self.uint8Val, 0) + elif choice == 3: + builder.PrependInt16Slot(int(j), self.int16Val, 0) + elif choice == 4: + builder.PrependUint16Slot(int(j), self.uint16Val, 0) + elif choice == 5: + builder.PrependInt32Slot(int(j), self.int32Val, 0) + elif choice == 6: + builder.PrependUint32Slot(int(j), self.uint32Val, 0) + elif choice == 7: + builder.PrependInt64Slot(int(j), self.int64Val, 0) + elif choice == 8: + builder.PrependUint64Slot(int(j), self.uint64Val, 0) + elif choice == 9: + builder.PrependFloat32Slot(int(j), self.float32Val, 0) + elif choice == 10: + builder.PrependFloat64Slot(int(j), self.float64Val, 0) + else: + raise RuntimeError('unreachable') + + off = builder.EndObject() + + # store the offset from the end of the builder buffer, + # since it will keep growing: + objects[i] = off + + # Do some bookkeeping to generate stats on fuzzes: + stats = defaultdict(int) + def check(table, desc, want, got): + stats[desc] += 1 + self.assertEqual(want, got, "%s != %s, %s" % (want, got, desc)) + + l = LCG() # Reset. + + # Test that all objects we generated are readable and return the + # expected values. We generate random objects in the same order + # so this is deterministic. + for i in compat_range(fuzzObjects): + + table = flatbuffers.table.Table(builder.Bytes, + len(builder.Bytes) - objects[i]) + + for j in compat_range(fuzzFields): + field_count = flatbuffers.builder.VtableMetadataFields + j + f = N.VOffsetTFlags.py_type(field_count * + N.VOffsetTFlags.bytewidth) + choice = int(l.Next()) % testValuesMax + + if choice == 0: + check(table, "bool", self.boolVal, + table.GetSlot(f, False, N.BoolFlags)) + elif choice == 1: + check(table, "int8", self.int8Val, + table.GetSlot(f, 0, N.Int8Flags)) + elif choice == 2: + check(table, "uint8", self.uint8Val, + table.GetSlot(f, 0, N.Uint8Flags)) + elif choice == 3: + check(table, "int16", self.int16Val, + table.GetSlot(f, 0, N.Int16Flags)) + elif choice == 4: + check(table, "uint16", self.uint16Val, + table.GetSlot(f, 0, N.Uint16Flags)) + elif choice == 5: + check(table, "int32", self.int32Val, + table.GetSlot(f, 0, N.Int32Flags)) + elif choice == 6: + check(table, "uint32", self.uint32Val, + table.GetSlot(f, 0, N.Uint32Flags)) + elif choice == 7: + check(table, "int64", self.int64Val, + table.GetSlot(f, 0, N.Int64Flags)) + elif choice == 8: + check(table, "uint64", self.uint64Val, + table.GetSlot(f, 0, N.Uint64Flags)) + elif choice == 9: + check(table, "float32", self.float32Val, + table.GetSlot(f, 0, N.Float32Flags)) + elif choice == 10: + check(table, "float64", self.float64Val, + table.GetSlot(f, 0, N.Float64Flags)) + else: + raise RuntimeError('unreachable') + + # If enough checks were made, verify that all scalar types were used: + self.assertEqual(testValuesMax, len(stats), + "fuzzing failed to test all scalar types: %s" % stats) + + +class TestByteLayout(unittest.TestCase): + ''' TestByteLayout checks the bytes of a Builder in various scenarios. ''' + + def assertBuilderEquals(self, builder, want_chars_or_ints): + def integerize(x): + if isinstance(x, compat.string_types): + return ord(x) + return x + + want_ints = list(map(integerize, want_chars_or_ints)) + want = bytearray(want_ints) + got = builder.Output() + self.assertEqual(want, got) + + def test_numbers(self): + b = flatbuffers.Builder(0) + self.assertBuilderEquals(b, []) + b.PrependBool(True) + self.assertBuilderEquals(b, [1]) + b.PrependInt8(-127) + self.assertBuilderEquals(b, [129, 1]) + b.PrependUint8(255) + self.assertBuilderEquals(b, [255, 129, 1]) + b.PrependInt16(-32222) + self.assertBuilderEquals(b, [0x22, 0x82, 0, 255, 129, 1]) # first pad + b.PrependUint16(0xFEEE) + # no pad this time: + self.assertBuilderEquals(b, [0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) + b.PrependInt32(-53687092) + self.assertBuilderEquals(b, [204, 204, 204, 252, 0xEE, 0xFE, + 0x22, 0x82, 0, 255, 129, 1]) + b.PrependUint32(0x98765432) + self.assertBuilderEquals(b, [0x32, 0x54, 0x76, 0x98, + 204, 204, 204, 252, + 0xEE, 0xFE, 0x22, 0x82, + 0, 255, 129, 1]) + + def test_numbers64(self): + b = flatbuffers.Builder(0) + b.PrependUint64(0x1122334455667788) + self.assertBuilderEquals(b, [0x88, 0x77, 0x66, 0x55, + 0x44, 0x33, 0x22, 0x11]) + + b = flatbuffers.Builder(0) + b.PrependInt64(0x1122334455667788) + self.assertBuilderEquals(b, [0x88, 0x77, 0x66, 0x55, + 0x44, 0x33, 0x22, 0x11]) + + def test_1xbyte_vector(self): + b = flatbuffers.Builder(0) + self.assertBuilderEquals(b, []) + b.StartVector(flatbuffers.number_types.Uint8Flags.bytewidth, 1, 1) + self.assertBuilderEquals(b, [0, 0, 0]) # align to 4bytes + b.PrependByte(1) + self.assertBuilderEquals(b, [1, 0, 0, 0]) + b.EndVector(1) + self.assertBuilderEquals(b, [1, 0, 0, 0, 1, 0, 0, 0]) # padding + + def test_2xbyte_vector(self): + b = flatbuffers.Builder(0) + b.StartVector(flatbuffers.number_types.Uint8Flags.bytewidth, 2, 1) + self.assertBuilderEquals(b, [0, 0]) # align to 4bytes + b.PrependByte(1) + self.assertBuilderEquals(b, [1, 0, 0]) + b.PrependByte(2) + self.assertBuilderEquals(b, [2, 1, 0, 0]) + b.EndVector(2) + self.assertBuilderEquals(b, [2, 0, 0, 0, 2, 1, 0, 0]) # padding + + def test_1xuint16_vector(self): + b = flatbuffers.Builder(0) + b.StartVector(flatbuffers.number_types.Uint16Flags.bytewidth, 1, 1) + self.assertBuilderEquals(b, [0, 0]) # align to 4bytes + b.PrependUint16(1) + self.assertBuilderEquals(b, [1, 0, 0, 0]) + b.EndVector(1) + self.assertBuilderEquals(b, [1, 0, 0, 0, 1, 0, 0, 0]) # padding + + def test_2xuint16_vector(self): + b = flatbuffers.Builder(0) + b.StartVector(flatbuffers.number_types.Uint16Flags.bytewidth, 2, 1) + self.assertBuilderEquals(b, []) # align to 4bytes + b.PrependUint16(0xABCD) + self.assertBuilderEquals(b, [0xCD, 0xAB]) + b.PrependUint16(0xDCBA) + self.assertBuilderEquals(b, [0xBA, 0xDC, 0xCD, 0xAB]) + b.EndVector(2) + self.assertBuilderEquals(b, [2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB]) + + def test_create_ascii_string(self): + b = flatbuffers.Builder(0) + b.CreateString(u"foo".encode('ascii')) + # 0-terminated, no pad: + self.assertBuilderEquals(b, [3, 0, 0, 0, 'f', 'o', 'o', 0]) + b.CreateString(u"moop".encode('ascii')) + # 0-terminated, 3-byte pad: + self.assertBuilderEquals(b, [4, 0, 0, 0, 'm', 'o', 'o', 'p', + 0, 0, 0, 0, + 3, 0, 0, 0, 'f', 'o', 'o', 0]) + + def test_create_arbitrary_string(self): + b = flatbuffers.Builder(0) + s = "\x01\x02\x03".encode('utf-8') + b.CreateString(s) + # 0-terminated, no pad: + self.assertBuilderEquals(b, [3, 0, 0, 0, 1, 2, 3, 0]) + s2 = "\x04\x05\x06\x07".encode('utf-8') + b.CreateString(s2) + # 0-terminated, 3-byte pad: + self.assertBuilderEquals(b, [4, 0, 0, 0, 4, 5, 6, 7, 0, 0, 0, 0, + 3, 0, 0, 0, 1, 2, 3, 0]) + + def test_empty_vtable(self): + b = flatbuffers.Builder(0) + b.StartObject(0) + self.assertBuilderEquals(b, []) + b.EndObject() + self.assertBuilderEquals(b, [4, 0, 4, 0, 4, 0, 0, 0]) + + def test_vtable_with_one_true_bool(self): + b = flatbuffers.Builder(0) + self.assertBuilderEquals(b, []) + b.StartObject(1) + self.assertBuilderEquals(b, []) + b.PrependBoolSlot(0, True, False) + b.EndObject() + self.assertBuilderEquals(b, [ + 6, 0, # vtable bytes + 8, 0, # length of object including vtable offset + 7, 0, # start of bool value + 6, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, 0, # padded to 4 bytes + 1, # bool value + ]) + + def test_vtable_with_one_default_bool(self): + b = flatbuffers.Builder(0) + self.assertBuilderEquals(b, []) + b.StartObject(1) + self.assertBuilderEquals(b, []) + b.PrependBoolSlot(0, False, False) + b.EndObject() + self.assertBuilderEquals(b, [ + 6, 0, # vtable bytes + 4, 0, # end of object from here + 0, 0, # entry 1 is zero + 6, 0, 0, 0, # offset for start of vtable (int32) + ]) + + def test_vtable_with_one_int16(self): + b = flatbuffers.Builder(0) + b.StartObject(1) + b.PrependInt16Slot(0, 0x789A, 0) + b.EndObject() + self.assertBuilderEquals(b, [ + 6, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value + 6, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding to 4 bytes + 0x9A, 0x78, + ]) + + def test_vtable_with_two_int16(self): + b = flatbuffers.Builder(0) + b.StartObject(2) + b.PrependInt16Slot(0, 0x3456, 0) + b.PrependInt16Slot(1, 0x789A, 0) + b.EndObject() + self.assertBuilderEquals(b, [ + 8, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value 0 + 4, 0, # offset to value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0x9A, 0x78, # value 1 + 0x56, 0x34, # value 0 + ]) + + def test_vtable_with_int16_and_bool(self): + b = flatbuffers.Builder(0) + b.StartObject(2) + b.PrependInt16Slot(0, 0x3456, 0) + b.PrependBoolSlot(1, True, False) + b.EndObject() + self.assertBuilderEquals(b, [ + 8, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value 0 + 5, 0, # offset to value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, # padding + 1, # value 1 + 0x56, 0x34, # value 0 + ]) + + def test_vtable_with_empty_vector(self): + b = flatbuffers.Builder(0) + b.StartVector(flatbuffers.number_types.Uint8Flags.bytewidth, 0, 1) + vecend = b.EndVector(0) + b.StartObject(1) + b.PrependUOffsetTRelativeSlot(0, vecend, 0) + b.EndObject() + self.assertBuilderEquals(b, [ + 6, 0, # vtable bytes + 8, 0, + 4, 0, # offset to vector offset + 6, 0, 0, 0, # offset for start of vtable (int32) + 4, 0, 0, 0, + 0, 0, 0, 0, # length of vector (not in struct) + ]) + + def test_vtable_with_empty_vector_of_byte_and_some_scalars(self): + b = flatbuffers.Builder(0) + b.StartVector(flatbuffers.number_types.Uint8Flags.bytewidth, 0, 1) + vecend = b.EndVector(0) + b.StartObject(2) + b.PrependInt16Slot(0, 55, 0) + b.PrependUOffsetTRelativeSlot(1, vecend, 0) + b.EndObject() + self.assertBuilderEquals(b, [ + 8, 0, # vtable bytes + 12, 0, + 10, 0, # offset to value 0 + 4, 0, # offset to vector offset + 8, 0, 0, 0, # vtable loc + 8, 0, 0, 0, # value 1 + 0, 0, 55, 0, # value 0 + + 0, 0, 0, 0, # length of vector (not in struct) + ]) + + def test_vtable_with_1_int16_and_2vector_of_int16(self): + b = flatbuffers.Builder(0) + b.StartVector(flatbuffers.number_types.Int16Flags.bytewidth, 2, 1) + b.PrependInt16(0x1234) + b.PrependInt16(0x5678) + vecend = b.EndVector(2) + b.StartObject(2) + b.PrependUOffsetTRelativeSlot(1, vecend, 0) + b.PrependInt16Slot(0, 55, 0) + b.EndObject() + self.assertBuilderEquals(b, [ + 8, 0, # vtable bytes + 12, 0, # length of object + 6, 0, # start of value 0 from end of vtable + 8, 0, # start of value 1 from end of buffer + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding + 55, 0, # value 0 + 4, 0, 0, 0, # vector position from here + 2, 0, 0, 0, # length of vector (uint32) + 0x78, 0x56, # vector value 1 + 0x34, 0x12, # vector value 0 + ]) + + def test_vtable_with_1_struct_of_1_int8__1_int16__1_int32(self): + b = flatbuffers.Builder(0) + b.StartObject(1) + b.Prep(4+4+4, 0) + b.PrependInt8(55) + b.Pad(3) + b.PrependInt16(0x1234) + b.Pad(2) + b.PrependInt32(0x12345678) + structStart = b.Offset() + b.PrependStructSlot(0, structStart, 0) + b.EndObject() + self.assertBuilderEquals(b, [ + 6, 0, # vtable bytes + 16, 0, # end of object from here + 4, 0, # start of struct from here + 6, 0, 0, 0, # offset for start of vtable (int32) + 0x78, 0x56, 0x34, 0x12, # value 2 + 0, 0, # padding + 0x34, 0x12, # value 1 + 0, 0, 0, # padding + 55, # value 0 + ]) + + def test_vtable_with_1_vector_of_2_struct_of_2_int8(self): + b = flatbuffers.Builder(0) + b.StartVector(flatbuffers.number_types.Int8Flags.bytewidth*2, 2, 1) + b.PrependInt8(33) + b.PrependInt8(44) + b.PrependInt8(55) + b.PrependInt8(66) + vecend = b.EndVector(2) + b.StartObject(1) + b.PrependUOffsetTRelativeSlot(0, vecend, 0) + b.EndObject() + self.assertBuilderEquals(b, [ + 6, 0, # vtable bytes + 8, 0, + 4, 0, # offset of vector offset + 6, 0, 0, 0, # offset for start of vtable (int32) + 4, 0, 0, 0, # vector start offset + + 2, 0, 0, 0, # vector length + 66, # vector value 1,1 + 55, # vector value 1,0 + 44, # vector value 0,1 + 33, # vector value 0,0 + ]) + + def test_table_with_some_elements(self): + b = flatbuffers.Builder(0) + b.StartObject(2) + b.PrependInt8Slot(0, 33, 0) + b.PrependInt16Slot(1, 66, 0) + off = b.EndObject() + b.Finish(off) + + self.assertBuilderEquals(b, [ + 12, 0, 0, 0, # root of table: points to vtable offset + + 8, 0, # vtable bytes + 8, 0, # end of object from here + 7, 0, # start of value 0 + 4, 0, # start of value 1 + + 8, 0, 0, 0, # offset for start of vtable (int32) + + 66, 0, # value 1 + 0, # padding + 33, # value 0 + ]) + + def test__one_unfinished_table_and_one_finished_table(self): + b = flatbuffers.Builder(0) + b.StartObject(2) + b.PrependInt8Slot(0, 33, 0) + b.PrependInt8Slot(1, 44, 0) + off = b.EndObject() + b.Finish(off) + + b.StartObject(3) + b.PrependInt8Slot(0, 55, 0) + b.PrependInt8Slot(1, 66, 0) + b.PrependInt8Slot(2, 77, 0) + off = b.EndObject() + b.Finish(off) + + self.assertBuilderEquals(b, [ + 16, 0, 0, 0, # root of table: points to object + 0, 0, # padding + + 10, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 5, 0, # start of value 2 + 10, 0, 0, 0, # offset for start of vtable (int32) + 0, # padding + 77, # value 2 + 66, # value 1 + 55, # value 0 + + 12, 0, 0, 0, # root of table: points to object + + 8, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding + 44, # value 1 + 33, # value 0 + ]) + + def test_a_bunch_of_bools(self): + b = flatbuffers.Builder(0) + b.StartObject(8) + b.PrependBoolSlot(0, True, False) + b.PrependBoolSlot(1, True, False) + b.PrependBoolSlot(2, True, False) + b.PrependBoolSlot(3, True, False) + b.PrependBoolSlot(4, True, False) + b.PrependBoolSlot(5, True, False) + b.PrependBoolSlot(6, True, False) + b.PrependBoolSlot(7, True, False) + off = b.EndObject() + b.Finish(off) + + self.assertBuilderEquals(b, [ + 24, 0, 0, 0, # root of table: points to vtable offset + + 20, 0, # vtable bytes + 12, 0, # size of object + 11, 0, # start of value 0 + 10, 0, # start of value 1 + 9, 0, # start of value 2 + 8, 0, # start of value 3 + 7, 0, # start of value 4 + 6, 0, # start of value 5 + 5, 0, # start of value 6 + 4, 0, # start of value 7 + 20, 0, 0, 0, # vtable offset + + 1, # value 7 + 1, # value 6 + 1, # value 5 + 1, # value 4 + 1, # value 3 + 1, # value 2 + 1, # value 1 + 1, # value 0 + ]) + + def test_three_bools(self): + b = flatbuffers.Builder(0) + b.StartObject(3) + b.PrependBoolSlot(0, True, False) + b.PrependBoolSlot(1, True, False) + b.PrependBoolSlot(2, True, False) + off = b.EndObject() + b.Finish(off) + + self.assertBuilderEquals(b, [ + 16, 0, 0, 0, # root of table: points to vtable offset + + 0, 0, # padding + + 10, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 5, 0, # start of value 2 + 10, 0, 0, 0, # vtable offset from here + + 0, # padding + 1, # value 2 + 1, # value 1 + 1, # value 0 + ]) + + def test_some_floats(self): + b = flatbuffers.Builder(0) + b.StartObject(1) + b.PrependFloat32Slot(0, 1.0, 0.0) + off = b.EndObject() + + self.assertBuilderEquals(b, [ + 6, 0, # vtable bytes + 8, 0, # size of object + 4, 0, # start of value 0 + 6, 0, 0, 0, # vtable offset + + 0, 0, 128, 63, # value 0 + ]) + + +def make_monster_from_generated_code(): + ''' Use generated code to build the example Monster. ''' + + b = flatbuffers.Builder(0) + string = b.CreateString("MyMonster") + test1 = b.CreateString("test1") + test2 = b.CreateString("test2") + fred = b.CreateString("Fred") + + MyGame.Example.Monster.MonsterStartInventoryVector(b, 5) + b.PrependByte(4) + b.PrependByte(3) + b.PrependByte(2) + b.PrependByte(1) + b.PrependByte(0) + inv = b.EndVector(5) + + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddName(b, fred) + mon2 = MyGame.Example.Monster.MonsterEnd(b) + + MyGame.Example.Monster.MonsterStartTest4Vector(b, 2) + MyGame.Example.Test.CreateTest(b, 10, 20) + MyGame.Example.Test.CreateTest(b, 30, 40) + test4 = b.EndVector(2) + + MyGame.Example.Monster.MonsterStartTestarrayofstringVector(b, 2) + b.PrependUOffsetTRelative(test2) + b.PrependUOffsetTRelative(test1) + testArrayOfString = b.EndVector(2) + + MyGame.Example.Monster.MonsterStart(b) + + pos = MyGame.Example.Vec3.CreateVec3(b, 1.0, 2.0, 3.0, 3.0, 2, 5, 6) + MyGame.Example.Monster.MonsterAddPos(b, pos) + + MyGame.Example.Monster.MonsterAddHp(b, 80) + MyGame.Example.Monster.MonsterAddName(b, string) + MyGame.Example.Monster.MonsterAddInventory(b, inv) + MyGame.Example.Monster.MonsterAddTestType(b, 1) + MyGame.Example.Monster.MonsterAddTest(b, mon2) + MyGame.Example.Monster.MonsterAddTest4(b, test4) + MyGame.Example.Monster.MonsterAddTestarrayofstring(b, testArrayOfString) + mon = MyGame.Example.Monster.MonsterEnd(b) + + b.Finish(mon) + + return b.Bytes, b.Head() + + +class TestAllCodePathsOfExampleSchema(unittest.TestCase): + def setUp(self, *args, **kwargs): + super(TestAllCodePathsOfExampleSchema, self).setUp(*args, **kwargs) + + b = flatbuffers.Builder(0) + MyGame.Example.Monster.MonsterStart(b) + gen_mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(gen_mon) + + self.mon = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + + def test_default_monster_pos(self): + self.assertTrue(self.mon.Pos() is None) + + def test_nondefault_monster_mana(self): + b = flatbuffers.Builder(0) + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddMana(b, 50) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + got_mon = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertEqual(50, got_mon.Mana()) + + def test_default_monster_hp(self): + self.assertEqual(100, self.mon.Hp()) + + def test_default_monster_name(self): + self.assertEqual('', self.mon.Name()) + + def test_default_monster_inventory_item(self): + self.assertEqual(0, self.mon.Inventory(0)) + + def test_default_monster_inventory_length(self): + self.assertEqual(0, self.mon.InventoryLength()) + + def test_default_monster_color(self): + self.assertEqual(MyGame.Example.Color.Color.Blue, self.mon.Color()) + + def test_nondefault_monster_color(self): + b = flatbuffers.Builder(0) + color = MyGame.Example.Color.Color.Red + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddColor(b, color) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + mon2 = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertEqual(MyGame.Example.Color.Color.Red, mon2.Color()) + + def test_default_monster_testtype(self): + self.assertEqual(0, self.mon.TestType()) + + def test_default_monster_test_field(self): + self.assertEqual(None, self.mon.Test()) + + def test_default_monster_test4_item(self): + self.assertEqual(None, self.mon.Test4(0)) + + def test_default_monster_test4_length(self): + self.assertEqual(0, self.mon.Test4Length()) + + def test_default_monster_testarrayofstring(self): + self.assertEqual("", self.mon.Testarrayofstring(0)) + + def test_default_monster_testarrayofstring_length(self): + self.assertEqual(0, self.mon.TestarrayofstringLength()) + + def test_default_monster_testarrayoftables(self): + self.assertEqual(None, self.mon.Testarrayoftables(0)) + + def test_nondefault_monster_testarrayoftables(self): + b = flatbuffers.Builder(0) + + # make a child Monster within a vector of Monsters: + MyGame.Example.Monster.MonsterStartTestarrayoftablesVector(b, 1) + + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddHp(b, 99) + sub_monster = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(sub_monster) + + tables = b.EndVector(1) + + # make the parent monster and include the vector of Monster: + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddTestarrayoftables(b, tables) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + # inspect the resulting data: + mon2 = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertEqual(99, mon2.Testarrayoftables(0).Hp()) + self.assertEqual(1, mon2.TestarrayoftablesLength()) + + def test_default_monster_testarrayoftables_length(self): + self.assertEqual(0, self.mon.TestarrayoftablesLength()) + + def test_nondefault_monster_enemy(self): + b = flatbuffers.Builder(0) + + # make an Enemy object: + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddHp(b, 88) + enemy = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(enemy) + + # make the parent monster and include the vector of Monster: + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddEnemy(b, enemy) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + # inspect the resulting data: + mon2 = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertEqual(88, mon2.Enemy().Hp()) + + def test_default_monster_testnestedflatbuffer(self): + self.assertEqual(0, self.mon.Testnestedflatbuffer(0)) + + def test_default_monster_testnestedflatbuffer_length(self): + self.assertEqual(0, self.mon.TestnestedflatbufferLength()) + + def test_nondefault_monster_testnestedflatbuffer(self): + b = flatbuffers.Builder(0) + + MyGame.Example.Monster.MonsterStartTestnestedflatbufferVector(b, 3) + b.PrependByte(4) + b.PrependByte(2) + b.PrependByte(0) + sub_buf = b.EndVector(3) + + # make the parent monster and include the vector of Monster: + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddTestnestedflatbuffer(b, sub_buf) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + # inspect the resulting data: + mon2 = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertEqual(3, mon2.TestnestedflatbufferLength()) + self.assertEqual(0, mon2.Testnestedflatbuffer(0)) + self.assertEqual(2, mon2.Testnestedflatbuffer(1)) + self.assertEqual(4, mon2.Testnestedflatbuffer(2)) + + def test_nondefault_monster_testempty(self): + b = flatbuffers.Builder(0) + + # make a Stat object: + MyGame.Example.Stat.StatStart(b) + MyGame.Example.Stat.StatAddVal(b, 123) + my_stat = MyGame.Example.Stat.StatEnd(b) + b.Finish(my_stat) + + # include the stat object in a monster: + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddTestempty(b, my_stat) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + # inspect the resulting data: + mon2 = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertEqual(123, mon2.Testempty().Val()) + + def test_default_monster_testbool(self): + self.assertFalse(self.mon.Testbool()) + + def test_nondefault_monster_testbool(self): + b = flatbuffers.Builder(0) + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddTestbool(b, True) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + # inspect the resulting data: + mon2 = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertTrue(mon2.Testbool()) + + def test_default_monster_testhashes(self): + self.assertEqual(0, self.mon.Testhashs32Fnv1()) + self.assertEqual(0, self.mon.Testhashu32Fnv1()) + self.assertEqual(0, self.mon.Testhashs64Fnv1()) + self.assertEqual(0, self.mon.Testhashu64Fnv1()) + self.assertEqual(0, self.mon.Testhashs32Fnv1a()) + self.assertEqual(0, self.mon.Testhashu32Fnv1a()) + self.assertEqual(0, self.mon.Testhashs64Fnv1a()) + self.assertEqual(0, self.mon.Testhashu64Fnv1a()) + + def test_nondefault_monster_testhashes(self): + b = flatbuffers.Builder(0) + MyGame.Example.Monster.MonsterStart(b) + MyGame.Example.Monster.MonsterAddTesthashs32Fnv1(b, 1) + MyGame.Example.Monster.MonsterAddTesthashu32Fnv1(b, 2) + MyGame.Example.Monster.MonsterAddTesthashs64Fnv1(b, 3) + MyGame.Example.Monster.MonsterAddTesthashu64Fnv1(b, 4) + MyGame.Example.Monster.MonsterAddTesthashs32Fnv1a(b, 5) + MyGame.Example.Monster.MonsterAddTesthashu32Fnv1a(b, 6) + MyGame.Example.Monster.MonsterAddTesthashs64Fnv1a(b, 7) + MyGame.Example.Monster.MonsterAddTesthashu64Fnv1a(b, 8) + mon = MyGame.Example.Monster.MonsterEnd(b) + b.Finish(mon) + + # inspect the resulting data: + mon2 = MyGame.Example.Monster.Monster.GetRootAsMonster(b.Bytes, + b.Head()) + self.assertEqual(1, mon2.Testhashs32Fnv1()) + self.assertEqual(2, mon2.Testhashu32Fnv1()) + self.assertEqual(3, mon2.Testhashs64Fnv1()) + self.assertEqual(4, mon2.Testhashu64Fnv1()) + self.assertEqual(5, mon2.Testhashs32Fnv1a()) + self.assertEqual(6, mon2.Testhashu32Fnv1a()) + self.assertEqual(7, mon2.Testhashs64Fnv1a()) + self.assertEqual(8, mon2.Testhashu64Fnv1a()) + + +class TestVtableDeduplication(unittest.TestCase): + ''' TestVtableDeduplication verifies that vtables are deduplicated. ''' + + def test_vtable_deduplication(self): + b = flatbuffers.Builder(0) + + b.StartObject(4) + b.PrependByteSlot(0, 0, 0) + b.PrependByteSlot(1, 11, 0) + b.PrependByteSlot(2, 22, 0) + b.PrependInt16Slot(3, 33, 0) + obj0 = b.EndObject() + + b.StartObject(4) + b.PrependByteSlot(0, 0, 0) + b.PrependByteSlot(1, 44, 0) + b.PrependByteSlot(2, 55, 0) + b.PrependInt16Slot(3, 66, 0) + obj1 = b.EndObject() + + b.StartObject(4) + b.PrependByteSlot(0, 0, 0) + b.PrependByteSlot(1, 77, 0) + b.PrependByteSlot(2, 88, 0) + b.PrependInt16Slot(3, 99, 0) + obj2 = b.EndObject() + + got = b.Output() + + want = bytearray([ + 240, 255, 255, 255, # == -12. offset to dedupped vtable. + 99, 0, + 88, + 77, + 248, 255, 255, 255, # == -8. offset to dedupped vtable. + 66, 0, + 55, + 44, + 12, 0, + 8, 0, + 0, 0, + 7, 0, + 6, 0, + 4, 0, + 12, 0, 0, 0, + 33, 0, + 22, + 11, + ]) + + self.assertEqual((len(want), want), (len(got), got)) + + table0 = flatbuffers.table.Table(b.Bytes, len(b.Bytes) - obj0) + table1 = flatbuffers.table.Table(b.Bytes, len(b.Bytes) - obj1) + table2 = flatbuffers.table.Table(b.Bytes, len(b.Bytes) - obj2) + + def _checkTable(tab, voffsett_value, b, c, d): + # vtable size + got = tab.GetVOffsetTSlot(0, 0) + self.assertEqual(12, got, 'case 0, 0') + + # object size + got = tab.GetVOffsetTSlot(2, 0) + self.assertEqual(8, got, 'case 2, 0') + + # default value + got = tab.GetVOffsetTSlot(4, 0) + self.assertEqual(voffsett_value, got, 'case 4, 0') + + got = tab.GetSlot(6, 0, N.Uint8Flags) + self.assertEqual(b, got, 'case 6, 0') + + val = tab.GetSlot(8, 0, N.Uint8Flags) + self.assertEqual(c, val, 'failed 8, 0') + + got = tab.GetSlot(10, 0, N.Uint8Flags) + self.assertEqual(d, got, 'failed 10, 0') + + _checkTable(table0, 0, 11, 22, 33) + _checkTable(table1, 0, 44, 55, 66) + _checkTable(table2, 0, 77, 88, 99) + + +class TestExceptions(unittest.TestCase): + def test_not_in_object_error(self): + b = flatbuffers.Builder(0) + exc = None + assertRaises(self, lambda: b.EndObject(), + flatbuffers.builder.NotInObjectError) + + def test_object_is_nested_error(self): + b = flatbuffers.Builder(0) + b.StartObject(0) + assertRaises(self, lambda: b.StartObject(0), + flatbuffers.builder.ObjectIsNestedError) + + def test_struct_is_not_inline_error(self): + b = flatbuffers.Builder(0) + b.StartObject(0) + assertRaises(self, lambda: b.PrependStructSlot(0, 1, 0), + flatbuffers.builder.StructIsNotInlineError) + + def test_unreachable_error(self): + b = flatbuffers.Builder(0) + assertRaises(self, lambda: b.PrependUOffsetTRelative(1), + flatbuffers.builder.OffsetArithmeticError) + + +def CheckAgainstGoldDataGo(): + try: + gen_buf, gen_off = make_monster_from_generated_code() + fn = 'monsterdata_go_wire.mon' + if not os.path.exists(fn): + print('Go-generated data does not exist, failed.') + return False + + # would like to use a context manager here, but it's less + # backwards-compatible: + f = open(fn, 'rb') + go_wire_data = f.read() + f.close() + + CheckReadBuffer(bytearray(go_wire_data), 0) + if not bytearray(gen_buf[gen_off:]) == bytearray(go_wire_data): + raise AssertionError('CheckAgainstGoldDataGo failed') + except: + print('Failed to test against Go-generated test data.') + return False + + print('Can read Go-generated test data, and Python generates bytewise identical data.') + return True + + +def CheckAgainstGoldDataJava(): + try: + gen_buf, gen_off = make_monster_from_generated_code() + fn = 'monsterdata_java_wire.mon' + if not os.path.exists(fn): + print('Java-generated data does not exist, failed.') + return False + f = open(fn, 'rb') + java_wire_data = f.read() + f.close() + + CheckReadBuffer(bytearray(java_wire_data), 0) + except: + print('Failed to read Java-generated test data.') + return False + + print('Can read Java-generated test data.') + return True + + +class LCG(object): + ''' Include simple random number generator to ensure results will be the + same cross platform. + http://en.wikipedia.org/wiki/Park%E2%80%93Miller_random_number_generator ''' + + __slots__ = ['n'] + + InitialLCGSeed = 48271 + + def __init__(self): + self.n = self.InitialLCGSeed + + def Reset(self): + self.n = self.InitialLCGSeed + + def Next(self): + self.n = ((self.n * 279470273) % 4294967291) & 0xFFFFFFFF + return self.n + + +def BenchmarkVtableDeduplication(count): + ''' + BenchmarkVtableDeduplication measures the speed of vtable deduplication + by creating `prePop` vtables, then populating `count` objects with a + different single vtable. + + When count is large (as in long benchmarks), memory usage may be high. + ''' + + prePop = 10 + builder = flatbuffers.Builder(0) + + # pre-populate some vtables: + for i in compat_range(prePop): + builder.StartObject(i) + for j in compat_range(i): + builder.PrependInt16Slot(j, j, 0) + builder.EndObject() + + # benchmark deduplication of a new vtable: + def f(): + builder.StartObject(prePop) + for j in compat_range(prePop): + builder.PrependInt16Slot(j, j, 0) + builder.EndObject() + + duration = timeit.timeit(stmt=f, number=count) + rate = float(count) / duration + print(('vtable deduplication rate: %.2f/sec' % rate)) + + +def BenchmarkCheckReadBuffer(count, buf, off): + ''' + BenchmarkCheckReadBuffer measures the speed of flatbuffer reading + by re-using the CheckReadBuffer function with the gold data. + ''' + + def f(): + CheckReadBuffer(buf, off) + + duration = timeit.timeit(stmt=f, number=count) + rate = float(count) / duration + data = float(len(buf) * count) / float(1024 * 1024) + data_rate = data / float(duration) + + print(('traversed %d %d-byte flatbuffers in %.2fsec: %.2f/sec, %.2fMB/sec') + % (count, len(buf), duration, rate, data_rate)) + + +def BenchmarkMakeMonsterFromGeneratedCode(count, length): + ''' + BenchmarkMakeMonsterFromGeneratedCode measures the speed of flatbuffer + creation by re-using the make_monster_from_generated_code function for + generating gold data examples. + ''' + + duration = timeit.timeit(stmt=make_monster_from_generated_code, + number=count) + rate = float(count) / duration + data = float(length * count) / float(1024 * 1024) + data_rate = data / float(duration) + + print(('built %d %d-byte flatbuffers in %.2fsec: %.2f/sec, %.2fMB/sec' % \ + (count, length, duration, rate, data_rate))) + + +def backward_compatible_run_tests(**kwargs): + if PY_VERSION < (2, 6): + sys.stderr.write("Python version less than 2.6 are not supported") + sys.stderr.flush() + return False + + # python2.6 has a reduced-functionality unittest.main function: + if PY_VERSION == (2, 6): + try: + unittest.main(**kwargs) + except SystemExit as e: + if not e.code == 0: + return False + return True + + # python2.7 and above let us not exit once unittest.main is run: + kwargs['exit'] = False + kwargs['verbosity'] = 0 + ret = unittest.main(**kwargs) + if ret.result.errors or ret.result.failures: + return False + + return True + +def main(): + import os + import sys + if not len(sys.argv) == 4: + sys.stderr.write(('Usage: %s ') + (' ') + ('\n' % sys.argv[0])) + sys.stderr.write((' Provide COMPARE_GENERATED_TO_GO=1 to check') + ('for bytewise comparison to Go data.\n')) + sys.stderr.write((' Provide COMPARE_GENERATED_TO_JAVA=1 to check') + ('for bytewise comparison to Java data.\n')) + sys.stderr.flush() + sys.exit(1) + + kwargs = dict(argv=sys.argv[:-3]) + + # run tests, and run some language comparison checks if needed: + success = backward_compatible_run_tests(**kwargs) + if success and os.environ.get('COMPARE_GENERATED_TO_GO', 0) == "1": + success = success and CheckAgainstGoldDataGo() + if success and os.environ.get('COMPARE_GENERATED_TO_JAVA', 0) == "1": + success = success and CheckAgainstGoldDataJava() + + if not success: + sys.stderr.write('Tests failed, skipping benchmarks.\n') + sys.stderr.flush() + sys.exit(1) + + # run benchmarks (if 0, they will be a noop): + bench_vtable = int(sys.argv[1]) + bench_traverse = int(sys.argv[2]) + bench_build = int(sys.argv[3]) + if bench_vtable: + BenchmarkVtableDeduplication(bench_vtable) + if bench_traverse: + buf, off = make_monster_from_generated_code() + BenchmarkCheckReadBuffer(bench_traverse, buf, off) + if bench_build: + buf, off = make_monster_from_generated_code() + BenchmarkMakeMonsterFromGeneratedCode(bench_build, len(buf)) + +if __name__ == '__main__': + main() diff --git a/tests/test.cpp b/tests/test.cpp index b0cace944..ebd4370ec 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -37,7 +37,7 @@ int testing_fails = 0; template void TestEq(T expval, U val, const char *exp, const char *file, int line) { - if (expval != val) { + if (U(expval) != val) { auto expval_str = flatbuffers::NumToString(expval); auto val_str = flatbuffers::NumToString(val); TEST_OUTPUT_LINE("TEST FAILED: %s:%d, %s (%s) != %s", file, line,