diff --git a/python/flatbuffers/builder.py b/python/flatbuffers/builder.py index dc93f3fae..d04ee854d 100644 --- a/python/flatbuffers/builder.py +++ b/python/flatbuffers/builder.py @@ -510,13 +510,21 @@ class Builder(object): self.current_vtable[slotnum] = self.Offset() ## @endcond - def __Finish(self, rootTable, sizePrefix): + def __Finish(self, rootTable, sizePrefix, file_identifier=None): """Finish finalizes a buffer, pointing to the given `rootTable`.""" N.enforce_number(rootTable, N.UOffsetTFlags) - prepSize = N.UOffsetTFlags.bytewidth - if sizePrefix: - prepSize += N.Int32Flags.bytewidth - self.Prep(self.minalign, prepSize) + + if file_identifier is not None: + self.Prep(N.UOffsetTFlags.bytewidth, N.Uint8Flags.bytewidth*4) + + # Convert bytes object file_identifier to an array of 4 8-bit integers, + # and use big-endian to enforce size compliance. + # https://docs.python.org/2/library/struct.html#format-characters + file_identifier = N.struct.unpack(">BBBB", file_identifier) + for i in range(encode.FILE_IDENTIFIER_LENGTH-1, -1, -1): + # Place the bytes of the file_identifer in reverse order: + self.Place(file_identifier[i], N.Uint8Flags) + self.PrependUOffsetTRelative(rootTable) if sizePrefix: size = len(self.Bytes) - self.Head() @@ -525,16 +533,16 @@ class Builder(object): self.finished = True return self.Head() - def Finish(self, rootTable): + def Finish(self, rootTable, file_identifier=None): """Finish finalizes a buffer, pointing to the given `rootTable`.""" - return self.__Finish(rootTable, False) + return self.__Finish(rootTable, False, file_identifier=file_identifier) - def FinishSizePrefixed(self, rootTable): + def FinishSizePrefixed(self, rootTable, file_identifier=None): """ Finish finalizes a buffer, pointing to the given `rootTable`, with the size prefixed. """ - return self.__Finish(rootTable, True) + return self.__Finish(rootTable, True, file_identifier=file_identifier) ## @cond FLATBUFFERS_INTERNAL def Prepend(self, flags, off): diff --git a/python/flatbuffers/encode.py b/python/flatbuffers/encode.py index 3a330dd87..c4f1a59ec 100644 --- a/python/flatbuffers/encode.py +++ b/python/flatbuffers/encode.py @@ -19,6 +19,8 @@ from .compat import import_numpy, NumpyRequiredForThisFeature np = import_numpy() +FILE_IDENTIFIER_LENGTH=4 + def Get(packer_type, buf, head): """ Get decodes a value at buf[head] using `packer_type`. """ return packer_type.unpack_from(memoryview_type(buf), head)[0] diff --git a/python/flatbuffers/util.py b/python/flatbuffers/util.py index abcb9c750..a5a783879 100644 --- a/python/flatbuffers/util.py +++ b/python/flatbuffers/util.py @@ -20,6 +20,21 @@ def GetSizePrefix(buf, offset): """Extract the size prefix from a buffer.""" return encode.Get(packer.int32, buf, offset) +def GetBufferIdentifier(buf, offset, size_prefixed=False): + """Extract the file_identifier from a buffer""" + if size_prefixed: + # increase offset by size of UOffsetTFlags + offset += number_types.UOffsetTFlags.bytewidth + # increase offset by size of root table pointer + offset += number_types.UOffsetTFlags.bytewidth + # end of FILE_IDENTIFIER + end = offset + encode.FILE_IDENTIFIER_LENGTH + return buf[offset:end] + +def BufferHasIdentifier(buf, offset, file_identifier, size_prefixed=False): + got = GetBufferIdentifier(buf, offset, size_prefixed=size_prefixed) + return got == file_identifier + def RemoveSizePrefix(buf, offset): """ Create a slice of a size-prefixed buffer that has diff --git a/src/idl_gen_python.cpp b/src/idl_gen_python.cpp index fde8a6588..c8db35973 100644 --- a/src/idl_gen_python.cpp +++ b/src/idl_gen_python.cpp @@ -612,6 +612,30 @@ class PythonGenerator : public BaseGenerator { GetEndOffsetOnTable(struct_def, code_ptr); } + // Generate function to check for proper file identifier + void GenHasFileIdentifier(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + std::string escapedID; + // In the event any of file_identifier characters are special(NULL, \, etc), + // problems occur. To prevent this, convert all chars to their hex-escaped + // equivalent. + for (auto it = parser_.file_identifier_.begin(); + it != parser_.file_identifier_.end(); ++it) { + escapedID += "\\x" + IntToStringHex(*it, 2); + } + + code += Indent + "@classmethod\n"; + code += Indent + "def " + NormalizedName(struct_def); + code += "BufferHasIdentifier(cls, buf, offset, size_prefixed=False):"; + code += "\n"; + code += Indent + Indent; + code += "return flatbuffers.util.BufferHasIdentifier(buf, offset, b\""; + code += escapedID; + code += "\", size_prefixed=size_prefixed)\n"; + code += "\n"; + } + // Generate struct or table methods. void GenStruct(const StructDef &struct_def, std::string *code_ptr) { if (struct_def.generated) return; @@ -622,6 +646,10 @@ class PythonGenerator : public BaseGenerator { // Generate a special accessor for the table that has been declared as // the root type. NewRootTypeFromBuffer(struct_def, code_ptr); + if (parser_.file_identifier_.length()){ + // Generate a special function to test file_identifier + GenHasFileIdentifier(struct_def, code_ptr); + } } // Generate the Init method that sets the field in a pre-existing // accessor object. This is to allow object reuse. diff --git a/tests/MyGame/Example/ArrayTable.py b/tests/MyGame/Example/ArrayTable.py index 12eefd318..6d583f928 100644 --- a/tests/MyGame/Example/ArrayTable.py +++ b/tests/MyGame/Example/ArrayTable.py @@ -14,6 +14,10 @@ class ArrayTable(object): x.Init(buf, n + offset) return x + @classmethod + def ArrayTableBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x41\x52\x52\x54", size_prefixed=size_prefixed) + # ArrayTable def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/Example/Monster.py b/tests/MyGame/Example/Monster.py index e83b9af4b..5baf64d44 100644 --- a/tests/MyGame/Example/Monster.py +++ b/tests/MyGame/Example/Monster.py @@ -15,6 +15,10 @@ class Monster(object): x.Init(buf, n + offset) return x + @classmethod + def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed) + # Monster def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/Example/Referrable.py b/tests/MyGame/Example/Referrable.py index 897b4ac48..eaec09bbf 100644 --- a/tests/MyGame/Example/Referrable.py +++ b/tests/MyGame/Example/Referrable.py @@ -14,6 +14,10 @@ class Referrable(object): x.Init(buf, n + offset) return x + @classmethod + def ReferrableBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed) + # Referrable def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/Example/Stat.py b/tests/MyGame/Example/Stat.py index ae33aef3d..0fbd2a7af 100644 --- a/tests/MyGame/Example/Stat.py +++ b/tests/MyGame/Example/Stat.py @@ -14,6 +14,10 @@ class Stat(object): x.Init(buf, n + offset) return x + @classmethod + def StatBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed) + # Stat def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.py b/tests/MyGame/Example/TestSimpleTableWithEnum.py index 70a3c1d57..cb9c631a9 100644 --- a/tests/MyGame/Example/TestSimpleTableWithEnum.py +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.py @@ -14,6 +14,10 @@ class TestSimpleTableWithEnum(object): x.Init(buf, n + offset) return x + @classmethod + def TestSimpleTableWithEnumBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed) + # TestSimpleTableWithEnum def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/Example/TypeAliases.py b/tests/MyGame/Example/TypeAliases.py index 707e53f14..81e9b0645 100644 --- a/tests/MyGame/Example/TypeAliases.py +++ b/tests/MyGame/Example/TypeAliases.py @@ -14,6 +14,10 @@ class TypeAliases(object): x.Init(buf, n + offset) return x + @classmethod + def TypeAliasesBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed) + # TypeAliases def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/Example2/Monster.py b/tests/MyGame/Example2/Monster.py index d334f8ea2..44cc90616 100644 --- a/tests/MyGame/Example2/Monster.py +++ b/tests/MyGame/Example2/Monster.py @@ -14,6 +14,10 @@ class Monster(object): x.Init(buf, n + offset) return x + @classmethod + def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed) + # Monster def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/InParentNamespace.py b/tests/MyGame/InParentNamespace.py index 428d9a94d..3bfcca707 100644 --- a/tests/MyGame/InParentNamespace.py +++ b/tests/MyGame/InParentNamespace.py @@ -14,6 +14,10 @@ class InParentNamespace(object): x.Init(buf, n + offset) return x + @classmethod + def InParentNamespaceBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed) + # InParentNamespace def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/MyGame/MonsterExtra.py b/tests/MyGame/MonsterExtra.py index 5fcace064..1f7dcb28d 100644 --- a/tests/MyGame/MonsterExtra.py +++ b/tests/MyGame/MonsterExtra.py @@ -14,6 +14,10 @@ class MonsterExtra(object): x.Init(buf, n + offset) return x + @classmethod + def MonsterExtraBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x45", size_prefixed=size_prefixed) + # MonsterExtra def Init(self, buf, pos): self._tab = flatbuffers.table.Table(buf, pos) diff --git a/tests/monsterdata_python_wire.mon b/tests/monsterdata_python_wire.mon index e41384a63..2fb956d58 100644 Binary files a/tests/monsterdata_python_wire.mon and b/tests/monsterdata_python_wire.mon differ diff --git a/tests/py_test.py b/tests/py_test.py index d640d3b3f..b15227091 100644 --- a/tests/py_test.py +++ b/tests/py_test.py @@ -65,8 +65,9 @@ class TestWireFormat(unittest.TestCase): # returning errors, and is interpreted correctly, for size prefixed # representation and regular: for sizePrefix in [True, False]: - gen_buf, gen_off = make_monster_from_generated_code(sizePrefix = sizePrefix) - CheckReadBuffer(gen_buf, gen_off, sizePrefix = sizePrefix) + for file_identifier in [None, b"MONS"]: + gen_buf, gen_off = make_monster_from_generated_code(sizePrefix=sizePrefix, file_identifier=file_identifier) + CheckReadBuffer(gen_buf, gen_off, sizePrefix=sizePrefix, file_identifier=file_identifier) # Verify that the canonical flatbuffer file is readable by the # generated Python code. Note that context managers are not part of @@ -74,7 +75,7 @@ class TestWireFormat(unittest.TestCase): f = open('monsterdata_test.mon', 'rb') canonicalWireData = f.read() f.close() - CheckReadBuffer(bytearray(canonicalWireData), 0) + CheckReadBuffer(bytearray(canonicalWireData), 0, file_identifier=b'MONS') # Write the generated buffer out to a file: f = open('monsterdata_python_wire.mon', 'wb') @@ -82,7 +83,7 @@ class TestWireFormat(unittest.TestCase): f.close() -def CheckReadBuffer(buf, offset, sizePrefix = False): +def CheckReadBuffer(buf, offset, sizePrefix=False, file_identifier=None): ''' CheckReadBuffer checks that the given buffer is evaluated correctly as the example Monster. ''' @@ -90,12 +91,18 @@ def CheckReadBuffer(buf, offset, sizePrefix = False): ''' An assertion helper that is separated from TestCase classes. ''' if not stmt: raise AssertionError('CheckReadBuffer case failed') - + if file_identifier: + # test prior to removal of size_prefix + asserter(util.GetBufferIdentifier(buf, offset, size_prefixed=sizePrefix) == file_identifier) + asserter(util.BufferHasIdentifier(buf, offset, file_identifier=file_identifier, size_prefixed=sizePrefix)) if sizePrefix: size = util.GetSizePrefix(buf, offset) - # taken from the size of monsterdata_python_wire.mon, minus 4 - asserter(size == 340) + asserter(size == len(buf[offset:])-4) buf, offset = util.RemoveSizePrefix(buf, offset) + if file_identifier: + asserter(MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset)) + else: + asserter(not MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset)) monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset) asserter(monster.Hp() == 80) @@ -1083,7 +1090,7 @@ class TestByteLayout(unittest.TestCase): ]) -def make_monster_from_generated_code(sizePrefix = False): +def make_monster_from_generated_code(sizePrefix = False, file_identifier=None): ''' Use generated code to build the example Monster. ''' b = flatbuffers.Builder(0) @@ -1145,9 +1152,9 @@ def make_monster_from_generated_code(sizePrefix = False): mon = MyGame.Example.Monster.MonsterEnd(b) if sizePrefix: - b.FinishSizePrefixed(mon) + b.FinishSizePrefixed(mon, file_identifier) else: - b.Finish(mon) + b.Finish(mon, file_identifier) return b.Bytes, b.Head()