Rooba
/
flatbuffers
mirror of https://github.com/google/flatbuffers.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Python: Added support for file_identifiers (#5123) 

* Python: Added support for file_identifiers

* Added tests. Fixed file_identifier code.

* Python: Fixed excessive padding of file_identifier. Repaired tests.

* Python: Made code compatible with python2.7

* Python: Typo fix in @endcond

* whitespace normaalization

* Stylistic change from if(not X is None) to if(X is not None). Added comment to type string.

* Python: Added support for automatic code generation of file_identifiers. Added tests for said code generation.

* converted sprintf to snprintf

* Bugfix, added snprint deffinition for MSVC

* changed snprint deffinition for MSVC to sprint_s

* changed scanf to IntToStringHex. Renamed HasFileIdentifier to GenHasFileIdentifier.

* Added updated genereated code to commit

* Python bugix: flatc no longer produces HasFileIdentfier for shcemas with no file identifier

* Added tests to verify `MonsterBufferHasIdentifier` returns false on no Identifier

* Python: added tests for GetBufferIdentifier and BufferHasIdentifier
Python: removed unessasary parenethesis in if statements
Minor format changes.

* Python : correceted instances of keyword arguments being called as positional arguments

* fixed typos and grammer in comments

* Minor style fixes

* Indentation fix

* Equals style changes

* Python: Fixed Alignment Issues. Changed test code to test against atual output

* Ran make(forgot to run make last commit)

* Python: Style changes

* Style changes

* indentation and style

* readded CONTRIBUTING.md

* Formatting tweak

Mostly to make CI run again

* More formatting fixes

* More formatting fixes

* More formatting fixes

* More formatting fixes

* Formatting fix

* More formatting fixes

* Formatting

* ran generate_code.sh
This commit is contained in:
Joseph Pyott 2019-07-26 14:06:25 -04:00 committed by Wouter van Oortmerssen
parent a5ca8bee4d
commit 9fa8245e81
15 changed files with 115 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
python/flatbuffers/builder.py
View File

@ -510,13 +510,21 @@ class Builder(object):
self.current_vtable[slotnum] = self.Offset()
## @endcond
def __Finish(self, rootTable, sizePrefix):
def __Finish(self, rootTable, sizePrefix, file_identifier=None):
"""Finish finalizes a buffer, pointing to the given `rootTable`."""
N.enforce_number(rootTable, N.UOffsetTFlags)
prepSize = N.UOffsetTFlags.bytewidth
if sizePrefix:
prepSize += N.Int32Flags.bytewidth
self.Prep(self.minalign, prepSize)
if file_identifier is not None:
self.Prep(N.UOffsetTFlags.bytewidth, N.Uint8Flags.bytewidth*4)
# Convert bytes object file_identifier to an array of 4 8-bit integers,
# and use big-endian to enforce size compliance.
# https://docs.python.org/2/library/struct.html#format-characters
file_identifier = N.struct.unpack(">BBBB", file_identifier)
for i in range(encode.FILE_IDENTIFIER_LENGTH-1, -1, -1):
# Place the bytes of the file_identifer in reverse order:
self.Place(file_identifier[i], N.Uint8Flags)
self.PrependUOffsetTRelative(rootTable)
if sizePrefix:
size = len(self.Bytes) - self.Head()
@ -525,16 +533,16 @@ class Builder(object):
self.finished = True
return self.Head()
def Finish(self, rootTable):
def Finish(self, rootTable, file_identifier=None):
"""Finish finalizes a buffer, pointing to the given `rootTable`."""
return self.__Finish(rootTable, False)
return self.__Finish(rootTable, False, file_identifier=file_identifier)
def FinishSizePrefixed(self, rootTable):
def FinishSizePrefixed(self, rootTable, file_identifier=None):
"""
Finish finalizes a buffer, pointing to the given `rootTable`,
with the size prefixed.
"""
return self.__Finish(rootTable, True)
return self.__Finish(rootTable, True, file_identifier=file_identifier)
## @cond FLATBUFFERS_INTERNAL
def Prepend(self, flags, off):

2
python/flatbuffers/encode.py
View File

@ -19,6 +19,8 @@ from .compat import import_numpy, NumpyRequiredForThisFeature
np = import_numpy()
FILE_IDENTIFIER_LENGTH=4
def Get(packer_type, buf, head):
""" Get decodes a value at buf[head] using `packer_type`. """
return packer_type.unpack_from(memoryview_type(buf), head)[0]

15
python/flatbuffers/util.py
View File

@ -20,6 +20,21 @@ def GetSizePrefix(buf, offset):
"""Extract the size prefix from a buffer."""
return encode.Get(packer.int32, buf, offset)
def GetBufferIdentifier(buf, offset, size_prefixed=False):
"""Extract the file_identifier from a buffer"""
if size_prefixed:
# increase offset by size of UOffsetTFlags
offset += number_types.UOffsetTFlags.bytewidth
# increase offset by size of root table pointer
offset += number_types.UOffsetTFlags.bytewidth
# end of FILE_IDENTIFIER
end = offset + encode.FILE_IDENTIFIER_LENGTH
return buf[offset:end]
def BufferHasIdentifier(buf, offset, file_identifier, size_prefixed=False):
got = GetBufferIdentifier(buf, offset, size_prefixed=size_prefixed)
return got == file_identifier
def RemoveSizePrefix(buf, offset):
"""
Create a slice of a size-prefixed buffer that has

28
src/idl_gen_python.cpp
View File

@ -612,6 +612,30 @@ class PythonGenerator : public BaseGenerator {
GetEndOffsetOnTable(struct_def, code_ptr);
}
// Generate function to check for proper file identifier
void GenHasFileIdentifier(const StructDef &struct_def,
std::string *code_ptr) {
std::string &code = *code_ptr;
std::string escapedID;
// In the event any of file_identifier characters are special(NULL, \, etc),
// problems occur. To prevent this, convert all chars to their hex-escaped
// equivalent.
for (auto it = parser_.file_identifier_.begin();
it != parser_.file_identifier_.end(); ++it) {
escapedID += "\\x" + IntToStringHex(*it, 2);
}
code += Indent + "@classmethod\n";
code += Indent + "def " + NormalizedName(struct_def);
code += "BufferHasIdentifier(cls, buf, offset, size_prefixed=False):";
code += "\n";
code += Indent + Indent;
code += "return flatbuffers.util.BufferHasIdentifier(buf, offset, b\"";
code += escapedID;
code += "\", size_prefixed=size_prefixed)\n";
code += "\n";
}
// Generate struct or table methods.
void GenStruct(const StructDef &struct_def, std::string *code_ptr) {
if (struct_def.generated) return;
@ -622,6 +646,10 @@ class PythonGenerator : public BaseGenerator {
// Generate a special accessor for the table that has been declared as
// the root type.
NewRootTypeFromBuffer(struct_def, code_ptr);
if (parser_.file_identifier_.length()){
// Generate a special function to test file_identifier
GenHasFileIdentifier(struct_def, code_ptr);
}
}
// Generate the Init method that sets the field in a pre-existing
// accessor object. This is to allow object reuse.

4
tests/MyGame/Example/ArrayTable.py
View File

@ -14,6 +14,10 @@ class ArrayTable(object):
x.Init(buf, n + offset)
return x
@classmethod
def ArrayTableBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x41\x52\x52\x54", size_prefixed=size_prefixed)
# ArrayTable
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/Example/Monster.py
View File

@ -15,6 +15,10 @@ class Monster(object):
x.Init(buf, n + offset)
return x
@classmethod
def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
# Monster
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/Example/Referrable.py
View File

@ -14,6 +14,10 @@ class Referrable(object):
x.Init(buf, n + offset)
return x
@classmethod
def ReferrableBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
# Referrable
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/Example/Stat.py
View File

@ -14,6 +14,10 @@ class Stat(object):
x.Init(buf, n + offset)
return x
@classmethod
def StatBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
# Stat
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/Example/TestSimpleTableWithEnum.py
View File

@ -14,6 +14,10 @@ class TestSimpleTableWithEnum(object):
x.Init(buf, n + offset)
return x
@classmethod
def TestSimpleTableWithEnumBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
# TestSimpleTableWithEnum
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/Example/TypeAliases.py
View File

@ -14,6 +14,10 @@ class TypeAliases(object):
x.Init(buf, n + offset)
return x
@classmethod
def TypeAliasesBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
# TypeAliases
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/Example2/Monster.py
View File

@ -14,6 +14,10 @@ class Monster(object):
x.Init(buf, n + offset)
return x
@classmethod
def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
# Monster
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/InParentNamespace.py
View File

@ -14,6 +14,10 @@ class InParentNamespace(object):
x.Init(buf, n + offset)
return x
@classmethod
def InParentNamespaceBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
# InParentNamespace
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

4
tests/MyGame/MonsterExtra.py
View File

@ -14,6 +14,10 @@ class MonsterExtra(object):
x.Init(buf, n + offset)
return x
@classmethod
def MonsterExtraBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x45", size_prefixed=size_prefixed)
# MonsterExtra
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)

BIN
tests/monsterdata_python_wire.mon
View File

Binary file not shown.

27
tests/py_test.py
View File

@ -65,8 +65,9 @@ class TestWireFormat(unittest.TestCase):
# returning errors, and is interpreted correctly, for size prefixed
# representation and regular:
for sizePrefix in [True, False]:
gen_buf, gen_off = make_monster_from_generated_code(sizePrefix = sizePrefix)
CheckReadBuffer(gen_buf, gen_off, sizePrefix = sizePrefix)
for file_identifier in [None, b"MONS"]:
gen_buf, gen_off = make_monster_from_generated_code(sizePrefix=sizePrefix, file_identifier=file_identifier)
CheckReadBuffer(gen_buf, gen_off, sizePrefix=sizePrefix, file_identifier=file_identifier)
# Verify that the canonical flatbuffer file is readable by the
# generated Python code. Note that context managers are not part of
@ -74,7 +75,7 @@ class TestWireFormat(unittest.TestCase):
f = open('monsterdata_test.mon', 'rb')
canonicalWireData = f.read()
f.close()
CheckReadBuffer(bytearray(canonicalWireData), 0)
CheckReadBuffer(bytearray(canonicalWireData), 0, file_identifier=b'MONS')
# Write the generated buffer out to a file:
f = open('monsterdata_python_wire.mon', 'wb')
@ -82,7 +83,7 @@ class TestWireFormat(unittest.TestCase):
f.close()
def CheckReadBuffer(buf, offset, sizePrefix = False):
def CheckReadBuffer(buf, offset, sizePrefix=False, file_identifier=None):
''' CheckReadBuffer checks that the given buffer is evaluated correctly
as the example Monster. '''
@ -90,12 +91,18 @@ def CheckReadBuffer(buf, offset, sizePrefix = False):
''' An assertion helper that is separated from TestCase classes. '''
if not stmt:
raise AssertionError('CheckReadBuffer case failed')
if file_identifier:
# test prior to removal of size_prefix
asserter(util.GetBufferIdentifier(buf, offset, size_prefixed=sizePrefix) == file_identifier)
asserter(util.BufferHasIdentifier(buf, offset, file_identifier=file_identifier, size_prefixed=sizePrefix))
if sizePrefix:
size = util.GetSizePrefix(buf, offset)
# taken from the size of monsterdata_python_wire.mon, minus 4
asserter(size == 340)
asserter(size == len(buf[offset:])-4)
buf, offset = util.RemoveSizePrefix(buf, offset)
if file_identifier:
asserter(MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset))
else:
asserter(not MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset))
monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset)
asserter(monster.Hp() == 80)
@ -1083,7 +1090,7 @@ class TestByteLayout(unittest.TestCase):
])
def make_monster_from_generated_code(sizePrefix = False):
def make_monster_from_generated_code(sizePrefix = False, file_identifier=None):
''' Use generated code to build the example Monster. '''
b = flatbuffers.Builder(0)
@ -1145,9 +1152,9 @@ def make_monster_from_generated_code(sizePrefix = False):
mon = MyGame.Example.Monster.MonsterEnd(b)
if sizePrefix:
b.FinishSizePrefixed(mon)
b.FinishSizePrefixed(mon, file_identifier)
else:
b.Finish(mon)
b.Finish(mon, file_identifier)
return b.Bytes, b.Head()