Python: Added support for file_identifiers (#5123)

* Python: Added support for file_identifiers

* Added tests. Fixed file_identifier code.

* Python: Fixed excessive padding of file_identifier. Repaired tests.

* Python: Made code compatible with python2.7

* Python: Typo fix in @endcond

* whitespace normaalization

* Stylistic change from if(not X is None) to if(X is not None). Added comment to type string.

* Python: Added support for automatic code generation of file_identifiers. Added tests for said code generation.

* converted sprintf to snprintf

* Bugfix, added snprint deffinition for MSVC

* changed snprint deffinition for MSVC to sprint_s

* changed scanf to IntToStringHex. Renamed HasFileIdentifier to GenHasFileIdentifier.

* Added updated genereated code to commit

* Python bugix: flatc no longer produces HasFileIdentfier for shcemas with no file identifier

* Added tests to verify `MonsterBufferHasIdentifier` returns false on no Identifier

* Python: added tests for GetBufferIdentifier and BufferHasIdentifier
Python: removed unessasary parenethesis in if statements
Minor format changes.

* Python : correceted instances of keyword arguments being called as positional arguments

* fixed typos and grammer in comments

* Minor style fixes

* Indentation fix

* Equals style changes

* Python: Fixed Alignment Issues. Changed test code to test against atual output

* Ran make(forgot to run make last commit)

* Python: Style changes

* Style changes

* indentation and style

* readded CONTRIBUTING.md

* Formatting tweak

Mostly to make CI run again

* More formatting fixes

* More formatting fixes

* More formatting fixes

* More formatting fixes

* Formatting fix

* More formatting fixes

* Formatting

* ran generate_code.sh

python/flatbuffers/builder.py

@@ -510,13 +510,21 @@ class Builder(object):
         self.current_vtable[slotnum] = self.Offset()
     ## @endcond
 
-    def __Finish(self, rootTable, sizePrefix):
+    def __Finish(self, rootTable, sizePrefix, file_identifier=None):
         """Finish finalizes a buffer, pointing to the given `rootTable`."""
         N.enforce_number(rootTable, N.UOffsetTFlags)
-        prepSize = N.UOffsetTFlags.bytewidth
+
-        if sizePrefix:
+        if file_identifier is not None:
-            prepSize += N.Int32Flags.bytewidth
+            self.Prep(N.UOffsetTFlags.bytewidth, N.Uint8Flags.bytewidth*4)
-        self.Prep(self.minalign, prepSize)
+            
+            # Convert bytes object file_identifier to an array of 4 8-bit integers,
+            # and use big-endian to enforce size compliance.
+            # https://docs.python.org/2/library/struct.html#format-characters
+            file_identifier = N.struct.unpack(">BBBB", file_identifier)
+            for i in range(encode.FILE_IDENTIFIER_LENGTH-1, -1, -1):
+                # Place the bytes of the file_identifer in reverse order:
+                self.Place(file_identifier[i], N.Uint8Flags)   
+                
         self.PrependUOffsetTRelative(rootTable)
         if sizePrefix:
             size = len(self.Bytes) - self.Head()
@@ -525,16 +533,16 @@ class Builder(object):
         self.finished = True
         return self.Head()
 
-    def Finish(self, rootTable):
+    def Finish(self, rootTable, file_identifier=None):
         """Finish finalizes a buffer, pointing to the given `rootTable`."""
-        return self.__Finish(rootTable, False)
+        return self.__Finish(rootTable, False, file_identifier=file_identifier)
 
-    def FinishSizePrefixed(self, rootTable):
+    def FinishSizePrefixed(self, rootTable, file_identifier=None):
         """
         Finish finalizes a buffer, pointing to the given `rootTable`,
         with the size prefixed.
         """
-        return self.__Finish(rootTable, True)
+        return self.__Finish(rootTable, True, file_identifier=file_identifier)
 
     ## @cond FLATBUFFERS_INTERNAL
     def Prepend(self, flags, off):

python/flatbuffers/encode.py

@@ -19,6 +19,8 @@ from .compat import import_numpy, NumpyRequiredForThisFeature
 
 np = import_numpy()
 
+FILE_IDENTIFIER_LENGTH=4
+
 def Get(packer_type, buf, head):
     """ Get decodes a value at buf[head] using `packer_type`. """
     return packer_type.unpack_from(memoryview_type(buf), head)[0]

python/flatbuffers/util.py

@@ -20,6 +20,21 @@ def GetSizePrefix(buf, offset):
 	"""Extract the size prefix from a buffer."""
 	return encode.Get(packer.int32, buf, offset)
 
+def GetBufferIdentifier(buf, offset, size_prefixed=False):
+        """Extract the file_identifier from a buffer"""
+        if size_prefixed:
+            # increase offset by size of UOffsetTFlags
+            offset += number_types.UOffsetTFlags.bytewidth
+        # increase offset by size of root table pointer
+        offset += number_types.UOffsetTFlags.bytewidth
+        # end of FILE_IDENTIFIER
+        end = offset + encode.FILE_IDENTIFIER_LENGTH
+        return buf[offset:end]
+
+def BufferHasIdentifier(buf, offset, file_identifier, size_prefixed=False):
+        got = GetBufferIdentifier(buf, offset, size_prefixed=size_prefixed)
+        return got == file_identifier
+
 def RemoveSizePrefix(buf, offset):
 	"""
 	Create a slice of a size-prefixed buffer that has

src/idl_gen_python.cpp

@@ -612,6 +612,30 @@ class PythonGenerator : public BaseGenerator {
     GetEndOffsetOnTable(struct_def, code_ptr);
   }
 
+  // Generate function to check for proper file identifier
+  void GenHasFileIdentifier(const StructDef &struct_def,
+                            std::string *code_ptr) {
+    std::string &code = *code_ptr;
+    std::string escapedID;
+    // In the event any of file_identifier characters are special(NULL, \, etc),
+    // problems occur. To prevent this, convert all chars to their hex-escaped
+    // equivalent.
+    for (auto it = parser_.file_identifier_.begin();
+         it != parser_.file_identifier_.end(); ++it) {
+      escapedID += "\\x" + IntToStringHex(*it, 2);
+    }
+
+    code += Indent + "@classmethod\n";
+    code += Indent + "def " + NormalizedName(struct_def);
+    code += "BufferHasIdentifier(cls, buf, offset, size_prefixed=False):";
+    code += "\n";
+    code += Indent + Indent;
+    code += "return flatbuffers.util.BufferHasIdentifier(buf, offset, b\"";
+    code += escapedID;
+    code += "\", size_prefixed=size_prefixed)\n";
+    code += "\n";
+  }
+  
   // Generate struct or table methods.
   void GenStruct(const StructDef &struct_def, std::string *code_ptr) {
     if (struct_def.generated) return;
@@ -622,6 +646,10 @@ class PythonGenerator : public BaseGenerator {
       // Generate a special accessor for the table that has been declared as
       // the root type.
       NewRootTypeFromBuffer(struct_def, code_ptr);
+      if (parser_.file_identifier_.length()){
+        // Generate a special function to test file_identifier
+        GenHasFileIdentifier(struct_def, code_ptr);
+      }
     }
     // Generate the Init method that sets the field in a pre-existing
     // accessor object. This is to allow object reuse.

tests/MyGame/Example/ArrayTable.py

@@ -14,6 +14,10 @@ class ArrayTable(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def ArrayTableBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x41\x52\x52\x54", size_prefixed=size_prefixed)
+
     # ArrayTable
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/Monster.py

@@ -15,6 +15,10 @@ class Monster(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Monster
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/Referrable.py

@@ -14,6 +14,10 @@ class Referrable(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def ReferrableBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Referrable
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/Stat.py

@@ -14,6 +14,10 @@ class Stat(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def StatBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Stat
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/TestSimpleTableWithEnum.py

@@ -14,6 +14,10 @@ class TestSimpleTableWithEnum(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def TestSimpleTableWithEnumBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # TestSimpleTableWithEnum
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/TypeAliases.py

@@ -14,6 +14,10 @@ class TypeAliases(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def TypeAliasesBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # TypeAliases
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example2/Monster.py

@@ -14,6 +14,10 @@ class Monster(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Monster
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/InParentNamespace.py

@@ -14,6 +14,10 @@ class InParentNamespace(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def InParentNamespaceBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # InParentNamespace
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/MonsterExtra.py

@@ -14,6 +14,10 @@ class MonsterExtra(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def MonsterExtraBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x45", size_prefixed=size_prefixed)
+
     # MonsterExtra
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/py_test.py

@@ -65,8 +65,9 @@ class TestWireFormat(unittest.TestCase):
         # returning errors, and is interpreted correctly, for size prefixed
         # representation and regular:
         for sizePrefix in [True, False]:
-            gen_buf, gen_off = make_monster_from_generated_code(sizePrefix = sizePrefix)
+            for file_identifier in [None, b"MONS"]:
-            CheckReadBuffer(gen_buf, gen_off, sizePrefix = sizePrefix)
+                gen_buf, gen_off = make_monster_from_generated_code(sizePrefix=sizePrefix, file_identifier=file_identifier)
+                CheckReadBuffer(gen_buf, gen_off, sizePrefix=sizePrefix, file_identifier=file_identifier)
 
         # Verify that the canonical flatbuffer file is readable by the
         # generated Python code. Note that context managers are not part of
@@ -74,7 +75,7 @@ class TestWireFormat(unittest.TestCase):
         f = open('monsterdata_test.mon', 'rb')
         canonicalWireData = f.read()
         f.close()
-        CheckReadBuffer(bytearray(canonicalWireData), 0)
+        CheckReadBuffer(bytearray(canonicalWireData), 0, file_identifier=b'MONS')
 
         # Write the generated buffer out to a file:
         f = open('monsterdata_python_wire.mon', 'wb')
@@ -82,7 +83,7 @@ class TestWireFormat(unittest.TestCase):
         f.close()
 
 
-def CheckReadBuffer(buf, offset, sizePrefix = False):
+def CheckReadBuffer(buf, offset, sizePrefix=False, file_identifier=None):
     ''' CheckReadBuffer checks that the given buffer is evaluated correctly
         as the example Monster. '''
 
@@ -90,12 +91,18 @@ def CheckReadBuffer(buf, offset, sizePrefix = False):
         ''' An assertion helper that is separated from TestCase classes. '''
         if not stmt:
             raise AssertionError('CheckReadBuffer case failed')
-
+    if file_identifier:
+        # test prior to removal of size_prefix
+        asserter(util.GetBufferIdentifier(buf, offset, size_prefixed=sizePrefix) == file_identifier)
+        asserter(util.BufferHasIdentifier(buf, offset, file_identifier=file_identifier, size_prefixed=sizePrefix))
     if sizePrefix:
         size = util.GetSizePrefix(buf, offset)
-        # taken from the size of monsterdata_python_wire.mon, minus 4
+        asserter(size == len(buf[offset:])-4)
-        asserter(size == 340)
         buf, offset = util.RemoveSizePrefix(buf, offset)
+    if file_identifier:
+        asserter(MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset))
+    else:
+        asserter(not MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset))
     monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset)
 
     asserter(monster.Hp() == 80)
@@ -1083,7 +1090,7 @@ class TestByteLayout(unittest.TestCase):
         ])
 
 
-def make_monster_from_generated_code(sizePrefix = False):
+def make_monster_from_generated_code(sizePrefix = False, file_identifier=None):
     ''' Use generated code to build the example Monster. '''
 
     b = flatbuffers.Builder(0)
@@ -1145,9 +1152,9 @@ def make_monster_from_generated_code(sizePrefix = False):
     mon = MyGame.Example.Monster.MonsterEnd(b)
 
     if sizePrefix:
-        b.FinishSizePrefixed(mon)
+        b.FinishSizePrefixed(mon, file_identifier)
     else:
-        b.Finish(mon)
+        b.Finish(mon, file_identifier)
 
     return b.Bytes, b.Head()