From a96f2bd6ca97df1550ad718402de9d507fdfecc3 Mon Sep 17 00:00:00 2001 From: Michael Seifert Date: Fri, 8 Jun 2018 19:55:19 +0200 Subject: [PATCH] Python: Escape enum member names if they correspond to a Python keyword (#4772) * Python: Generated enum member names are now escaped if they correspond to a Python keyword. * Keyword list in Python generator is now a const char* instead of an std::string array. * Moved static functions and keyword list of Python generator into the PythonGenerator class. * Python generator escapes keyword identifiers in all definitions. --- src/idl_gen_python.cpp | 1201 +++++++++++++++++++++------------------- 1 file changed, 623 insertions(+), 578 deletions(-) diff --git a/src/idl_gen_python.cpp b/src/idl_gen_python.cpp index 419270c46..d2f518f2f 100644 --- a/src/idl_gen_python.cpp +++ b/src/idl_gen_python.cpp @@ -23,594 +23,637 @@ #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include + namespace flatbuffers { namespace python { -static std::string GenGetter(const Type &type); -static std::string GenMethod(const FieldDef &field); -static void GenStructBuilder(const StructDef &struct_def, - std::string *code_ptr); -static void GenReceiver(const StructDef &struct_def, std::string *code_ptr); -static std::string GenTypeBasic(const Type &type); -static std::string GenTypeGet(const Type &type); -static std::string TypeName(const FieldDef &field); - // Hardcode spaces per indentation. const std::string Indent = " "; -// Most field accessors need to retrieve and test the field offset first, -// this is the prefix code for that. -std::string OffsetPrefix(const FieldDef &field) { - return "\n" + Indent + Indent + - "o = flatbuffers.number_types.UOffsetTFlags.py_type" + - "(self._tab.Offset(" + NumToString(field.value.offset) + "))\n" + - Indent + Indent + "if o != 0:\n"; -} - -// Begin a class declaration. -static void BeginClass(const StructDef &struct_def, std::string *code_ptr) { - std::string &code = *code_ptr; - code += "class " + struct_def.name + "(object):\n"; - code += Indent + "__slots__ = ['_tab']"; - code += "\n\n"; -} - -// Begin enum code with a class declaration. -static void BeginEnum(const std::string class_name, std::string *code_ptr) { - std::string &code = *code_ptr; - code += "class " + class_name + "(object):\n"; -} - -// A single enum member. -static void EnumMember(const EnumVal ev, std::string *code_ptr) { - std::string &code = *code_ptr; - code += Indent; - code += ev.name; - code += " = "; - code += NumToString(ev.value) + "\n"; -} - -// End enum code. -static void EndEnum(std::string *code_ptr) { - std::string &code = *code_ptr; - code += "\n"; -} - -// Initialize a new struct or table from existing data. -static void NewRootTypeFromBuffer(const StructDef &struct_def, - std::string *code_ptr) { - std::string &code = *code_ptr; - - code += Indent + "@classmethod\n"; - code += Indent + "def GetRootAs"; - code += struct_def.name; - code += "(cls, buf, offset):"; - code += "\n"; - code += Indent + Indent; - code += "n = flatbuffers.encode.Get"; - code += "(flatbuffers.packer.uoffset, buf, offset)\n"; - code += Indent + Indent + "x = " + struct_def.name + "()\n"; - code += Indent + Indent + "x.Init(buf, n + offset)\n"; - code += Indent + Indent + "return x\n"; - code += "\n"; -} - -// Initialize an existing object with other data, to avoid an allocation. -static void InitializeExisting(const StructDef &struct_def, - std::string *code_ptr) { - std::string &code = *code_ptr; - - GenReceiver(struct_def, code_ptr); - code += "Init(self, buf, pos):\n"; - code += Indent + Indent + "self._tab = flatbuffers.table.Table(buf, pos)\n"; - code += "\n"; -} - -// Get the length of a vector. -static void GetVectorLen(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name) + "Length(self"; - code += "):" + OffsetPrefix(field); - code += Indent + Indent + Indent + "return self._tab.VectorLen(o)\n"; - code += Indent + Indent + "return 0\n\n"; -} - -// Get the value of a struct's scalar. -static void GetScalarFieldOfStruct(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - std::string getter = GenGetter(field.value.type); - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name); - code += "(self): return " + getter; - code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; - code += NumToString(field.value.offset) + "))\n"; -} - -// Get the value of a table's scalar. -static void GetScalarFieldOfTable(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - std::string getter = GenGetter(field.value.type); - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name); - code += "(self):"; - code += OffsetPrefix(field); - getter += "o + self._tab.Pos)"; - auto is_bool = field.value.type.base_type == BASE_TYPE_BOOL; - if (is_bool) { - getter = "bool(" + getter + ")"; - } - code += Indent + Indent + Indent + "return " + getter + "\n"; - std::string default_value; - if (is_bool) { - default_value = field.value.constant == "0" ? "False" : "True"; - } else { - default_value = field.value.constant; - } - code += Indent + Indent + "return " + default_value + "\n\n"; -} - -// Get a struct by initializing an existing struct. -// Specific to Struct. -static void GetStructFieldOfStruct(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name); - code += "(self, obj):\n"; - code += Indent + Indent + "obj.Init(self._tab.Bytes, self._tab.Pos + "; - code += NumToString(field.value.offset) + ")"; - code += "\n" + Indent + Indent + "return obj\n\n"; -} - -// Get a struct by initializing an existing struct. -// Specific to Table. -static void GetStructFieldOfTable(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name); - code += "(self):"; - code += OffsetPrefix(field); - if (field.value.type.struct_def->fixed) { - code += Indent + Indent + Indent + "x = o + self._tab.Pos\n"; - } else { - code += Indent + Indent + Indent; - code += "x = self._tab.Indirect(o + self._tab.Pos)\n"; - } - code += Indent + Indent + Indent; - code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; - code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; - code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; - code += Indent + Indent + Indent + "return obj\n"; - code += Indent + Indent + "return None\n\n"; -} - -// Get the value of a string. -static void GetStringField(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name); - code += "(self):"; - code += OffsetPrefix(field); - code += Indent + Indent + Indent + "return " + GenGetter(field.value.type); - code += "o + self._tab.Pos)\n"; - code += Indent + Indent + "return None\n\n"; -} - -// Get the value of a union from an object. -static void GetUnionField(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name) + "(self):"; - code += OffsetPrefix(field); - - // TODO(rw): this works and is not the good way to it: - bool is_native_table = TypeName(field) == "*flatbuffers.Table"; - if (is_native_table) { - code += Indent + Indent + Indent + "from flatbuffers.table import Table\n"; - } else { - code += Indent + Indent + Indent; - code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; - } - code += Indent + Indent + Indent + "obj = Table(bytearray(), 0)\n"; - code += Indent + Indent + Indent + GenGetter(field.value.type); - code += "obj, o)\n" + Indent + Indent + Indent + "return obj\n"; - code += Indent + Indent + "return None\n\n"; -} - -// Get the value of a vector's struct member. -static void GetMemberOfVectorOfStruct(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - auto vectortype = field.value.type.VectorType(); - - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name); - code += "(self, j):" + OffsetPrefix(field); - code += Indent + Indent + Indent + "x = self._tab.Vector(o)\n"; - code += Indent + Indent + Indent; - code += "x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * "; - code += NumToString(InlineSize(vectortype)) + "\n"; - if (!(vectortype.struct_def->fixed)) { - code += Indent + Indent + Indent + "x = self._tab.Indirect(x)\n"; - } - code += Indent + Indent + Indent; - code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; - code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; - code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; - code += Indent + Indent + Indent + "return obj\n"; - code += Indent + Indent + "return None\n\n"; -} - -// Get the value of a vector's non-struct member. Uses a named return -// argument to conveniently set the zero value for the result. -static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - auto vectortype = field.value.type.VectorType(); - - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name); - code += "(self, j):"; - code += OffsetPrefix(field); - code += Indent + Indent + Indent + "a = self._tab.Vector(o)\n"; - code += Indent + Indent + Indent; - code += "return " + GenGetter(field.value.type); - code += "a + flatbuffers.number_types.UOffsetTFlags.py_type(j * "; - code += NumToString(InlineSize(vectortype)) + "))\n"; - if (vectortype.base_type == BASE_TYPE_STRING) { - code += Indent + Indent + "return \"\"\n"; - } else { - code += Indent + Indent + "return 0\n"; - } - code += "\n"; -} - -// Returns a non-struct vector as a numpy array. Much faster -// than iterating over the vector element by element. -static void GetVectorOfNonStructAsNumpy(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { - std::string &code = *code_ptr; - auto vectortype = field.value.type.VectorType(); - - // Currently, we only support accessing as numpy array if - // the vector type is a scalar. - if (!(IsScalar(vectortype.base_type))) { return; } - - GenReceiver(struct_def, code_ptr); - code += MakeCamel(field.name) + "AsNumpy(self):"; - code += OffsetPrefix(field); - - code += Indent + Indent + Indent; - code += "return "; - code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types."; - code += MakeCamel(GenTypeGet(field.value.type)); - code += "Flags, o)\n"; - - if (vectortype.base_type == BASE_TYPE_STRING) { - code += Indent + Indent + "return \"\"\n"; - } else { - code += Indent + Indent + "return 0\n"; - } - code += "\n"; -} - -// Begin the creator function signature. -static void BeginBuilderArgs(const StructDef &struct_def, - std::string *code_ptr) { - std::string &code = *code_ptr; - - code += "\n"; - code += "def Create" + struct_def.name; - code += "(builder"; -} - -// Recursively generate arguments for a constructor, to deal with nested -// structs. -static void StructBuilderArgs(const StructDef &struct_def, - const char *nameprefix, std::string *code_ptr) { - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; - if (IsStruct(field.value.type)) { - // Generate arguments for a struct inside a struct. To ensure names - // don't clash, and to make it obvious these arguments are constructing - // a nested struct, prefix the name with the field name. - StructBuilderArgs(*field.value.type.struct_def, - (nameprefix + (field.name + "_")).c_str(), code_ptr); - } else { - std::string &code = *code_ptr; - code += (std::string) ", " + nameprefix; - code += MakeCamel(field.name, false); - } - } -} - -// End the creator function signature. -static void EndBuilderArgs(std::string *code_ptr) { - std::string &code = *code_ptr; - code += "):\n"; -} - -// Recursively generate struct construction statements and instert manual -// padding. -static void StructBuilderBody(const StructDef &struct_def, - const char *nameprefix, std::string *code_ptr) { - std::string &code = *code_ptr; - code += " builder.Prep(" + NumToString(struct_def.minalign) + ", "; - code += NumToString(struct_def.bytesize) + ")\n"; - for (auto it = struct_def.fields.vec.rbegin(); - it != struct_def.fields.vec.rend(); ++it) { - auto &field = **it; - if (field.padding) - code += " builder.Pad(" + NumToString(field.padding) + ")\n"; - if (IsStruct(field.value.type)) { - StructBuilderBody(*field.value.type.struct_def, - (nameprefix + (field.name + "_")).c_str(), code_ptr); - } else { - code += " builder.Prepend" + GenMethod(field) + "("; - code += nameprefix + MakeCamel(field.name, false) + ")\n"; - } - } -} - -static void EndBuilderBody(std::string *code_ptr) { - std::string &code = *code_ptr; - code += " return builder.Offset()\n"; -} - -// Get the value of a table's starting offset. -static void GetStartOfTable(const StructDef &struct_def, - std::string *code_ptr) { - std::string &code = *code_ptr; - code += "def " + struct_def.name + "Start"; - code += "(builder): "; - code += "builder.StartObject("; - code += NumToString(struct_def.fields.vec.size()); - code += ")\n"; -} - -// Set the value of a table's field. -static void BuildFieldOfTable(const StructDef &struct_def, - const FieldDef &field, const size_t offset, - std::string *code_ptr) { - std::string &code = *code_ptr; - code += "def " + struct_def.name + "Add" + MakeCamel(field.name); - code += "(builder, "; - code += MakeCamel(field.name, false); - code += "): "; - code += "builder.Prepend"; - code += GenMethod(field) + "Slot("; - code += NumToString(offset) + ", "; - if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { - code += "flatbuffers.number_types.UOffsetTFlags.py_type"; - code += "("; - code += MakeCamel(field.name, false) + ")"; - } else { - code += MakeCamel(field.name, false); - } - code += ", " + field.value.constant; - code += ")\n"; -} - -// Set the value of one of the members of a table's vector. -static void BuildVectorOfTable(const StructDef &struct_def, - const FieldDef &field, std::string *code_ptr) { - std::string &code = *code_ptr; - code += "def " + struct_def.name + "Start"; - code += MakeCamel(field.name); - code += "Vector(builder, numElems): return builder.StartVector("; - auto vector_type = field.value.type.VectorType(); - auto alignment = InlineAlignment(vector_type); - auto elem_size = InlineSize(vector_type); - code += NumToString(elem_size); - code += ", numElems, " + NumToString(alignment); - code += ")\n"; -} - -// Get the offset of the end of a table. -static void GetEndOffsetOnTable(const StructDef &struct_def, - std::string *code_ptr) { - std::string &code = *code_ptr; - code += "def " + struct_def.name + "End"; - code += "(builder): "; - code += "return builder.EndObject()\n"; -} - -// Generate the receiver for function signatures. -static void GenReceiver(const StructDef &struct_def, std::string *code_ptr) { - std::string &code = *code_ptr; - code += Indent + "# " + struct_def.name + "\n"; - code += Indent + "def "; -} - -// Generate a struct field, conditioned on its child type(s). -static void GenStructAccessor(const StructDef &struct_def, - const FieldDef &field, std::string *code_ptr) { - GenComment(field.doc_comment, code_ptr, nullptr, "# "); - if (IsScalar(field.value.type.base_type)) { - if (struct_def.fixed) { - GetScalarFieldOfStruct(struct_def, field, code_ptr); - } else { - GetScalarFieldOfTable(struct_def, field, code_ptr); - } - } else { - switch (field.value.type.base_type) { - case BASE_TYPE_STRUCT: - if (struct_def.fixed) { - GetStructFieldOfStruct(struct_def, field, code_ptr); - } else { - GetStructFieldOfTable(struct_def, field, code_ptr); - } - break; - case BASE_TYPE_STRING: GetStringField(struct_def, field, code_ptr); break; - case BASE_TYPE_VECTOR: { - auto vectortype = field.value.type.VectorType(); - if (vectortype.base_type == BASE_TYPE_STRUCT) { - GetMemberOfVectorOfStruct(struct_def, field, code_ptr); - } else { - GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr); - GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr); - } - break; - } - case BASE_TYPE_UNION: GetUnionField(struct_def, field, code_ptr); break; - default: FLATBUFFERS_ASSERT(0); - } - } - if (field.value.type.base_type == BASE_TYPE_VECTOR) { - GetVectorLen(struct_def, field, code_ptr); - } -} - -// Generate table constructors, conditioned on its members' types. -static void GenTableBuilders(const StructDef &struct_def, - std::string *code_ptr) { - GetStartOfTable(struct_def, code_ptr); - - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; - if (field.deprecated) continue; - - auto offset = it - struct_def.fields.vec.begin(); - BuildFieldOfTable(struct_def, field, offset, code_ptr); - if (field.value.type.base_type == BASE_TYPE_VECTOR) { - BuildVectorOfTable(struct_def, field, code_ptr); - } - } - - GetEndOffsetOnTable(struct_def, code_ptr); -} - -// Generate struct or table methods. -static void GenStruct(const StructDef &struct_def, std::string *code_ptr) { - if (struct_def.generated) return; - - GenComment(struct_def.doc_comment, code_ptr, nullptr, "# "); - BeginClass(struct_def, code_ptr); - if (!struct_def.fixed) { - // Generate a special accessor for the table that has been declared as - // the root type. - NewRootTypeFromBuffer(struct_def, code_ptr); - } - // Generate the Init method that sets the field in a pre-existing - // accessor object. This is to allow object reuse. - InitializeExisting(struct_def, code_ptr); - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; - if (field.deprecated) continue; - - GenStructAccessor(struct_def, field, code_ptr); - } - - if (struct_def.fixed) { - // create a struct constructor function - GenStructBuilder(struct_def, code_ptr); - } else { - // Create a set of functions that allow table construction. - GenTableBuilders(struct_def, code_ptr); - } -} - -// Generate enum declarations. -static void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { - if (enum_def.generated) return; - - GenComment(enum_def.doc_comment, code_ptr, nullptr, "# "); - BeginEnum(enum_def.name, code_ptr); - for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); - ++it) { - auto &ev = **it; - GenComment(ev.doc_comment, code_ptr, nullptr, "# "); - EnumMember(ev, code_ptr); - } - EndEnum(code_ptr); -} - -// Returns the function name that is able to read a value of the given type. -static std::string GenGetter(const Type &type) { - switch (type.base_type) { - case BASE_TYPE_STRING: return "self._tab.String("; - case BASE_TYPE_UNION: return "self._tab.Union("; - case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); - default: - return "self._tab.Get(flatbuffers.number_types." + - MakeCamel(GenTypeGet(type)) + "Flags, "; - } -} - -// Returns the method name for use with add/put calls. -static std::string GenMethod(const FieldDef &field) { - return IsScalar(field.value.type.base_type) - ? MakeCamel(GenTypeBasic(field.value.type)) - : (IsStruct(field.value.type) ? "Struct" : "UOffsetTRelative"); -} - -static std::string GenTypeBasic(const Type &type) { - static const char *ctypename[] = { - // clang-format off - #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ - #PTYPE, - FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) - #undef FLATBUFFERS_TD - // clang-format on - }; - return ctypename[type.base_type]; -} - -static std::string GenTypePointer(const Type &type) { - switch (type.base_type) { - case BASE_TYPE_STRING: return "string"; - case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); - case BASE_TYPE_STRUCT: return type.struct_def->name; - case BASE_TYPE_UNION: - // fall through - default: return "*flatbuffers.Table"; - } -} - -static std::string GenTypeGet(const Type &type) { - return IsScalar(type.base_type) ? GenTypeBasic(type) : GenTypePointer(type); -} - -static std::string TypeName(const FieldDef &field) { - return GenTypeGet(field.value.type); -} - -// Create a struct with a builder and the struct's arguments. -static void GenStructBuilder(const StructDef &struct_def, - std::string *code_ptr) { - BeginBuilderArgs(struct_def, code_ptr); - StructBuilderArgs(struct_def, "", code_ptr); - EndBuilderArgs(code_ptr); - - StructBuilderBody(struct_def, "", code_ptr); - EndBuilderBody(code_ptr); -} - class PythonGenerator : public BaseGenerator { public: PythonGenerator(const Parser &parser, const std::string &path, const std::string &file_name) : BaseGenerator(parser, path, file_name, "" /* not used */, - "" /* not used */){}; + "" /* not used */){ + static const char * const keywords[] = { + "False", + "None", + "True", + "and", + "as", + "assert", + "break", + "class", + "continue", + "def", + "del", + "elif", + "else", + "except", + "finally", + "for", + "from", + "global", + "if", + "import", + "in", + "is", + "lambda", + "nonlocal", + "not", + "or", + "pass", + "raise", + "return", + "try", + "while", + "with", + "yield" + }; + for (auto kw = keywords; *kw; kw++) keywords_.insert(*kw); + } + + // Most field accessors need to retrieve and test the field offset first, + // this is the prefix code for that. + std::string OffsetPrefix(const FieldDef &field) { + return "\n" + Indent + Indent + + "o = flatbuffers.number_types.UOffsetTFlags.py_type" + + "(self._tab.Offset(" + NumToString(field.value.offset) + "))\n" + + Indent + Indent + "if o != 0:\n"; + } + + // Begin a class declaration. + void BeginClass(const StructDef &struct_def, std::string *code_ptr) { + std::string &code = *code_ptr; + code += "class " + NormalizedName(struct_def) + "(object):\n"; + code += Indent + "__slots__ = ['_tab']"; + code += "\n\n"; + } + + // Begin enum code with a class declaration. + void BeginEnum(const std::string class_name, std::string *code_ptr) { + std::string &code = *code_ptr; + code += "class " + class_name + "(object):\n"; + } + + std::string EscapeKeyword(const std::string &name) const { + return keywords_.find(name) == keywords_.end() ? name : name + "_"; + } + + std::string NormalizedName(const Definition &definition) const { + return EscapeKeyword(definition.name); + } + + std::string NormalizedName(const EnumVal &ev) const { + return EscapeKeyword(ev.name); + } + + // A single enum member. + void EnumMember(const EnumVal ev, std::string *code_ptr) { + std::string &code = *code_ptr; + code += Indent; + code += NormalizedName(ev); + code += " = "; + code += NumToString(ev.value) + "\n"; + } + + // End enum code. + void EndEnum(std::string *code_ptr) { + std::string &code = *code_ptr; + code += "\n"; + } + + // Initialize a new struct or table from existing data. + void NewRootTypeFromBuffer(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + code += Indent + "@classmethod\n"; + code += Indent + "def GetRootAs"; + code += NormalizedName(struct_def); + code += "(cls, buf, offset):"; + code += "\n"; + code += Indent + Indent; + code += "n = flatbuffers.encode.Get"; + code += "(flatbuffers.packer.uoffset, buf, offset)\n"; + code += Indent + Indent + "x = " + NormalizedName(struct_def) + "()\n"; + code += Indent + Indent + "x.Init(buf, n + offset)\n"; + code += Indent + Indent + "return x\n"; + code += "\n"; + } + + // Initialize an existing object with other data, to avoid an allocation. + void InitializeExisting(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += "Init(self, buf, pos):\n"; + code += Indent + Indent + "self._tab = flatbuffers.table.Table(buf, pos)\n"; + code += "\n"; + } + + // Get the length of a vector. + void GetVectorLen(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)) + "Length(self"; + code += "):" + OffsetPrefix(field); + code += Indent + Indent + Indent + "return self._tab.VectorLen(o)\n"; + code += Indent + Indent + "return 0\n\n"; + } + + // Get the value of a struct's scalar. + void GetScalarFieldOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + std::string getter = GenGetter(field.value.type); + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)); + code += "(self): return " + getter; + code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; + code += NumToString(field.value.offset) + "))\n"; + } + + // Get the value of a table's scalar. + void GetScalarFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + std::string getter = GenGetter(field.value.type); + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)); + code += "(self):"; + code += OffsetPrefix(field); + getter += "o + self._tab.Pos)"; + auto is_bool = field.value.type.base_type == BASE_TYPE_BOOL; + if (is_bool) { + getter = "bool(" + getter + ")"; + } + code += Indent + Indent + Indent + "return " + getter + "\n"; + std::string default_value; + if (is_bool) { + default_value = field.value.constant == "0" ? "False" : "True"; + } else { + default_value = field.value.constant; + } + code += Indent + Indent + "return " + default_value + "\n\n"; + } + + // Get a struct by initializing an existing struct. + // Specific to Struct. + void GetStructFieldOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)); + code += "(self, obj):\n"; + code += Indent + Indent + "obj.Init(self._tab.Bytes, self._tab.Pos + "; + code += NumToString(field.value.offset) + ")"; + code += "\n" + Indent + Indent + "return obj\n\n"; + } + + // Get a struct by initializing an existing struct. + // Specific to Table. + void GetStructFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)); + code += "(self):"; + code += OffsetPrefix(field); + if (field.value.type.struct_def->fixed) { + code += Indent + Indent + Indent + "x = o + self._tab.Pos\n"; + } else { + code += Indent + Indent + Indent; + code += "x = self._tab.Indirect(o + self._tab.Pos)\n"; + } + code += Indent + Indent + Indent; + code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; + code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; + code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; + code += Indent + Indent + Indent + "return obj\n"; + code += Indent + Indent + "return None\n\n"; + } + + // Get the value of a string. + void GetStringField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)); + code += "(self):"; + code += OffsetPrefix(field); + code += Indent + Indent + Indent + "return " + GenGetter(field.value.type); + code += "o + self._tab.Pos)\n"; + code += Indent + Indent + "return None\n\n"; + } + + // Get the value of a union from an object. + void GetUnionField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)) + "(self):"; + code += OffsetPrefix(field); + + // TODO(rw): this works and is not the good way to it: + bool is_native_table = TypeName(field) == "*flatbuffers.Table"; + if (is_native_table) { + code += Indent + Indent + Indent + "from flatbuffers.table import Table\n"; + } else { + code += Indent + Indent + Indent; + code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; + } + code += Indent + Indent + Indent + "obj = Table(bytearray(), 0)\n"; + code += Indent + Indent + Indent + GenGetter(field.value.type); + code += "obj, o)\n" + Indent + Indent + Indent + "return obj\n"; + code += Indent + Indent + "return None\n\n"; + } + + // Get the value of a vector's struct member. + void GetMemberOfVectorOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)); + code += "(self, j):" + OffsetPrefix(field); + code += Indent + Indent + Indent + "x = self._tab.Vector(o)\n"; + code += Indent + Indent + Indent; + code += "x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * "; + code += NumToString(InlineSize(vectortype)) + "\n"; + if (!(vectortype.struct_def->fixed)) { + code += Indent + Indent + Indent + "x = self._tab.Indirect(x)\n"; + } + code += Indent + Indent + Indent; + code += "from ." + TypeName(field) + " import " + TypeName(field) + "\n"; + code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; + code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; + code += Indent + Indent + Indent + "return obj\n"; + code += Indent + Indent + "return None\n\n"; + } + + // Get the value of a vector's non-struct member. Uses a named return + // argument to conveniently set the zero value for the result. + void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)); + code += "(self, j):"; + code += OffsetPrefix(field); + code += Indent + Indent + Indent + "a = self._tab.Vector(o)\n"; + code += Indent + Indent + Indent; + code += "return " + GenGetter(field.value.type); + code += "a + flatbuffers.number_types.UOffsetTFlags.py_type(j * "; + code += NumToString(InlineSize(vectortype)) + "))\n"; + if (vectortype.base_type == BASE_TYPE_STRING) { + code += Indent + Indent + "return \"\"\n"; + } else { + code += Indent + Indent + "return 0\n"; + } + code += "\n"; + } + + // Returns a non-struct vector as a numpy array. Much faster + // than iterating over the vector element by element. + void GetVectorOfNonStructAsNumpy(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + + // Currently, we only support accessing as numpy array if + // the vector type is a scalar. + if (!(IsScalar(vectortype.base_type))) { return; } + + GenReceiver(struct_def, code_ptr); + code += MakeCamel(NormalizedName(field)) + "AsNumpy(self):"; + code += OffsetPrefix(field); + + code += Indent + Indent + Indent; + code += "return "; + code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types."; + code += MakeCamel(GenTypeGet(field.value.type)); + code += "Flags, o)\n"; + + if (vectortype.base_type == BASE_TYPE_STRING) { + code += Indent + Indent + "return \"\"\n"; + } else { + code += Indent + Indent + "return 0\n"; + } + code += "\n"; + } + + // Begin the creator function signature. + void BeginBuilderArgs(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + code += "\n"; + code += "def Create" + NormalizedName(struct_def); + code += "(builder"; + } + + // Recursively generate arguments for a constructor, to deal with nested + // structs. + void StructBuilderArgs(const StructDef &struct_def, + const char *nameprefix, std::string *code_ptr) { + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (IsStruct(field.value.type)) { + // Generate arguments for a struct inside a struct. To ensure names + // don't clash, and to make it obvious these arguments are constructing + // a nested struct, prefix the name with the field name. + StructBuilderArgs(*field.value.type.struct_def, + (nameprefix + (NormalizedName(field) + "_")).c_str(), code_ptr); + } else { + std::string &code = *code_ptr; + code += (std::string) ", " + nameprefix; + code += MakeCamel(NormalizedName(field), false); + } + } + } + + // End the creator function signature. + void EndBuilderArgs(std::string *code_ptr) { + std::string &code = *code_ptr; + code += "):\n"; + } + + // Recursively generate struct construction statements and instert manual + // padding. + void StructBuilderBody(const StructDef &struct_def, + const char *nameprefix, std::string *code_ptr) { + std::string &code = *code_ptr; + code += " builder.Prep(" + NumToString(struct_def.minalign) + ", "; + code += NumToString(struct_def.bytesize) + ")\n"; + for (auto it = struct_def.fields.vec.rbegin(); + it != struct_def.fields.vec.rend(); ++it) { + auto &field = **it; + if (field.padding) + code += " builder.Pad(" + NumToString(field.padding) + ")\n"; + if (IsStruct(field.value.type)) { + StructBuilderBody(*field.value.type.struct_def, + (nameprefix + (NormalizedName(field) + "_")).c_str(), code_ptr); + } else { + code += " builder.Prepend" + GenMethod(field) + "("; + code += nameprefix + MakeCamel(NormalizedName(field), false) + ")\n"; + } + } + } + + void EndBuilderBody(std::string *code_ptr) { + std::string &code = *code_ptr; + code += " return builder.Offset()\n"; + } + + // Get the value of a table's starting offset. + void GetStartOfTable(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + NormalizedName(struct_def) + "Start"; + code += "(builder): "; + code += "builder.StartObject("; + code += NumToString(struct_def.fields.vec.size()); + code += ")\n"; + } + + // Set the value of a table's field. + void BuildFieldOfTable(const StructDef &struct_def, + const FieldDef &field, const size_t offset, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + NormalizedName(struct_def) + "Add" + MakeCamel(NormalizedName(field)); + code += "(builder, "; + code += MakeCamel(NormalizedName(field), false); + code += "): "; + code += "builder.Prepend"; + code += GenMethod(field) + "Slot("; + code += NumToString(offset) + ", "; + if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { + code += "flatbuffers.number_types.UOffsetTFlags.py_type"; + code += "("; + code += MakeCamel(NormalizedName(field), false) + ")"; + } else { + code += MakeCamel(NormalizedName(field), false); + } + code += ", " + field.value.constant; + code += ")\n"; + } + + // Set the value of one of the members of a table's vector. + void BuildVectorOfTable(const StructDef &struct_def, + const FieldDef &field, std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + NormalizedName(struct_def) + "Start"; + code += MakeCamel(NormalizedName(field)); + code += "Vector(builder, numElems): return builder.StartVector("; + auto vector_type = field.value.type.VectorType(); + auto alignment = InlineAlignment(vector_type); + auto elem_size = InlineSize(vector_type); + code += NumToString(elem_size); + code += ", numElems, " + NumToString(alignment); + code += ")\n"; + } + + // Get the offset of the end of a table. + void GetEndOffsetOnTable(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "def " + NormalizedName(struct_def) + "End"; + code += "(builder): "; + code += "return builder.EndObject()\n"; + } + + // Generate the receiver for function signatures. + void GenReceiver(const StructDef &struct_def, std::string *code_ptr) { + std::string &code = *code_ptr; + code += Indent + "# " + NormalizedName(struct_def) + "\n"; + code += Indent + "def "; + } + + // Generate a struct field, conditioned on its child type(s). + void GenStructAccessor(const StructDef &struct_def, + const FieldDef &field, std::string *code_ptr) { + GenComment(field.doc_comment, code_ptr, nullptr, "# "); + if (IsScalar(field.value.type.base_type)) { + if (struct_def.fixed) { + GetScalarFieldOfStruct(struct_def, field, code_ptr); + } else { + GetScalarFieldOfTable(struct_def, field, code_ptr); + } + } else { + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: + if (struct_def.fixed) { + GetStructFieldOfStruct(struct_def, field, code_ptr); + } else { + GetStructFieldOfTable(struct_def, field, code_ptr); + } + break; + case BASE_TYPE_STRING: GetStringField(struct_def, field, code_ptr); break; + case BASE_TYPE_VECTOR: { + auto vectortype = field.value.type.VectorType(); + if (vectortype.base_type == BASE_TYPE_STRUCT) { + GetMemberOfVectorOfStruct(struct_def, field, code_ptr); + } else { + GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr); + GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr); + } + break; + } + case BASE_TYPE_UNION: GetUnionField(struct_def, field, code_ptr); break; + default: FLATBUFFERS_ASSERT(0); + } + } + if (field.value.type.base_type == BASE_TYPE_VECTOR) { + GetVectorLen(struct_def, field, code_ptr); + } + } + + // Generate table constructors, conditioned on its members' types. + void GenTableBuilders(const StructDef &struct_def, + std::string *code_ptr) { + GetStartOfTable(struct_def, code_ptr); + + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + + auto offset = it - struct_def.fields.vec.begin(); + BuildFieldOfTable(struct_def, field, offset, code_ptr); + if (field.value.type.base_type == BASE_TYPE_VECTOR) { + BuildVectorOfTable(struct_def, field, code_ptr); + } + } + + GetEndOffsetOnTable(struct_def, code_ptr); + } + + // Generate struct or table methods. + void GenStruct(const StructDef &struct_def, std::string *code_ptr) { + if (struct_def.generated) return; + + GenComment(struct_def.doc_comment, code_ptr, nullptr, "# "); + BeginClass(struct_def, code_ptr); + if (!struct_def.fixed) { + // Generate a special accessor for the table that has been declared as + // the root type. + NewRootTypeFromBuffer(struct_def, code_ptr); + } + // Generate the Init method that sets the field in a pre-existing + // accessor object. This is to allow object reuse. + InitializeExisting(struct_def, code_ptr); + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + + GenStructAccessor(struct_def, field, code_ptr); + } + + if (struct_def.fixed) { + // create a struct constructor function + GenStructBuilder(struct_def, code_ptr); + } else { + // Create a set of functions that allow table construction. + GenTableBuilders(struct_def, code_ptr); + } + } + + // Generate enum declarations. + void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { + if (enum_def.generated) return; + + GenComment(enum_def.doc_comment, code_ptr, nullptr, "# "); + BeginEnum(NormalizedName(enum_def), code_ptr); + for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + ++it) { + auto &ev = **it; + GenComment(ev.doc_comment, code_ptr, nullptr, "# "); + EnumMember(ev, code_ptr); + } + EndEnum(code_ptr); + } + + // Returns the function name that is able to read a value of the given type. + std::string GenGetter(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_STRING: return "self._tab.String("; + case BASE_TYPE_UNION: return "self._tab.Union("; + case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); + default: + return "self._tab.Get(flatbuffers.number_types." + + MakeCamel(GenTypeGet(type)) + "Flags, "; + } + } + + // Returns the method name for use with add/put calls. + std::string GenMethod(const FieldDef &field) { + return IsScalar(field.value.type.base_type) + ? MakeCamel(GenTypeBasic(field.value.type)) + : (IsStruct(field.value.type) ? "Struct" : "UOffsetTRelative"); + } + + std::string GenTypeBasic(const Type &type) { + static const char *ctypename[] = { + // clang-format off + #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \ + #PTYPE, + FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) + #undef FLATBUFFERS_TD + // clang-format on + }; + return ctypename[type.base_type]; + } + + std::string GenTypePointer(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_STRING: return "string"; + case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); + case BASE_TYPE_STRUCT: return type.struct_def->name; + case BASE_TYPE_UNION: + // fall through + default: return "*flatbuffers.Table"; + } + } + + std::string GenTypeGet(const Type &type) { + return IsScalar(type.base_type) ? GenTypeBasic(type) : GenTypePointer(type); + } + + std::string TypeName(const FieldDef &field) { + return GenTypeGet(field.value.type); + } + + // Create a struct with a builder and the struct's arguments. + void GenStructBuilder(const StructDef &struct_def, + std::string *code_ptr) { + BeginBuilderArgs(struct_def, code_ptr); + StructBuilderArgs(struct_def, "", code_ptr); + EndBuilderArgs(code_ptr); + + StructBuilderBody(struct_def, "", code_ptr); + EndBuilderBody(code_ptr); + } + bool generate() { if (!generateEnums()) return false; if (!generateStructs()) return false; @@ -667,9 +710,11 @@ class PythonGenerator : public BaseGenerator { BeginFile(LastNamespacePart(*def.defined_namespace), needs_imports, &code); code += classcode; std::string filename = - NamespaceDir(*def.defined_namespace) + def.name + ".py"; + NamespaceDir(*def.defined_namespace) + NormalizedName(def) + ".py"; return SaveFile(filename.c_str(), code, false); } + private: + std::unordered_set keywords_; }; } // namespace python