From dba962ebb836739e002049a6d484f54a176e9afe Mon Sep 17 00:00:00 2001 From: tira-misu Date: Thu, 13 Dec 2018 20:59:27 +0100 Subject: [PATCH] Enable flatbuffer to initialize Parser from bfbs (#4283) (#5077) * Enable flatbuffer to initialize Parser from bfbs (#4283) Now its possible to generate json data from bfbs data type and flatbuffers data and visa versa. * add deserialize functionality in parser from bfbs * add small usage sample * Fix build break * Merge branch 'pr/1' into fix-issue4283 * Fix buildbreak * Build monster_test.bfbs with --bfbs-builtins Attribute flexbuffer has be included in bfbs. Only with this attribute test will run. By initialization a parser by a bfbs the attribute has to be known for this filed. monsterdata_test.golden has a flexbuffer field so parse would fail. * Fix generate_code.sh * Revert automatic indent changes by IDE * Auto detect size prefixed binary schema files * Use identifier (bfbs) to detect schema files --- .gitignore | 2 + CMakeLists.txt | 9 ++ include/flatbuffers/flatc.h | 3 + include/flatbuffers/idl.h | 28 ++++ samples/monster.bfbs | Bin 0 -> 1768 bytes samples/sample_bfbs.cpp | 77 +++++++++ src/flatc.cpp | 50 ++++-- src/idl_parser.cpp | 313 ++++++++++++++++++++++++++++++++++++ tests/generate_code.bat | 3 +- tests/generate_code.sh | 3 +- tests/monster_test.bfbs | Bin 7280 -> 11624 bytes tests/test.cpp | 29 +++- 12 files changed, 493 insertions(+), 24 deletions(-) create mode 100644 samples/monster.bfbs create mode 100644 samples/sample_bfbs.cpp diff --git a/.gitignore b/.gitignore index 9e00828a3..56a7f6d41 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ flatsamplebinary flatsamplebinary.exe flatsampletext flatsampletext.exe +flatsamplebfbs +flatsamplebfbs.exe grpctest grpctest.exe snapshot.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 389596e4f..b42c4d14b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,14 @@ set(FlatBuffers_Sample_Text_SRCS ${CMAKE_CURRENT_BINARY_DIR}/samples/monster_generated.h ) +set(FlatBuffers_Sample_BFBS_SRCS + ${FlatBuffers_Library_SRCS} + src/idl_gen_general.cpp + samples/sample_bfbs.cpp + # file generated by running compiler on samples/monster.fbs + ${CMAKE_CURRENT_BINARY_DIR}/samples/monster_generated.h +) + set(FlatBuffers_GRPCTest_SRCS include/flatbuffers/flatbuffers.h include/flatbuffers/grpc.h @@ -316,6 +324,7 @@ if(FLATBUFFERS_BUILD_TESTS) include_directories(${CMAKE_CURRENT_BINARY_DIR}/samples) add_executable(flatsamplebinary ${FlatBuffers_Sample_Binary_SRCS}) add_executable(flatsampletext ${FlatBuffers_Sample_Text_SRCS}) + add_executable(flatsamplebfbs ${FlatBuffers_Sample_BFBS_SRCS}) endif() if(FLATBUFFERS_BUILD_GRPCTEST) diff --git a/include/flatbuffers/flatc.h b/include/flatbuffers/flatc.h index c7b60febc..f2765d239 100644 --- a/include/flatbuffers/flatc.h +++ b/include/flatbuffers/flatc.h @@ -80,6 +80,9 @@ class FlatCompiler { const std::string &contents, std::vector &include_directories) const; + void LoadBinarySchema(Parser &parser, const std::string &filename, + const std::string &contents); + void Warn(const std::string &warn, bool show_exe_name = true) const; void Error(const std::string &err, bool usage = true, diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 18e1db9d5..00e6af5e6 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -153,6 +153,8 @@ struct Type { Offset Serialize(FlatBufferBuilder *builder) const; + bool Deserialize(const Parser &parser, const reflection::Type *type); + BaseType base_type; BaseType element; // only set if t == BASE_TYPE_VECTOR StructDef *struct_def; // only set if t or element == BASE_TYPE_STRUCT @@ -235,6 +237,9 @@ struct Definition { flatbuffers::Vector>> SerializeAttributes(FlatBufferBuilder *builder, const Parser &parser) const; + bool DeserializeAttributes(Parser &parser, + const Vector> *attrs); + std::string name; std::string file; std::vector doc_comment; @@ -261,6 +266,8 @@ struct FieldDef : public Definition { Offset Serialize(FlatBufferBuilder *builder, uint16_t id, const Parser &parser) const; + bool Deserialize(Parser &parser, const reflection::Field *field); + Value value; bool deprecated; // Field is allowed to be present in old data, but can't be. // written in new data nor accessed in new code. @@ -291,6 +298,8 @@ struct StructDef : public Definition { Offset Serialize(FlatBufferBuilder *builder, const Parser &parser) const; + bool Deserialize(Parser &parser, const reflection::Object *object); + SymbolTable fields; bool fixed; // If it's struct, not a table. @@ -317,9 +326,12 @@ inline size_t InlineAlignment(const Type &type) { struct EnumVal { EnumVal(const std::string &_name, int64_t _val) : name(_name), value(_val) {} + EnumVal(){}; Offset Serialize(FlatBufferBuilder *builder, const Parser &parser) const; + bool Deserialize(const Parser &parser, const reflection::EnumVal *val); + std::string name; std::vector doc_comment; int64_t value; @@ -340,6 +352,8 @@ struct EnumDef : public Definition { Offset Serialize(FlatBufferBuilder *builder, const Parser &parser) const; + bool Deserialize(Parser &parser, const reflection::Enum *values); + SymbolTable vals; bool is_union; // Type is a union which uses type aliases where at least one type is @@ -358,11 +372,14 @@ inline bool EqualByName(const Type &a, const Type &b) { struct RPCCall : public Definition { Offset Serialize(FlatBufferBuilder *builder, const Parser &parser) const; + bool Deserialize(Parser &parser, const reflection::RPCCall *call); + StructDef *request, *response; }; struct ServiceDef : public Definition { Offset Serialize(FlatBufferBuilder *builder, const Parser &parser) const; + bool Deserialize(Parser &parser, const reflection::Service *service); SymbolTable calls; }; @@ -647,6 +664,15 @@ class Parser : public ParserState { // See reflection/reflection.fbs void Serialize(); + // Deserialize a schema buffer + bool Deserialize(const uint8_t *buf, const size_t size); + + // Fills internal structure as if the schema passed had been loaded by parsing + // with Parse except that included filenames will not be populated. + bool Deserialize(const reflection::Schema* schema); + + Type* DeserializeType(const reflection::Type* type); + // Checks that the schema represented by this parser is a safe evolution // of the schema provided. Returns non-empty error on any problems. std::string ConformTo(const Parser &base); @@ -661,6 +687,8 @@ class Parser : public ParserState { StructDef *LookupStruct(const std::string &id) const; + std::string UnqualifiedName(std::string fullQualifiedName); + private: void Message(const std::string &msg); void Warning(const std::string &msg); diff --git a/samples/monster.bfbs b/samples/monster.bfbs new file mode 100644 index 0000000000000000000000000000000000000000..95a977a36bdbfdfbb18abe4a8ae3904a571610cc GIT binary patch literal 1768 zcmZuxyKWOf6uox*N^qQ&F|djvj*~lw9i^t9Fv*<9gR#pH8~5w) zD54H-!k8gs1#$+mFLiV{TN5HyQ^5U3lyx4U1QBjzhu4N;D=VodS59YmQKvJ z^|Kt=9^|0sJAvO;ZFvI!I^Hdh){fX3kQ)%jzSbC ztI!uw5zilavz&{}fy06nk%@I-4)hEc@%CKL@IYQ4Mo|GQus74@=mPXm>AB=wwDjbO zjFB_1&yw-8IZMc8GQwCYxmi3r{xclCpg)pD)W%!G;vo|HyU^&B$w7Y$uU8?-{&Pn8 z%{k=VL+;o+=A~{0!W!Eu?5mbHB%?nJZ&})I_dCjOcSm@YZPYXa>&Y3bm|7EavncgA zV?nb+Z!SmU%|h3}mpv0mnOYA_ygbH5b~sGBepA7;c%1t>tZX0N(j{KwI2P;edA`S3 z<_`ETzyXPl3^_VVg6)a==%Zk2*SDL zxk{gjJF0b2(contents.c_str()), + contents.size())) { + Error("failed to load binary schema: " + filename, false, false); + } +} + void FlatCompiler::Warn(const std::string &warn, bool show_exe_name) const { params_.warn_fn(this, warn, show_exe_name); } @@ -127,8 +136,9 @@ std::string FlatCompiler::GetUsageString(const char *program_name) const { " --force-defaults Emit default values in binary output from JSON\n" " --force-empty When serializing from object API representation,\n" " force strings and vectors to empty rather than null.\n" - "FILEs may be schemas (must end in .fbs), or JSON files (conforming to preceding\n" - "schema). FILEs after the -- must be binary flatbuffer format files.\n" + "FILEs may be schemas (must end in .fbs), binary schemas (must end in .bfbs),\n" + "or JSON files (conforming to preceding schema). FILEs after the -- must be\n" + "binary flatbuffer format files.\n" "Output files are named using the base file name of the input,\n" "and written to the current directory or the path given by -o.\n" "example: " << program_name << " -c -b schema1.fbs schema2.fbs data.json\n"; @@ -323,8 +333,14 @@ int FlatCompiler::Compile(int argc, const char **argv) { std::string contents; if (!flatbuffers::LoadFile(conform_to_schema.c_str(), true, &contents)) Error("unable to load schema: " + conform_to_schema); - ParseFile(conform_parser, conform_to_schema, contents, - conform_include_directories); + + if (flatbuffers::GetExtension(conform_to_schema) == + reflection::SchemaExtension()) { + LoadBinarySchema(conform_parser, conform_to_schema, contents); + } else { + ParseFile(conform_parser, conform_to_schema, contents, + conform_include_directories); + } } std::unique_ptr parser(new flatbuffers::Parser(opts)); @@ -340,6 +356,7 @@ int FlatCompiler::Compile(int argc, const char **argv) { static_cast(file_it - filenames.begin()) >= binary_files_from; auto ext = flatbuffers::GetExtension(filename); auto is_schema = ext == "fbs" || ext == "proto"; + auto is_binary_schema = ext == reflection::SchemaExtension(); if (is_binary) { parser->builder_.Clear(); parser->builder_.PushFlatBuffer( @@ -366,7 +383,7 @@ int FlatCompiler::Compile(int argc, const char **argv) { } } else { // Check if file contains 0 bytes. - if (contents.length() != strlen(contents.c_str())) { + if (!is_binary_schema && contents.length() != strlen(contents.c_str())) { Error("input file appears to be binary: " + filename, true); } if (is_schema) { @@ -375,15 +392,19 @@ int FlatCompiler::Compile(int argc, const char **argv) { // so explicitly using an include. parser.reset(new flatbuffers::Parser(opts)); } - ParseFile(*parser.get(), filename, contents, include_directories); - if (!is_schema && !parser->builder_.GetSize()) { - // If a file doesn't end in .fbs, it must be json/binary. Ensure we - // didn't just parse a schema with a different extension. - Error( - "input file is neither json nor a .fbs (schema) file: " + filename, - true); + if (is_binary_schema) { + LoadBinarySchema(*parser.get(), filename, contents); + } else { + ParseFile(*parser.get(), filename, contents, include_directories); + if (!is_schema && !parser->builder_.GetSize()) { + // If a file doesn't end in .fbs, it must be json/binary. Ensure we + // didn't just parse a schema with a different extension. + Error("input file is neither json nor a .fbs (schema) file: " + + filename, + true); + } } - if (is_schema && !conform_to_schema.empty()) { + if ((is_schema || is_binary_schema) && !conform_to_schema.empty()) { auto err = parser->ConformTo(conform_parser); if (!err.empty()) Error("schemas don\'t conform: " + err); } @@ -401,7 +422,8 @@ int FlatCompiler::Compile(int argc, const char **argv) { if (generator_enabled[i]) { if (!print_make_rules) { flatbuffers::EnsureDirExists(output_path); - if ((!params_.generators[i].schema_only || is_schema) && + if ((!params_.generators[i].schema_only || + (is_schema || is_binary_schema)) && !params_.generators[i].generate(*parser.get(), output_path, filebase)) { Error(std::string("Unable to generate ") + diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 357337a74..9e607ca60 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -91,6 +91,13 @@ std::string MakeCamel(const std::string &in, bool first) { return s; } +void DeserializeDoc( std::vector &doc, + const Vector> *documentation) { + if (documentation == nullptr) return; + for (uoffset_t index = 0; index < documentation->Length(); index++) + doc.push_back(documentation->Get(index)->str()); +} + void Parser::Message(const std::string &msg) { error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : ""; // clang-format off @@ -1763,6 +1770,21 @@ Namespace *Parser::UniqueNamespace(Namespace *ns) { return ns; } +std::string Parser::UnqualifiedName(std::string full_qualified_name) { + Namespace *ns = new Namespace(); + + std::size_t current, previous = 0; + current = full_qualified_name.find('.'); + while (current != std::string::npos) { + ns->components.push_back( + full_qualified_name.substr(previous, current - previous)); + previous = current + 1; + current = full_qualified_name.find('.', previous); + } + current_namespace_ = UniqueNamespace(ns); + return full_qualified_name.substr(previous, current - previous); +} + static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) { auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str()); auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str()); @@ -2675,6 +2697,32 @@ void Parser::Serialize() { } } +static Namespace *GetNamespace( + const std::string &qualified_name, std::vector &namespaces, + std::map &namespaces_index) { + size_t dot = qualified_name.find_last_of('.'); + std::string namespace_name = (dot != std::string::npos) + ? std::string(qualified_name.c_str(), dot) + : ""; + Namespace *&ns = namespaces_index[namespace_name]; + + if (!ns) { + ns = new Namespace(); + namespaces.push_back(ns); + + size_t pos = 0; + + for (;;) { + dot = qualified_name.find('.', pos); + if (dot == std::string::npos) { break; } + ns->components.push_back(qualified_name.substr(pos, dot-pos)); + pos = dot + 1; + } + } + + return ns; +} + Offset StructDef::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { std::vector> field_offsets; @@ -2695,6 +2743,45 @@ Offset StructDef::Serialize(FlatBufferBuilder *builder, attr__, docs__); } +bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) { + if (!DeserializeAttributes(parser, object->attributes())) + return false; + DeserializeDoc(doc_comment, object->documentation()); + name = parser.UnqualifiedName(object->name()->str()); + fixed = object->is_struct(); + minalign = object->minalign(); + predecl = false; + sortbysize = attributes.Lookup("original_order") == nullptr && !fixed; + std::vector indexes = + std::vector(object->fields()->Length()); + for (uoffset_t i = 0; i < object->fields()->Length(); i++) + indexes[object->fields()->Get(i)->id()] = i; + for (size_t i = 0; i < indexes.size(); i++) { + auto field = object->fields()->Get(indexes[i]); + auto field_def = new FieldDef(); + if (!field_def->Deserialize(parser, field) || + fields.Add(field_def->name, field_def)) { + delete field_def; + return false; + } + if (fixed) { + // Recompute padding since that's currently not serialized. + auto size = InlineSize(field_def->value.type); + auto next_field = + i + 1 < indexes.size() + ? object->fields()->Get(indexes[i+1]) + : nullptr; + bytesize += size; + field_def->padding = + next_field ? (next_field->offset() - field_def->value.offset) - size + : PaddingBytes(bytesize, minalign); + bytesize += field_def->padding; + } + } + FLATBUFFERS_ASSERT(static_cast(bytesize) == object->bytesize()); + return true; +} + Offset FieldDef::Serialize(FlatBufferBuilder *builder, uint16_t id, const Parser &parser) const { @@ -2715,6 +2802,38 @@ Offset FieldDef::Serialize(FlatBufferBuilder *builder, // space by sharing it. Same for common values of value.type. } +bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) { + name = parser.UnqualifiedName(field->name()->str()); + defined_namespace = parser.current_namespace_; + if (!value.type.Deserialize(parser, field->type())) + return false; + value.offset = field->offset(); + if (IsInteger(value.type.base_type)) { + value.constant = NumToString(field->default_integer()); + } else if (IsFloat(value.type.base_type)) { + value.constant = FloatToString(field->default_real(), 16); + size_t last_zero = value.constant.find_last_not_of('0'); + if (last_zero != std::string::npos && last_zero != 0) { + value.constant.erase(last_zero, std::string::npos); + } + } + deprecated = field->deprecated(); + required = field->required(); + key = field->key(); + if (!DeserializeAttributes(parser, field->attributes())) + return false; + // TODO: this should probably be handled by a separate attribute + if (attributes.Lookup("flexbuffer")) { + flexbuffer = true; + parser.uses_flexbuffers_ = true; + if (value.type.base_type != BASE_TYPE_VECTOR || + value.type.element != BASE_TYPE_UCHAR) + return false; + } + DeserializeDoc(doc_comment, field->documentation()); + return true; +} + Offset RPCCall::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { auto name__ = builder->CreateString(name); @@ -2728,6 +2847,17 @@ Offset RPCCall::Serialize(FlatBufferBuilder *builder, attr__, docs__); } +bool RPCCall::Deserialize(Parser &parser, const reflection::RPCCall *call) { + name = call->name()->str(); + if (!DeserializeAttributes(parser, call->attributes())) + return false; + DeserializeDoc(doc_comment, call->documentation()); + request = parser.structs_.Lookup(call->request()->name()->str()); + response = parser.structs_.Lookup(call->response()->name()->str()); + if (!request || !response) { return false; } + return true; +} + Offset ServiceDef::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { std::vector> servicecall_offsets; @@ -2744,6 +2874,25 @@ Offset ServiceDef::Serialize(FlatBufferBuilder *builder, return reflection::CreateService(*builder, name__, call__, attr__, docs__); } +bool ServiceDef::Deserialize(Parser &parser, + const reflection::Service *service) { + name = parser.UnqualifiedName(service->name()->str()); + if (service->calls()) { + for (uoffset_t i = 0; i < service->calls()->size(); ++i) { + auto call = new RPCCall(); + if (!call->Deserialize(parser, service->calls()->Get(i)) || + calls.Add(call->name, call)) { + delete call; + return false; + } + } + } + if (!DeserializeAttributes(parser, service->attributes())) + return false; + DeserializeDoc(doc_comment, service->documentation()); + return true; +} + Offset EnumDef::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { std::vector> enumval_offsets; @@ -2762,6 +2911,26 @@ Offset EnumDef::Serialize(FlatBufferBuilder *builder, attr__, docs__); } +bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) { + name = parser.UnqualifiedName(_enum->name()->str()); + for (uoffset_t i = 0; i < _enum->values()->size(); ++i) { + auto val = new EnumVal(); + if (!val->Deserialize(parser, _enum->values()->Get(i)) || + vals.Add(val->name, val)) { + delete val; + return false; + } + } + is_union = _enum->is_union(); + if (!underlying_type.Deserialize(parser, _enum->underlying_type())) { + return false; + } + if (!DeserializeAttributes(parser, _enum->attributes())) + return false; + DeserializeDoc(doc_comment, _enum->documentation()); + return true; +} + Offset EnumVal::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { auto name__ = builder->CreateString(name); @@ -2774,6 +2943,16 @@ Offset EnumVal::Serialize(FlatBufferBuilder *builder, type__, docs__); } +bool EnumVal::Deserialize(const Parser &parser, + const reflection::EnumVal *val) { + name = val->name()->str(); + value = val->value(); + if (!union_type.Deserialize(parser, val->union_type())) + return false; + DeserializeDoc(doc_comment, val->documentation()); + return true; +} + Offset Type::Serialize(FlatBufferBuilder *builder) const { return reflection::CreateType( *builder, @@ -2782,6 +2961,31 @@ Offset Type::Serialize(FlatBufferBuilder *builder) const { struct_def ? struct_def->index : (enum_def ? enum_def->index : -1)); } +bool Type::Deserialize(const Parser &parser, const reflection::Type *type) { + if (type == nullptr) return true; + base_type = static_cast(type->base_type()); + element = static_cast(type->element()); + if (type->index() >= 0) { + if (type->base_type() == reflection::Obj || + (type->base_type() == reflection::Vector && + type->element() == reflection::Obj)) { + if (static_cast(type->index()) < parser.structs_.vec.size()) { + struct_def = parser.structs_.vec[type->index()]; + struct_def->refcount++; + } else { + return false; + } + } else { + if (static_cast(type->index()) < parser.enums_.vec.size()) { + enum_def = parser.enums_.vec[type->index()]; + } else { + return false; + } + } + } + return true; +} + flatbuffers::Offset< flatbuffers::Vector>> Definition::SerializeAttributes(FlatBufferBuilder *builder, @@ -2803,6 +3007,115 @@ Definition::SerializeAttributes(FlatBufferBuilder *builder, } } +bool Definition::DeserializeAttributes( + Parser &parser, const Vector> *attrs) { + if (attrs == nullptr) + return true; + for (uoffset_t i = 0; i < attrs->size(); ++i) { + auto kv = attrs->Get(i); + auto value = new Value(); + if (kv->value()) { value->constant = kv->value()->str(); } + if (attributes.Add(kv->key()->str(), value)) { + delete value; + return false; + } + parser.known_attributes_[kv->key()->str()]; + } + return true; +} + +/************************************************************************/ +/* DESERIALIZATION */ +/************************************************************************/ +bool Parser::Deserialize(const uint8_t *buf, const size_t size) { + flatbuffers::Verifier verifier(reinterpret_cast(buf), size); + bool size_prefixed = false; + if(!reflection::SchemaBufferHasIdentifier(buf)) { + if (!flatbuffers::BufferHasIdentifier(buf, reflection::SchemaIdentifier(), + true)) + return false; + else + size_prefixed = true; + } + auto verify_fn = size_prefixed ? &reflection::VerifySizePrefixedSchemaBuffer + : &reflection::VerifySchemaBuffer; + if (!verify_fn(verifier)) { + return false; + } + auto schema = size_prefixed ? reflection::GetSizePrefixedSchema(buf) + : reflection::GetSchema(buf); + return Deserialize(schema); +} + +bool Parser::Deserialize(const reflection::Schema *schema) { + file_identifier_ = schema->file_ident() ? schema->file_ident()->str() : ""; + file_extension_ = schema->file_ext() ? schema->file_ext()->str() : ""; + std::map namespaces_index; + + // Create defs without deserializing so references from fields to structs and + // enums can be resolved. + for (auto it = schema->objects()->begin(); it != schema->objects()->end(); + ++it) { + auto struct_def = new StructDef(); + if (structs_.Add(it->name()->str(), struct_def)) { + delete struct_def; + return false; + } + auto type = new Type(BASE_TYPE_STRUCT, struct_def, nullptr); + if (types_.Add(it->name()->str(), type)) { + delete type; + return false; + } + } + for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) { + auto enum_def = new EnumDef(); + if (enums_.Add(it->name()->str(), enum_def)) { + delete enum_def; + return false; + } + auto type = new Type(BASE_TYPE_UNION, nullptr, enum_def); + if (types_.Add(it->name()->str(), type)) { + delete type; + return false; + } + } + + // Now fields can refer to structs and enums by index. + for (auto it = schema->objects()->begin(); it != schema->objects()->end(); + ++it) { + std::string qualified_name = it->name()->str(); + auto struct_def = structs_.Lookup(qualified_name); + struct_def->defined_namespace = + GetNamespace(qualified_name, namespaces_, namespaces_index); + if (!struct_def->Deserialize(*this, * it)) { return false; } + if (schema->root_table() == *it) { root_struct_def_ = struct_def; } + } + for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) { + std::string qualified_name = it->name()->str(); + auto enum_def = enums_.Lookup(qualified_name); + enum_def->defined_namespace = + GetNamespace(qualified_name, namespaces_, namespaces_index); + if (!enum_def->Deserialize(*this, *it)) { return false; } + } + + if (schema->services()) { + for (auto it = schema->services()->begin(); it != schema->services()->end(); + ++it) { + std::string qualified_name = it->name()->str(); + auto service_def = new ServiceDef(); + service_def->defined_namespace = + GetNamespace(qualified_name, namespaces_, namespaces_index); + if (!service_def->Deserialize(*this, *it) || + services_.Add(qualified_name, service_def)) { + delete service_def; + return false; + } + } + } + + return true; +} + std::string Parser::ConformTo(const Parser &base) { for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) { auto &struct_def = **sit; diff --git a/tests/generate_code.bat b/tests/generate_code.bat index fedb08e92..304fecec5 100644 --- a/tests/generate_code.bat +++ b/tests/generate_code.bat @@ -18,10 +18,11 @@ if "%1"=="-b" set buildtype=%2 ..\%buildtype%\flatc.exe --cpp --java --csharp --go --binary --python --lobster --lua --js --rust --ts --php --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr --no-fb-import -I include_test monster_test.fbs monsterdata_test.json || goto FAIL ..\%buildtype%\flatc.exe --cpp --java --csharp --go --binary --python --lobster --lua --js --rust --ts --php --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs || goto FAIL ..\%buildtype%\flatc.exe --cpp --js --ts --php --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs || goto FAIL -..\%buildtype%\flatc.exe -b --schema --bfbs-comments -I include_test monster_test.fbs || goto FAIL +..\%buildtype%\flatc.exe -b --schema --bfbs-comments --bfbs-builtins -I include_test monster_test.fbs || goto FAIL ..\%buildtype%\flatc.exe --jsonschema --schema -I include_test monster_test.fbs || goto FAIL cd ../samples ..\%buildtype%\flatc.exe --cpp --lobster --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr monster.fbs || goto FAIL +..\%buildtype%\flatc.exe -b --schema --bfbs-comments --bfbs-builtins monster.fbs || goto FAIL cd ../reflection call generate_code.bat %1 %2 || goto FAIL diff --git a/tests/generate_code.sh b/tests/generate_code.sh index 89b868fd1..d5ade21a8 100755 --- a/tests/generate_code.sh +++ b/tests/generate_code.sh @@ -18,9 +18,10 @@ set -e ../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr --no-fb-import -I include_test monster_test.fbs monsterdata_test.json ../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs ../flatc --cpp --js --ts --php --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs -../flatc -b --schema --bfbs-comments -I include_test monster_test.fbs +../flatc -b --schema --bfbs-comments --bfbs-builtins -I include_test monster_test.fbs ../flatc --jsonschema --schema -I include_test monster_test.fbs cd ../samples ../flatc --cpp --lobster --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr monster.fbs +../flatc -b --schema --bfbs-comments --bfbs-builtins monster.fbs cd ../reflection ./generate_code.sh diff --git a/tests/monster_test.bfbs b/tests/monster_test.bfbs index 2bfda2195033133bb2b6e3f2d829b366827a9d05..f96b9fd2bc1d417b065c52349bbf4ab898e6b368 100644 GIT binary patch literal 11624 zcmb7~4`^N2eaDY%S(0VhQXSP*RMmND6<2jNPmwjxGor|z<2XTfM67xt1j|o)SJqpe z?t6WI6gz}s$_S=}VoC@hj1o#PCX`}|2_uwYf*EC)F^b8y%#{8Kye`Yq%=O*p zd+s}W=UqMh=@UQQ`_B2D-=FXAcYo)1?|ps7m;;9n9OGcROrL2sZKi{>ZqvhmW5%H& zCuwzhj#496GyF_4UAoZX?z_#Fm$&VGump*X2vh?Fs1_;eGeJ) zAvCztn1QbtbCLO%zHH1-_=>M|SC~ga7^ZEVhGFtNR=VeZ={BXaM>G zbmYvV>G@#SBd62zg9-gEC<69Qz{pO^n5lv#ram~(V%=J8y-oYKAbI2FlXWzEaws%ce^K zu?oot@#o0J9*)96`R#vRjCFNNm+(EZ+n5X0btB$7&H10dOs*RycK9>^(_MltJmiz) zy3enZYaS+N|1ZYQ_x}fsDPBjPIQodzUc(*^4Y4{z9%xSXY0WL@B9wt-%LC9-tiB#* zzflPsP_K}4!l{qzbvc}V80+b5gU{lGzrt5K_C8ee*T?B*b&cXK945)F6jpXuYm#zH zdauFY^!j))Wp(}s9wSbwos&ejpsWFZ_{~^XeI7-fB zP%0nGss>J`r*pxdWXp4pgq3+_w%CgA<171eIMpa0zg3kh$4PbL>^VwY9lyncxiQ9c z?s8K2x*D$CMc39zXdD`a{Ct*tt-W|-X)RJsoX(b~W^?INT(V}{d)C+W!cp`p^bC}O zWK$vzf1JFz{7?@1>-_v3L8dT&v={_o6o9;!dRD~o%agkmhr>*~JecvdogxOx`L8$8U6Xm&}c$6LRSRp+V z>^fA;&p#H<<#iPPse{t=vt~#F1fd(--%(!fFapKh4b&cce&}Al0 zLLJa2&*JKJ@_^dpCsS$n6J&sL+@X~*9&S*A-`~W18>~F|z6M{DX<5Seam?2&oGW{cxe3n7 zqNKd-cHG!_WZ^r3ecj~5IO{mroO4IEJCjO>sUU9PQkncrWu6WsUC!pi)J%R}U-qSF zKh@Wz^3#7B%#>L(1dSr(RY>_afl-6(u^kk{xo%_5ZlWLJeDZ$!Q;t_TUc}H#IQc#W zW{|`@M?o0aM!tR)88kp;&aczJUSaJxV<{4{jI3)E`_z5LJjL7%#^#_Q#@e_*dOkx> zOTl=RV^=478M_1DIQ$us_ZGU8yranPz{Y027qGL3;)=TDwc*OGdHUa?==aUEIS7x| zqtfEWVG+YyzpT|x#bo3U<9#@rG3KWq*z3pEOw2M1z?Lsg;8~MA;$ra@v zyqS={rzZbRas;tmiX-ltba-Ye?Yb?#_oDyz%!w7{zmb&x2Q~Tazp(O~Bwu@No&2dX zC4}Az9fRxyD`C|3&j}v@Q&_U+piKptH~f)DJa7*OU5&Yx?dmr|VFB+UI7&g@@Bq zaoyC|T)H&J!Svj!@sZ~2MYBP%sFq>UuA31WyHV!wQThRh9or?$60zRK{!LhKK)@7eZ?IA zop8GA*)9A>C^9ZCJ^xbke+Z_GLJ69a~AURsE_^9*=z9`s^PPjxwYicNsn(ed|rar`K){2UF3|O2gn9LH(W0L z82%J<$|dn+g4uK>S6<9HhOD3dT8)j|j`M0e=e4|=$u~$V^ESL+Hcy&s@Lqv@+@$v` zGy(NP`abcm^_cDcqZ(#8=GKDQ%O2lqm|cO_VYb@!^d0gY6WiHxOapJth5wAA9e8t= z`2ytos`xt}N8V5PxVPrx1am2Pg~jg@lc)d7#-vkv)Z5nL!jqM}yVh+1Dlu)Iyc0za znWJlt%lk>2y*;p*J#H;=8T9y8i_25+t~D-K@kMox*2DArxghFwXx3@2U;`y>teM_;wIc!$*c^uxg`rJ$2Vp2QplPOWlbHP--!;J6wbrgNc zJig-IdIg}2YE3@skdU=H>-t1dZ{`*P!Uj4hkHH7F*+H)m; zmYmZadCEQ!Xyy(tepaDTJ&(}6dM{5H4s#rZx*^3a{oo7hX*!?J**&Y}f1~JgX4vGIKM5_D>n|sDcs1Y>bFLn* zRJWh!nm|RnYK3f3U(d6LDckQe_FZh*#+9;~Eh+YSLq0CLUSEJFpngc-FDGsC>dfCZw26lw)wS;a<`ejow~{A1#*+jN zOYUt_?75C_wf^dM>}Q+rXJz;BBIYf>k*vX0F@G7Ib(jy;Fu%myQp}h7_q^?KO{;H9 z$+P9V*m9rQxoX`_vFG@ik?E`9@fS%uyt;aQ4R*X1+tKRV;qpUv{0KWbd0RHd`3U4+ldisw zvM<$P;nmkqWBKb;qZD@)K~7Ek-6+zVoP(S%C->!~To_%=`IP);X{Vk+s^5}q-bR+s za6XId&1SoO7cKmJo3);lv9b6F`F@<-!!%thq2y)sN}=pp2h-2~TIZ)%kI$l0eTC%7 zmUw;La&r%!4(@7J^7&j+hgWBBli!H9@3B;af-snuzsG+YKNG*i`RmYWNVz6o`>N-9 zN0FN6t~i^^hC!;F&l~fY##VopTAs_6QqN^`xl|BlQk6m~8>V=TkUE9_Vj7fq?8^It z+Fn1-{#S>SSC{ui7ANb2t4Y;9rmUw*+8}q_i=W$%alTw`TuSN~s^vxp>xsqYgnYWN z_`Up#%LnY+ zQ{Sg56fwj-kY4~$HtBL1bwtqCX zX3p}9iZGMoGEe%zeIyxJ^P#Cp?{v-~OMTKxK781~2F17-e4~;r+Q+b_i+#}dQ+5iA zXM;0lFfm6NT=egQ*!IoBN;ke7%=gJzy+;y`W01=$T&?@Kym~1iUrn3C?{RXQuN(Az zuAt@qeS##%AD2^lFO}5$iesuCo!(}<=f-i?m}2YnzEjmJzvk0nT6*JLRvc;{r&m%A zGQ3>@o%2Huo$d^o!n5mX4CeB_??YXX(`=P{p)7qo@hPG}%R(ubq z?9Z|OB;@Cf?CFQ(bBeu7x^*u7ve)IqD04zybIPOT-X*B-WxCz@d5Abme$pq&)wl|t z>cM?b!Z+KdzXgVk-cN?Ho|d@oRP?hca}(Z1?=rmh)wcD{`gqD6Z36$ zwx6(krb%{oL(X>{^>NtEalQ>o*uQ_8eOT4vd9Z70pl>*q-zGdWS?=|DyW!^?!_|*| zol=b>p&4UzxtzO2eP1qy>U&w*=KRxJv=q^dfBm8=(`VnS!PAuC-$6*SQjm|6%gqV) zKc`DFc};Be+;BEu*hRT&p-o^66-pLi%^4@9}i}Bn>X=cgRO=2deu*| ztZpAxU3w3GMN`*~Z|A<4e{p}({;E3Dj@|3=Y)SmdABj*-iC48@54?@??{>_)Nmx7& zoq(@VjJL%$SUyt09OrvdZ`DrAbD9X$ela8byKPKfU~K`CKHo>-J_IQzdA+z85AkVS z^@V+FfnCQ&TEE3nzI8!b;EH|3OS zrmh0(BoW_#UhMm)0LeLf(LoG->J?k9}-RXpC*&!pCAjcga`qwg7LFVrBXWGgeh ztX~OvzUz!R5A3?pn4b+8)7{Sa z2Ilx(7=#8ma1(R%(noq})Xm@ZcI5P6Xb@o@0=57*1BVyxEY60*cbq8B&XvRA!*Nut zg_UEqxKcbGvi|!jgQ=Y!EklNn$KN{g|Bc_GbOG|Bud31@Nq&uvh+-3`3rJd((beOlTB$`C*jI2+*=W**D4) z$^8ikUT)f7ESc959-0>%28QvomnWOm0_HUKKI?;{c4vo;dcXb4a#J5}{FC#q_*i}j zdXw=Cpxj&s z;g768JLM<&c^7aikl1{8Z6>T7q1qmsD^7;P`z!J6UD0H@J{6*ysnlz)LJV1at$R0gnP(fb*=qPCrV&3E=qfy882A zbYX6(Oqf-FZ8X+^CEdi|%erz&cK3Aa_x$wE9du?G+%7P)xLf{VaaSu&^EhY>yiN*8 z7Ow}pKf8=%MZN$;H@T#S-D?2ymrfP+fEaKw~B1Y z;%)xe`lZeC1MehwEv7w(cf{ef%Fp#(#_YxpMJ&s2-r?DJl)!r#ynb@x5Oj1l#=hsc zGC?s4LU&mRrsB!^Y#7ywwNe}flku#6><+l%=nfIY6Ay=zHE2{P#Pa~~IPh!WG>V-d zncrlizeM7E#`yI$j(5WjcPU+Cl>0E~iP0PiB3`xN8*n0t@@G`!BM zkkLQK+5zAS^G`B&iZi*7eu#b#{R8yB1ulToDmrv}8W~F7*XeKPw+h%Uba)h?#Ao-0 zFPCQUVx#dbGm)zYpImmi4ja=8%;S!@G9FKlhm`B8ISSpdc_>GBmY9OxClz!LEJh3C zMSr{D_fyb)(>$1?`^eF?h)!qL3cB$cTiKX%&<;_Nv$bpU-<(WLwKYbxb$d(Tjsnd$ z%>6l<`yGwFd&s_J8YgVz?~TS*b6+m!hW=&!(sc_o%RY(XXgq!-Dn-Y~E8%ol38P6a zar5x_x*5yy7e9cyk#ZUo%H@JYG!XUki^W zlxCQkxY^v3^VzD;t?j+?vtqDpf3;SLz3mm9>)c>jhtF5dk(~Uw&zui=0;G665*B}u z_8$8g`c3TStPdh$3TYSDb5q6V<#2o@G4q$8{far5v*BFI2GK9uU<0?L{0OyvqdAbH ze+HjI*2}gvqMs`g?Amz5(OA{dI9}s|!+Zt$Th0EQ?qw&x`xJs>bLCzv!RQnF&o=+Q%4Q5YHM#15?3V*sBo}e?k z!M)jhH^;k&dp?|`j=k53cd=3_F2vK-xKazJNQpWXbP@XX=C+)ynv=Cwdp%9dt1LP` zLjFT{li8i4dziRFvWD_!>85$}==5K2Zy&9kChi`lS=}sjFBs%D6VPkF>Fjr{ut)tX zS$@bCMeslP-Dq~&JBRX8cbDjin=XESYv&$@w)GGX25TtKPDlD5{j<>+ph9PJ`BCS` zRpOKK!$cgH?H+msx*N<$j!yXnNtg0SbTh^340nU^>1cjik-1CIUC-&2m8bk-xid{y zP0?+o?q6qyb95g#x$28ko`I?{C26y49EyVh*Y(H(VkUEVKg z``~BZfo>hI#H_rh99?Ic&f6!tF6!qsW^<124M$f<(|P+uw-35LqMnuaMPgr?&f6!t zMd*5Y^=Iiuh!4xFIKFNC6ma360bei8>>1aEy)5_YJXY;d%){Ba+5+SEp;>D-#5Jpo$eJ&_P0nd)$II=56H0PSe zdxJX!?;F(~>E`n)sXaA+_*_EIW_S3Hd5=N6HbY0XqVB+ND|J?P`YntVGuh-?7qz3E zH+jL{F(hAWevj{9JPobx`B`6{BR(uEMJzwXLgSO*>%O1GzwGd3-cqcSE0B2u{dM>E z`iZG>DBX72^VQgQtn`~q9;)rsT)hVZ=9ClOwgMcSWlWl$R^?YIn&(`LD+|PWJ(l+^ z+06x%rq_&WE3HLuhkF>@4ye9*zA`|+^)A#GHt8vSwXU^ofac~ZrMObEsciF$DZ4MG z`9c>>l}S6j+tNB1tFN?hI<&MjdzYMaepoAcI-pn9b$P%ycTVWdvWY3hU3yOW<)r`6 z@oQ7Q&&+8ueeUG>>1`8$r4{+%B;ebKJbek#9Glqt>+ifFc8ec<%%lI;4Oex7O>v+v1rP>8aSX0?I@6YQodn4AA85w0DBqrfeOfzP3P>^_9CDnSyvumADu2 z8c?oypEI2_MKtj_Zp`u-b9xrM-j@=+cR(e*75f12dMlPs0}lXO042kDy3;_DIE;mp z+sz>Lq|N5>%DuaY&mmh;<8;+JXuoF-Nfv` zxl|9duM{EQe2iOdUTH=3Kymd=dN5w{pRd^;y7?C6qv%AVc>9>^_Y|-TXcA-T%8JiD zcRyw#X-=;exMuzIPNI)|J^`%5=UJS~4yR2xMPMiAvT(9BiK)~#WdHw~5&sVx`z`uv zbed26e^+#meQfD7qBNd%}Kc-pd`_-W&4ub=uCGUOvm%`Z>Xwoc?Mm zi`KDrH932dwM5U%w^A#8-*ODnH5Wp7yECkH=}h;r)z=LD7eLiXy3u--H_qR>H~9Y& zC^H7Ej#>`*qZywko}cLU0J5XGuJ=auDj$u}$p0q})Ku%Xx(3X6{%)}yO*;Hp(&6+o F=D(#6LoEOR diff --git a/tests/test.cpp b/tests/test.cpp index b8a8c56eb..656849a49 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -569,25 +569,37 @@ void JsonDefaultTest() { // example of parsing text straight into a buffer, and generating // text back from it: -void ParseAndGenerateTextTest() { +void ParseAndGenerateTextTest(bool binary) { // load FlatBuffer schema (.fbs) and JSON from disk std::string schemafile; std::string jsonfile; - TEST_EQ(flatbuffers::LoadFile((test_data_path + "monster_test.fbs").c_str(), - false, &schemafile), + TEST_EQ(flatbuffers::LoadFile( + (test_data_path + "monster_test." + (binary ? "bfbs" : "fbs")) + .c_str(), + binary, &schemafile), true); TEST_EQ(flatbuffers::LoadFile( (test_data_path + "monsterdata_test.golden").c_str(), false, &jsonfile), true); - // parse schema first, so we can use it to parse the data after - flatbuffers::Parser parser; auto include_test_path = - flatbuffers::ConCatPathFileName(test_data_path, "include_test"); + flatbuffers::ConCatPathFileName(test_data_path, "include_test"); const char *include_directories[] = { test_data_path.c_str(), include_test_path.c_str(), nullptr }; - TEST_EQ(parser.Parse(schemafile.c_str(), include_directories), true); + + // parse schema first, so we can use it to parse the data after + flatbuffers::Parser parser; + if (binary) { + flatbuffers::Verifier verifier( + reinterpret_cast(schemafile.c_str()), + schemafile.size()); + TEST_EQ(reflection::VerifySchemaBuffer(verifier), true); + //auto schema = reflection::GetSchema(schemafile.c_str()); + TEST_EQ(parser.Deserialize((const uint8_t *)schemafile.c_str(), schemafile.size()), true); + } else { + TEST_EQ(parser.Parse(schemafile.c_str(), include_directories), true); + } TEST_EQ(parser.Parse(jsonfile.c_str(), include_directories), true); // here, parser.builder_ contains a binary buffer that is the parsed data. @@ -2467,7 +2479,8 @@ int FlatBufferTests() { test_data_path = FLATBUFFERS_STRING(FLATBUFFERS_TEST_PATH_PREFIX) + test_data_path; #endif - ParseAndGenerateTextTest(); + ParseAndGenerateTextTest(false); + ParseAndGenerateTextTest(true); ReflectionTest(flatbuf.data(), flatbuf.size()); ParseProtoTest(); UnionVectorTest();