From df4909e5f6b7e60a90b32973567d24b90608c6fb Mon Sep 17 00:00:00 2001 From: Gabriel Martinez Date: Tue, 4 Nov 2014 10:00:56 -0800 Subject: [PATCH] Add options to print build rule dependencies Tested: on Linux Bug: 16465909 Change-Id: I2f1a6def13e47716110426b00990c2c625c03251 --- include/flatbuffers/idl.h | 51 +++++++++++++++++- include/flatbuffers/util.h | 33 ++++++++++++ src/flatc.cpp | 100 ++++++++++++++++------------------ src/idl_gen_cpp.cpp | 22 +++++++- src/idl_gen_general.cpp | 107 ++++++++++++++++++++++++++++++++++++- src/idl_gen_text.cpp | 35 ++++++++++++ src/idl_parser.cpp | 65 +++++++++++++++++----- 7 files changed, 343 insertions(+), 70 deletions(-) diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 3caf790a3..4ed789e37 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -179,6 +180,7 @@ struct Definition { Definition() : generated(false), defined_namespace(nullptr) {} std::string name; + std::string file; std::vector doc_comment; SymbolTable attributes; bool generated; // did we already output code for this definition? @@ -309,6 +311,11 @@ class Parser { // Mark all definitions as already having code generated. void MarkGenerated(); + // Get the files recursively included by the given file. The returned + // container will have at least the given file. + std::set GetIncludedFilesRecursive( + const std::string &file_name) const; + private: int64_t ParseHexNum(int nibbles); void Next(); @@ -349,11 +356,13 @@ class Parser { std::string file_extension_; std::map included_files_; + std::map> files_included_per_file_; private: const char *source_, *cursor_; int line_; // the current line being parsed int token_; + std::stack files_being_parsed_; bool proto_mode_; bool strict_json_; std::string attribute_; @@ -381,7 +390,7 @@ struct GeneratorOptions { bool include_dependence_headers; // Possible options for the more general generator below. - enum Language { kJava, kCSharp, kMAX }; + enum Language { kJava, kCSharp, kGo, kMAX }; Language lang; @@ -401,6 +410,18 @@ extern void GenerateText(const Parser &parser, const void *flatbuffer, const GeneratorOptions &opts, std::string *text); +extern bool GenerateTextFile(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); + +// Generate binary files from a given FlatBuffer, and a given Parser +// object that has been populated with the corresponding schema. +// See idl_gen_general.cpp. +extern bool GenerateBinary(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); // Generate a C++ header from the definitions in the Parser object. // See idl_gen_cpp. @@ -450,6 +471,34 @@ extern bool GenerateFBS(const Parser &parser, const std::string &file_name, const GeneratorOptions &opts); +// Generate a make rule for the generated C++ header. +// See idl_gen_cpp.cpp. +extern std::string CPPMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); + +// Generate a make rule for the generated Java/C#/... files. +// See idl_gen_general.cpp. +extern std::string GeneralMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); + +// Generate a make rule for the generated text (JSON) files. +// See idl_gen_text.cpp. +extern std::string TextMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); + +// Generate a make rule for the generated binary files. +// See idl_gen_general.cpp. +extern std::string BinaryMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); + } // namespace flatbuffers #endif // FLATBUFFERS_IDL_H_ diff --git a/include/flatbuffers/util.h b/include/flatbuffers/util.h index fa3fc4e93..ab47d63ca 100644 --- a/include/flatbuffers/util.h +++ b/include/flatbuffers/util.h @@ -80,6 +80,12 @@ inline int64_t StringToInt(const char *str, int base = 10) { #endif } +// Check if file "name" exists. +inline bool FileExists(const char *name) { + std::ifstream ifs(name); + return ifs.good(); +} + // Load file "name" into "buf" returning true if successful // false otherwise. If "binary" is false data is read // using ifstream's text mode, otherwise data is read with @@ -234,6 +240,33 @@ inline int FromUTF8(const char **in) { return ucc; } +// Wraps a string to a maximum length, inserting new lines where necessary. Any +// existing whitespace will be collapsed down to a single space. A prefix or +// suffix can be provided, which will be inserted before or after a wrapped +// line, respectively. +inline std::string WordWrap(const std::string in, size_t max_length, + const std::string wrapped_line_prefix, + const std::string wrapped_line_suffix) { + std::istringstream in_stream(in); + std::string wrapped, line, word; + + in_stream >> word; + line = word; + + while (in_stream >> word) { + if ((line.length() + 1 + word.length() + wrapped_line_suffix.length()) < + max_length) { + line += " " + word; + } else { + wrapped += line + wrapped_line_suffix + "\n"; + line = wrapped_line_prefix + word; + } + } + wrapped += line; + + return wrapped; +} + } // namespace flatbuffers #endif // FLATBUFFERS_UTIL_H_ diff --git a/src/flatc.cpp b/src/flatc.cpp index c13229f12..46c64f775 100755 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -21,38 +21,6 @@ static void Error(const char *err, const char *obj = nullptr, bool usage = false, bool show_exe_name = true); -namespace flatbuffers { - -bool GenerateBinary(const Parser &parser, - const std::string &path, - const std::string &file_name, - const GeneratorOptions & /*opts*/) { - auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin"; - return !parser.builder_.GetSize() || - flatbuffers::SaveFile( - (path + file_name + "." + ext).c_str(), - reinterpret_cast(parser.builder_.GetBufferPointer()), - parser.builder_.GetSize(), - true); -} - -bool GenerateTextFile(const Parser &parser, - const std::string &path, - const std::string &file_name, - const GeneratorOptions &opts) { - if (!parser.builder_.GetSize()) return true; - if (!parser.root_struct_def) Error("root_type not set"); - std::string text; - GenerateText(parser, parser.builder_.GetBufferPointer(), opts, - &text); - return flatbuffers::SaveFile((path + file_name + ".json").c_str(), - text, - false); - -} - -} - // This struct allows us to create a table of all possible output generators // for the various programming languages and formats we support. struct Generator { @@ -60,31 +28,42 @@ struct Generator { const std::string &path, const std::string &file_name, const flatbuffers::GeneratorOptions &opts); - const char *opt; - const char *name; + const char *generator_opt; + const char *lang_name; flatbuffers::GeneratorOptions::Language lang; - const char *help; + const char *generator_help; + + std::string (*make_rule)(const flatbuffers::Parser &parser, + const std::string &path, + const std::string &file_name, + const flatbuffers::GeneratorOptions &opts); }; const Generator generators[] = { { flatbuffers::GenerateBinary, "-b", "binary", flatbuffers::GeneratorOptions::kMAX, - "Generate wire format binaries for any data definitions" }, + "Generate wire format binaries for any data definitions", + flatbuffers::BinaryMakeRule }, { flatbuffers::GenerateTextFile, "-t", "text", flatbuffers::GeneratorOptions::kMAX, - "Generate text output for any data definitions" }, + "Generate text output for any data definitions", + flatbuffers::TextMakeRule }, { flatbuffers::GenerateCPP, "-c", "C++", flatbuffers::GeneratorOptions::kMAX, - "Generate C++ headers for tables/structs" }, + "Generate C++ headers for tables/structs", + flatbuffers::CPPMakeRule }, { flatbuffers::GenerateGo, "-g", "Go", - flatbuffers::GeneratorOptions::kMAX, - "Generate Go files for tables/structs" }, + flatbuffers::GeneratorOptions::kGo, + "Generate Go files for tables/structs", + flatbuffers::GeneralMakeRule }, { flatbuffers::GenerateGeneral, "-j", "Java", flatbuffers::GeneratorOptions::kJava, - "Generate Java classes for tables/structs" }, + "Generate Java classes for tables/structs", + flatbuffers::GeneralMakeRule }, { flatbuffers::GenerateGeneral, "-n", "C#", flatbuffers::GeneratorOptions::kCSharp, - "Generate C# classes for tables/structs" } + "Generate C# classes for tables/structs", + flatbuffers::GeneralMakeRule }, }; const char *program_name = NULL; @@ -98,10 +77,13 @@ static void Error(const char *err, const char *obj, bool usage, if (usage) { printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name); for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i) - printf(" %s %s.\n", generators[i].opt, generators[i].help); + printf(" %s %s.\n", + generators[i].generator_opt, + generators[i].generator_help); printf( " -o PATH Prefix PATH to all generated files.\n" " -I PATH Search for includes in the specified path.\n" + " -M Print make rules for generated files.\n" " --strict-json Strict JSON: field names must be / will be quoted,\n" " no trailing commas in tables/vectors.\n" " --no-prefix Don\'t prefix enum values with the enum type in C++.\n" @@ -110,7 +92,7 @@ static void Error(const char *err, const char *obj, bool usage, " --proto Input is a .proto, translate to .fbs.\n" "FILEs may depend on declarations in earlier files.\n" "FILEs after the -- must be binary flatbuffer format files.\n" - "Output files are named using the base file name of the input," + "Output files are named using the base file name of the input,\n" "and written to the current directory or the path given by -o.\n" "example: %s -c -b schema1.fbs schema2.fbs data.json\n", program_name); @@ -125,6 +107,7 @@ int main(int argc, const char *argv[]) { const size_t num_generators = sizeof(generators) / sizeof(generators[0]); bool generator_enabled[num_generators] = { false }; bool any_generator = false; + bool print_make_rules = false; bool proto_mode = false; std::vector filenames; std::vector include_directories; @@ -152,9 +135,11 @@ int main(int argc, const char *argv[]) { } else if(opt == "--proto") { proto_mode = true; any_generator = true; + } else if(opt == "-M") { + print_make_rules = true; } else { for (size_t i = 0; i < num_generators; ++i) { - if(opt == generators[i].opt) { + if (opt == generators[i].generator_opt) { generator_enabled[i] = true; any_generator = true; goto found; @@ -171,8 +156,7 @@ int main(int argc, const char *argv[]) { if (!filenames.size()) Error("missing input files", nullptr, true); if (!any_generator) - Error("no options: no output files generated.", - "specify one of -c -g -j -t -b etc.", true); + Error("no options", "specify one of -c -g -j -t -b etc.", true); // Now process the files: flatbuffers::Parser parser(opts.strict_json, proto_mode); @@ -205,14 +189,22 @@ int main(int argc, const char *argv[]) { flatbuffers::StripExtension(*file_it)); for (size_t i = 0; i < num_generators; ++i) { + opts.lang = generators[i].lang; if (generator_enabled[i]) { - flatbuffers::EnsureDirExists(output_path); - opts.lang = generators[i].lang; - if (!generators[i].generate(parser, output_path, filebase, opts)) { - Error((std::string("Unable to generate ") + - generators[i].name + - " for " + - filebase).c_str()); + if (!print_make_rules) { + flatbuffers::EnsureDirExists(output_path); + if (!generators[i].generate(parser, output_path, filebase, opts)) { + Error((std::string("Unable to generate ") + + generators[i].lang_name + + " for " + + filebase).c_str()); + } + } else { + std::string make_rule = generators[i].make_rule( + parser, output_path, *file_it, opts); + if (!make_rule.empty()) + printf("%s\n", flatbuffers::WordWrap( + make_rule, 80, " ", " \\").c_str()); } } } diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index 15b51513c..e728c6ef6 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -696,13 +696,33 @@ std::string GenerateCPP(const Parser &parser, return std::string(); } +static std::string GeneratedFileName(const std::string &path, + const std::string &file_name) { + return path + file_name + "_generated.h"; +} + bool GenerateCPP(const Parser &parser, const std::string &path, const std::string &file_name, const GeneratorOptions &opts) { auto code = GenerateCPP(parser, file_name, opts); return !code.length() || - SaveFile((path + file_name + "_generated.h").c_str(), code, false); + SaveFile(GeneratedFileName(path, file_name).c_str(), code, false); +} + +std::string CPPMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions & /*opts*/) { + std::string filebase = flatbuffers::StripPath( + flatbuffers::StripExtension(file_name)); + std::string make_rule = GeneratedFileName(path, filebase) + ": "; + auto included_files = parser.GetIncludedFilesRecursive(file_name); + for (auto it = included_files.begin(); + it != included_files.end(); ++it) { + make_rule += " " + *it; + } + return make_rule; } } // namespace flatbuffers diff --git a/src/idl_gen_general.cpp b/src/idl_gen_general.cpp index 975d08eb6..4baf85752 100644 --- a/src/idl_gen_general.cpp +++ b/src/idl_gen_general.cpp @@ -98,6 +98,23 @@ LanguageParameters language_parameters[] = { "\n}\n", "", "using FlatBuffers;\n\n", + }, + // TODO: add Go support to the general generator. + // WARNING: this is currently only used for generating make rules for Go. + { + GeneratorOptions::kGo, + true, + ".go", + "string", + "bool ", + "\n{\n", + "const ", + "", + "package ", + "", + "", + "", + "import (\n\tflatbuffers \"github.com/google/flatbuffers/go\"\n)", } }; @@ -116,10 +133,11 @@ static std::string GenTypeBasic(const LanguageParameters &lang, const Type &type) { static const char *gtypename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \ - #JTYPE, #NTYPE, + #JTYPE, #NTYPE, #GTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; + return gtypename[type.base_type * GeneratorOptions::kMAX + lang.language]; } @@ -653,4 +671,91 @@ bool GenerateGeneral(const Parser &parser, return true; } +static std::string ClassFileName(const LanguageParameters &lang, + const Parser &parser, const Definition &def, + const std::string &path) { + std::string namespace_general; + std::string namespace_dir = path; + auto &namespaces = parser.namespaces_.back()->components; + for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { + if (namespace_general.length()) { + namespace_general += "."; + namespace_dir += kPathSeparator; + } + namespace_general += *it; + namespace_dir += *it; + } + + return namespace_dir + kPathSeparator + def.name + lang.file_extension; +} + +std::string GeneralMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts) { + assert(opts.lang <= GeneratorOptions::kMAX); + auto lang = language_parameters[opts.lang]; + + std::string make_rule; + + for (auto it = parser.enums_.vec.begin(); + it != parser.enums_.vec.end(); ++it) { + if (make_rule != "") + make_rule += " "; + make_rule += ClassFileName(lang, parser, **it, path); + } + + for (auto it = parser.structs_.vec.begin(); + it != parser.structs_.vec.end(); ++it) { + if (make_rule != "") + make_rule += " "; + make_rule += ClassFileName(lang, parser, **it, path); + } + + make_rule += ": "; + auto included_files = parser.GetIncludedFilesRecursive(file_name); + for (auto it = included_files.begin(); + it != included_files.end(); ++it) { + make_rule += " " + *it; + } + return make_rule; +} + +std::string BinaryFileName(const Parser &parser, + const std::string &path, + const std::string &file_name) { + auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin"; + return path + file_name + "." + ext; +} + +bool GenerateBinary(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions & /*opts*/) { + return !parser.builder_.GetSize() || + flatbuffers::SaveFile( + BinaryFileName(parser, path, file_name).c_str(), + reinterpret_cast(parser.builder_.GetBufferPointer()), + parser.builder_.GetSize(), + true); +} + +std::string BinaryMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions & /*opts*/) { + if (!parser.builder_.GetSize()) return ""; + std::string filebase = flatbuffers::StripPath( + flatbuffers::StripExtension(file_name)); + std::string make_rule = BinaryFileName(parser, path, filebase) + ": " + + file_name; + auto included_files = parser.GetIncludedFilesRecursive( + parser.root_struct_def->file); + for (auto it = included_files.begin(); + it != included_files.end(); ++it) { + make_rule += " " + *it; + } + return make_rule; +} + } // namespace flatbuffers diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index e0299379d..71be2be93 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -268,5 +268,40 @@ void GenerateText(const Parser &parser, const void *flatbuffer, text += NewLine(opts); } +std::string TextFileName(const std::string &path, + const std::string &file_name) { + return path + file_name + ".json"; +} + +bool GenerateTextFile(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts) { + if (!parser.builder_.GetSize() || !parser.root_struct_def) return true; + std::string text; + GenerateText(parser, parser.builder_.GetBufferPointer(), opts, + &text); + return flatbuffers::SaveFile(TextFileName(path, file_name).c_str(), + text, + false); +} + +std::string TextMakeRule(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions & /*opts*/) { + if (!parser.builder_.GetSize() || !parser.root_struct_def) return ""; + std::string filebase = flatbuffers::StripPath( + flatbuffers::StripExtension(file_name)); + std::string make_rule = TextFileName(path, filebase) + ": " + file_name; + auto included_files = parser.GetIncludedFilesRecursive( + parser.root_struct_def->file); + for (auto it = included_files.begin(); + it != included_files.end(); ++it) { + make_rule += " " + *it; + } + return make_rule; +} + } // namespace flatbuffers diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 8eabc895f..d3deef7f9 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -15,6 +15,7 @@ */ #include +#include #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" @@ -333,6 +334,7 @@ FieldDef &Parser::AddField(StructDef &struct_def, field.value.offset = FieldIndexToOffset(static_cast(struct_def.fields.vec.size())); field.name = name; + field.file = struct_def.file; field.value.type = type; if (struct_def.fixed) { // statically compute the field offset auto size = InlineSize(type); @@ -732,6 +734,7 @@ void Parser::ParseEnum(bool is_union) { Expect(kTokenIdentifier); auto &enum_def = *new EnumDef(); enum_def.name = name; + if (!files_being_parsed_.empty()) enum_def.file = files_being_parsed_.top(); enum_def.doc_comment = dc; enum_def.is_union = is_union; enum_def.defined_namespace = namespaces_.back(); @@ -799,6 +802,7 @@ StructDef &Parser::StartStruct() { if (!struct_def.predecl) Error("datatype already exists: " + name); struct_def.predecl = false; struct_def.name = name; + if (!files_being_parsed_.empty()) struct_def.file = files_being_parsed_.top(); // Move this struct to the back of the vector just in case it was predeclared, // to preserve declaration order. remove(structs_.vec.begin(), structs_.vec.end(), &struct_def); @@ -1022,7 +1026,16 @@ Type Parser::ParseTypeFromProtoType() { bool Parser::Parse(const char *source, const char **include_paths, const char *source_filename) { - if (source_filename) included_files_[source_filename] = true; + if (source_filename && + included_files_.find(source_filename) == included_files_.end()) { + included_files_[source_filename] = true; + files_included_per_file_[source_filename] = std::set(); + files_being_parsed_.push(source_filename); + } + if (!include_paths) { + const char *current_directory[] = { "", nullptr }; + include_paths = current_directory; + } source_ = cursor_ = source; line_ = 1; error_.clear(); @@ -1033,22 +1046,23 @@ bool Parser::Parse(const char *source, const char **include_paths, while (IsNext(kTokenInclude)) { auto name = attribute_; Expect(kTokenStringConstant); - if (included_files_.find(name) == included_files_.end()) { + // Look for the file in include_paths. + std::string filepath; + for (auto paths = include_paths; paths && *paths; paths++) { + filepath = flatbuffers::ConCatPathFileName(*paths, name); + if(FileExists(filepath.c_str())) break; + } + if (filepath.empty()) + Error("unable to locate include file: " + name); + if (source_filename) + files_included_per_file_[source_filename].insert(filepath); + if (included_files_.find(filepath) == included_files_.end()) { // We found an include file that we have not parsed yet. // Load it and parse it. std::string contents; - if (!include_paths) { - const char *current_directory[] = { "", nullptr }; - include_paths = current_directory; - } - for (auto paths = include_paths; paths && *paths; paths++) { - auto filepath = flatbuffers::ConCatPathFileName(*paths, name); - if(LoadFile(filepath.c_str(), true, &contents)) break; - } - if (contents.empty()) + if (!LoadFile(filepath.c_str(), true, &contents)) Error("unable to load include file: " + name); - included_files_[name] = true; - if (!Parse(contents.c_str(), include_paths)) { + if (!Parse(contents.c_str(), include_paths, filepath.c_str())) { // Any errors, we're done. return false; } @@ -1143,10 +1157,35 @@ bool Parser::Parse(const char *source, const char **include_paths, error_ += NumToString(line_) + ":0"; // gcc alike #endif error_ += ": error: " + msg; + if (source_filename) files_being_parsed_.pop(); return false; } + if (source_filename) files_being_parsed_.pop(); assert(!struct_stack_.size()); return true; } +std::set Parser::GetIncludedFilesRecursive( + const std::string &file_name) const { + std::set included_files; + std::list to_process; + + if (file_name.empty()) return included_files; + to_process.push_back(file_name); + + while (!to_process.empty()) { + std::string current = to_process.front(); + to_process.pop_front(); + included_files.insert(current); + + auto new_files = files_included_per_file_.at(current); + for (auto it = new_files.begin(); it != new_files.end(); ++it) { + if (included_files.find(*it) == included_files.end()) + to_process.push_back(*it); + } + } + + return included_files; +} + } // namespace flatbuffers