Add options to print build rule dependencies

Tested: on Linux

Bug: 16465909
Change-Id: I2f1a6def13e47716110426b00990c2c625c03251
This commit is contained in:
Gabriel Martinez 2014-11-04 10:00:56 -08:00
parent cf7135ff58
commit df4909e5f6
7 changed files with 343 additions and 70 deletions

View File

@ -19,6 +19,7 @@
#include <map>
#include <set>
#include <stack>
#include <memory>
#include <functional>
@ -179,6 +180,7 @@ struct Definition {
Definition() : generated(false), defined_namespace(nullptr) {}
std::string name;
std::string file;
std::vector<std::string> doc_comment;
SymbolTable<Value> attributes;
bool generated; // did we already output code for this definition?
@ -309,6 +311,11 @@ class Parser {
// Mark all definitions as already having code generated.
void MarkGenerated();
// Get the files recursively included by the given file. The returned
// container will have at least the given file.
std::set<std::string> GetIncludedFilesRecursive(
const std::string &file_name) const;
private:
int64_t ParseHexNum(int nibbles);
void Next();
@ -349,11 +356,13 @@ class Parser {
std::string file_extension_;
std::map<std::string, bool> included_files_;
std::map<std::string, std::set<std::string>> files_included_per_file_;
private:
const char *source_, *cursor_;
int line_; // the current line being parsed
int token_;
std::stack<std::string> files_being_parsed_;
bool proto_mode_;
bool strict_json_;
std::string attribute_;
@ -381,7 +390,7 @@ struct GeneratorOptions {
bool include_dependence_headers;
// Possible options for the more general generator below.
enum Language { kJava, kCSharp, kMAX };
enum Language { kJava, kCSharp, kGo, kMAX };
Language lang;
@ -401,6 +410,18 @@ extern void GenerateText(const Parser &parser,
const void *flatbuffer,
const GeneratorOptions &opts,
std::string *text);
extern bool GenerateTextFile(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts);
// Generate binary files from a given FlatBuffer, and a given Parser
// object that has been populated with the corresponding schema.
// See idl_gen_general.cpp.
extern bool GenerateBinary(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts);
// Generate a C++ header from the definitions in the Parser object.
// See idl_gen_cpp.
@ -450,6 +471,34 @@ extern bool GenerateFBS(const Parser &parser,
const std::string &file_name,
const GeneratorOptions &opts);
// Generate a make rule for the generated C++ header.
// See idl_gen_cpp.cpp.
extern std::string CPPMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts);
// Generate a make rule for the generated Java/C#/... files.
// See idl_gen_general.cpp.
extern std::string GeneralMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts);
// Generate a make rule for the generated text (JSON) files.
// See idl_gen_text.cpp.
extern std::string TextMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts);
// Generate a make rule for the generated binary files.
// See idl_gen_general.cpp.
extern std::string BinaryMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts);
} // namespace flatbuffers
#endif // FLATBUFFERS_IDL_H_

View File

@ -80,6 +80,12 @@ inline int64_t StringToInt(const char *str, int base = 10) {
#endif
}
// Check if file "name" exists.
inline bool FileExists(const char *name) {
std::ifstream ifs(name);
return ifs.good();
}
// Load file "name" into "buf" returning true if successful
// false otherwise. If "binary" is false data is read
// using ifstream's text mode, otherwise data is read with
@ -234,6 +240,33 @@ inline int FromUTF8(const char **in) {
return ucc;
}
// Wraps a string to a maximum length, inserting new lines where necessary. Any
// existing whitespace will be collapsed down to a single space. A prefix or
// suffix can be provided, which will be inserted before or after a wrapped
// line, respectively.
inline std::string WordWrap(const std::string in, size_t max_length,
const std::string wrapped_line_prefix,
const std::string wrapped_line_suffix) {
std::istringstream in_stream(in);
std::string wrapped, line, word;
in_stream >> word;
line = word;
while (in_stream >> word) {
if ((line.length() + 1 + word.length() + wrapped_line_suffix.length()) <
max_length) {
line += " " + word;
} else {
wrapped += line + wrapped_line_suffix + "\n";
line = wrapped_line_prefix + word;
}
}
wrapped += line;
return wrapped;
}
} // namespace flatbuffers
#endif // FLATBUFFERS_UTIL_H_

View File

@ -21,38 +21,6 @@
static void Error(const char *err, const char *obj = nullptr,
bool usage = false, bool show_exe_name = true);
namespace flatbuffers {
bool GenerateBinary(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions & /*opts*/) {
auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin";
return !parser.builder_.GetSize() ||
flatbuffers::SaveFile(
(path + file_name + "." + ext).c_str(),
reinterpret_cast<char *>(parser.builder_.GetBufferPointer()),
parser.builder_.GetSize(),
true);
}
bool GenerateTextFile(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts) {
if (!parser.builder_.GetSize()) return true;
if (!parser.root_struct_def) Error("root_type not set");
std::string text;
GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
&text);
return flatbuffers::SaveFile((path + file_name + ".json").c_str(),
text,
false);
}
}
// This struct allows us to create a table of all possible output generators
// for the various programming languages and formats we support.
struct Generator {
@ -60,31 +28,42 @@ struct Generator {
const std::string &path,
const std::string &file_name,
const flatbuffers::GeneratorOptions &opts);
const char *opt;
const char *name;
const char *generator_opt;
const char *lang_name;
flatbuffers::GeneratorOptions::Language lang;
const char *help;
const char *generator_help;
std::string (*make_rule)(const flatbuffers::Parser &parser,
const std::string &path,
const std::string &file_name,
const flatbuffers::GeneratorOptions &opts);
};
const Generator generators[] = {
{ flatbuffers::GenerateBinary, "-b", "binary",
flatbuffers::GeneratorOptions::kMAX,
"Generate wire format binaries for any data definitions" },
"Generate wire format binaries for any data definitions",
flatbuffers::BinaryMakeRule },
{ flatbuffers::GenerateTextFile, "-t", "text",
flatbuffers::GeneratorOptions::kMAX,
"Generate text output for any data definitions" },
"Generate text output for any data definitions",
flatbuffers::TextMakeRule },
{ flatbuffers::GenerateCPP, "-c", "C++",
flatbuffers::GeneratorOptions::kMAX,
"Generate C++ headers for tables/structs" },
"Generate C++ headers for tables/structs",
flatbuffers::CPPMakeRule },
{ flatbuffers::GenerateGo, "-g", "Go",
flatbuffers::GeneratorOptions::kMAX,
"Generate Go files for tables/structs" },
flatbuffers::GeneratorOptions::kGo,
"Generate Go files for tables/structs",
flatbuffers::GeneralMakeRule },
{ flatbuffers::GenerateGeneral, "-j", "Java",
flatbuffers::GeneratorOptions::kJava,
"Generate Java classes for tables/structs" },
"Generate Java classes for tables/structs",
flatbuffers::GeneralMakeRule },
{ flatbuffers::GenerateGeneral, "-n", "C#",
flatbuffers::GeneratorOptions::kCSharp,
"Generate C# classes for tables/structs" }
"Generate C# classes for tables/structs",
flatbuffers::GeneralMakeRule },
};
const char *program_name = NULL;
@ -98,10 +77,13 @@ static void Error(const char *err, const char *obj, bool usage,
if (usage) {
printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name);
for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
printf(" %s %s.\n", generators[i].opt, generators[i].help);
printf(" %s %s.\n",
generators[i].generator_opt,
generators[i].generator_help);
printf(
" -o PATH Prefix PATH to all generated files.\n"
" -I PATH Search for includes in the specified path.\n"
" -M Print make rules for generated files.\n"
" --strict-json Strict JSON: field names must be / will be quoted,\n"
" no trailing commas in tables/vectors.\n"
" --no-prefix Don\'t prefix enum values with the enum type in C++.\n"
@ -110,7 +92,7 @@ static void Error(const char *err, const char *obj, bool usage,
" --proto Input is a .proto, translate to .fbs.\n"
"FILEs may depend on declarations in earlier files.\n"
"FILEs after the -- must be binary flatbuffer format files.\n"
"Output files are named using the base file name of the input,"
"Output files are named using the base file name of the input,\n"
"and written to the current directory or the path given by -o.\n"
"example: %s -c -b schema1.fbs schema2.fbs data.json\n",
program_name);
@ -125,6 +107,7 @@ int main(int argc, const char *argv[]) {
const size_t num_generators = sizeof(generators) / sizeof(generators[0]);
bool generator_enabled[num_generators] = { false };
bool any_generator = false;
bool print_make_rules = false;
bool proto_mode = false;
std::vector<std::string> filenames;
std::vector<const char *> include_directories;
@ -152,9 +135,11 @@ int main(int argc, const char *argv[]) {
} else if(opt == "--proto") {
proto_mode = true;
any_generator = true;
} else if(opt == "-M") {
print_make_rules = true;
} else {
for (size_t i = 0; i < num_generators; ++i) {
if(opt == generators[i].opt) {
if (opt == generators[i].generator_opt) {
generator_enabled[i] = true;
any_generator = true;
goto found;
@ -171,8 +156,7 @@ int main(int argc, const char *argv[]) {
if (!filenames.size()) Error("missing input files", nullptr, true);
if (!any_generator)
Error("no options: no output files generated.",
"specify one of -c -g -j -t -b etc.", true);
Error("no options", "specify one of -c -g -j -t -b etc.", true);
// Now process the files:
flatbuffers::Parser parser(opts.strict_json, proto_mode);
@ -205,14 +189,22 @@ int main(int argc, const char *argv[]) {
flatbuffers::StripExtension(*file_it));
for (size_t i = 0; i < num_generators; ++i) {
opts.lang = generators[i].lang;
if (generator_enabled[i]) {
flatbuffers::EnsureDirExists(output_path);
opts.lang = generators[i].lang;
if (!generators[i].generate(parser, output_path, filebase, opts)) {
Error((std::string("Unable to generate ") +
generators[i].name +
" for " +
filebase).c_str());
if (!print_make_rules) {
flatbuffers::EnsureDirExists(output_path);
if (!generators[i].generate(parser, output_path, filebase, opts)) {
Error((std::string("Unable to generate ") +
generators[i].lang_name +
" for " +
filebase).c_str());
}
} else {
std::string make_rule = generators[i].make_rule(
parser, output_path, *file_it, opts);
if (!make_rule.empty())
printf("%s\n", flatbuffers::WordWrap(
make_rule, 80, " ", " \\").c_str());
}
}
}

View File

@ -696,13 +696,33 @@ std::string GenerateCPP(const Parser &parser,
return std::string();
}
static std::string GeneratedFileName(const std::string &path,
const std::string &file_name) {
return path + file_name + "_generated.h";
}
bool GenerateCPP(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts) {
auto code = GenerateCPP(parser, file_name, opts);
return !code.length() ||
SaveFile((path + file_name + "_generated.h").c_str(), code, false);
SaveFile(GeneratedFileName(path, file_name).c_str(), code, false);
}
std::string CPPMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions & /*opts*/) {
std::string filebase = flatbuffers::StripPath(
flatbuffers::StripExtension(file_name));
std::string make_rule = GeneratedFileName(path, filebase) + ": ";
auto included_files = parser.GetIncludedFilesRecursive(file_name);
for (auto it = included_files.begin();
it != included_files.end(); ++it) {
make_rule += " " + *it;
}
return make_rule;
}
} // namespace flatbuffers

View File

@ -98,6 +98,23 @@ LanguageParameters language_parameters[] = {
"\n}\n",
"",
"using FlatBuffers;\n\n",
},
// TODO: add Go support to the general generator.
// WARNING: this is currently only used for generating make rules for Go.
{
GeneratorOptions::kGo,
true,
".go",
"string",
"bool ",
"\n{\n",
"const ",
"",
"package ",
"",
"",
"",
"import (\n\tflatbuffers \"github.com/google/flatbuffers/go\"\n)",
}
};
@ -116,10 +133,11 @@ static std::string GenTypeBasic(const LanguageParameters &lang,
const Type &type) {
static const char *gtypename[] = {
#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \
#JTYPE, #NTYPE,
#JTYPE, #NTYPE, #GTYPE,
FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
#undef FLATBUFFERS_TD
};
return gtypename[type.base_type * GeneratorOptions::kMAX + lang.language];
}
@ -653,4 +671,91 @@ bool GenerateGeneral(const Parser &parser,
return true;
}
static std::string ClassFileName(const LanguageParameters &lang,
const Parser &parser, const Definition &def,
const std::string &path) {
std::string namespace_general;
std::string namespace_dir = path;
auto &namespaces = parser.namespaces_.back()->components;
for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
if (namespace_general.length()) {
namespace_general += ".";
namespace_dir += kPathSeparator;
}
namespace_general += *it;
namespace_dir += *it;
}
return namespace_dir + kPathSeparator + def.name + lang.file_extension;
}
std::string GeneralMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts) {
assert(opts.lang <= GeneratorOptions::kMAX);
auto lang = language_parameters[opts.lang];
std::string make_rule;
for (auto it = parser.enums_.vec.begin();
it != parser.enums_.vec.end(); ++it) {
if (make_rule != "")
make_rule += " ";
make_rule += ClassFileName(lang, parser, **it, path);
}
for (auto it = parser.structs_.vec.begin();
it != parser.structs_.vec.end(); ++it) {
if (make_rule != "")
make_rule += " ";
make_rule += ClassFileName(lang, parser, **it, path);
}
make_rule += ": ";
auto included_files = parser.GetIncludedFilesRecursive(file_name);
for (auto it = included_files.begin();
it != included_files.end(); ++it) {
make_rule += " " + *it;
}
return make_rule;
}
std::string BinaryFileName(const Parser &parser,
const std::string &path,
const std::string &file_name) {
auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin";
return path + file_name + "." + ext;
}
bool GenerateBinary(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions & /*opts*/) {
return !parser.builder_.GetSize() ||
flatbuffers::SaveFile(
BinaryFileName(parser, path, file_name).c_str(),
reinterpret_cast<char *>(parser.builder_.GetBufferPointer()),
parser.builder_.GetSize(),
true);
}
std::string BinaryMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions & /*opts*/) {
if (!parser.builder_.GetSize()) return "";
std::string filebase = flatbuffers::StripPath(
flatbuffers::StripExtension(file_name));
std::string make_rule = BinaryFileName(parser, path, filebase) + ": " +
file_name;
auto included_files = parser.GetIncludedFilesRecursive(
parser.root_struct_def->file);
for (auto it = included_files.begin();
it != included_files.end(); ++it) {
make_rule += " " + *it;
}
return make_rule;
}
} // namespace flatbuffers

View File

@ -268,5 +268,40 @@ void GenerateText(const Parser &parser, const void *flatbuffer,
text += NewLine(opts);
}
std::string TextFileName(const std::string &path,
const std::string &file_name) {
return path + file_name + ".json";
}
bool GenerateTextFile(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts) {
if (!parser.builder_.GetSize() || !parser.root_struct_def) return true;
std::string text;
GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
&text);
return flatbuffers::SaveFile(TextFileName(path, file_name).c_str(),
text,
false);
}
std::string TextMakeRule(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions & /*opts*/) {
if (!parser.builder_.GetSize() || !parser.root_struct_def) return "";
std::string filebase = flatbuffers::StripPath(
flatbuffers::StripExtension(file_name));
std::string make_rule = TextFileName(path, filebase) + ": " + file_name;
auto included_files = parser.GetIncludedFilesRecursive(
parser.root_struct_def->file);
for (auto it = included_files.begin();
it != included_files.end(); ++it) {
make_rule += " " + *it;
}
return make_rule;
}
} // namespace flatbuffers

View File

@ -15,6 +15,7 @@
*/
#include <algorithm>
#include <list>
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/idl.h"
@ -333,6 +334,7 @@ FieldDef &Parser::AddField(StructDef &struct_def,
field.value.offset =
FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
field.name = name;
field.file = struct_def.file;
field.value.type = type;
if (struct_def.fixed) { // statically compute the field offset
auto size = InlineSize(type);
@ -732,6 +734,7 @@ void Parser::ParseEnum(bool is_union) {
Expect(kTokenIdentifier);
auto &enum_def = *new EnumDef();
enum_def.name = name;
if (!files_being_parsed_.empty()) enum_def.file = files_being_parsed_.top();
enum_def.doc_comment = dc;
enum_def.is_union = is_union;
enum_def.defined_namespace = namespaces_.back();
@ -799,6 +802,7 @@ StructDef &Parser::StartStruct() {
if (!struct_def.predecl) Error("datatype already exists: " + name);
struct_def.predecl = false;
struct_def.name = name;
if (!files_being_parsed_.empty()) struct_def.file = files_being_parsed_.top();
// Move this struct to the back of the vector just in case it was predeclared,
// to preserve declaration order.
remove(structs_.vec.begin(), structs_.vec.end(), &struct_def);
@ -1022,7 +1026,16 @@ Type Parser::ParseTypeFromProtoType() {
bool Parser::Parse(const char *source, const char **include_paths,
const char *source_filename) {
if (source_filename) included_files_[source_filename] = true;
if (source_filename &&
included_files_.find(source_filename) == included_files_.end()) {
included_files_[source_filename] = true;
files_included_per_file_[source_filename] = std::set<std::string>();
files_being_parsed_.push(source_filename);
}
if (!include_paths) {
const char *current_directory[] = { "", nullptr };
include_paths = current_directory;
}
source_ = cursor_ = source;
line_ = 1;
error_.clear();
@ -1033,22 +1046,23 @@ bool Parser::Parse(const char *source, const char **include_paths,
while (IsNext(kTokenInclude)) {
auto name = attribute_;
Expect(kTokenStringConstant);
if (included_files_.find(name) == included_files_.end()) {
// Look for the file in include_paths.
std::string filepath;
for (auto paths = include_paths; paths && *paths; paths++) {
filepath = flatbuffers::ConCatPathFileName(*paths, name);
if(FileExists(filepath.c_str())) break;
}
if (filepath.empty())
Error("unable to locate include file: " + name);
if (source_filename)
files_included_per_file_[source_filename].insert(filepath);
if (included_files_.find(filepath) == included_files_.end()) {
// We found an include file that we have not parsed yet.
// Load it and parse it.
std::string contents;
if (!include_paths) {
const char *current_directory[] = { "", nullptr };
include_paths = current_directory;
}
for (auto paths = include_paths; paths && *paths; paths++) {
auto filepath = flatbuffers::ConCatPathFileName(*paths, name);
if(LoadFile(filepath.c_str(), true, &contents)) break;
}
if (contents.empty())
if (!LoadFile(filepath.c_str(), true, &contents))
Error("unable to load include file: " + name);
included_files_[name] = true;
if (!Parse(contents.c_str(), include_paths)) {
if (!Parse(contents.c_str(), include_paths, filepath.c_str())) {
// Any errors, we're done.
return false;
}
@ -1143,10 +1157,35 @@ bool Parser::Parse(const char *source, const char **include_paths,
error_ += NumToString(line_) + ":0"; // gcc alike
#endif
error_ += ": error: " + msg;
if (source_filename) files_being_parsed_.pop();
return false;
}
if (source_filename) files_being_parsed_.pop();
assert(!struct_stack_.size());
return true;
}
std::set<std::string> Parser::GetIncludedFilesRecursive(
const std::string &file_name) const {
std::set<std::string> included_files;
std::list<std::string> to_process;
if (file_name.empty()) return included_files;
to_process.push_back(file_name);
while (!to_process.empty()) {
std::string current = to_process.front();
to_process.pop_front();
included_files.insert(current);
auto new_files = files_included_per_file_.at(current);
for (auto it = new_files.begin(); it != new_files.end(); ++it) {
if (included_files.find(*it) == included_files.end())
to_process.push_back(*it);
}
}
return included_files;
}
} // namespace flatbuffers