Added option to not requires an EoF token when parsing JSON (#7620)

Previously when parsing a JSON representation of a Flatbuffer, the
parser required that the input string contain one and only one root
table. This change adds a flag that removes that requirement, so that
if a Flatbuffer table is embedded in some larger string the parser will
simply stop parsing once it reaches the end of the root table, and does
not validate that it has reached the end of the string.

This change also adds a BytesConsumed function, which returns the number
of bytes the parser consumed. This is useful if the table embedded in
some larger string that is being parsed, and that outer parser needs to
know how many bytes the table was so that it can step over it.

include/flatbuffers/idl.h

@@ -642,6 +642,7 @@ struct IDLOptions {
   bool json_nested_legacy_flatbuffers;
   bool ts_flat_file;
   bool no_leak_private_annotations;
+  bool require_json_eof;
 
   // Possible options for the more general generator below.
   enum Language {
@@ -743,6 +744,7 @@ struct IDLOptions {
         json_nested_legacy_flatbuffers(false),
         ts_flat_file(false),
         no_leak_private_annotations(false),
+        require_json_eof(true),
         mini_reflect(IDLOptions::kNone),
         require_explicit_ids(false),
         rust_serialize(false),
@@ -905,6 +907,9 @@ class Parser : public ParserState {
 
   bool ParseJson(const char *json, const char *json_filename = nullptr);
 
+  // Returns the number of characters were consumed when parsing a JSON string.
+  std::ptrdiff_t BytesConsumed() const;
+   
   // Set the root type. May override the one set in the schema.
   bool SetRootType(const char *name);
 

src/idl_parser.cpp

@@ -3264,6 +3264,10 @@ bool Parser::ParseJson(const char *json, const char *json_filename) {
   return done;
 }
 
+std::ptrdiff_t Parser::BytesConsumed() const {
+  return std::distance(source_, cursor_);
+}
+
 CheckedError Parser::StartParseFile(const char *source,
                                     const char *source_filename) {
   file_being_parsed_ = source_filename ? source_filename : "";
@@ -3601,9 +3605,11 @@ CheckedError Parser::DoParseJson() {
                                                : nullptr);
     }
   }
+  if (opts.require_json_eof) {
     // Check that JSON file doesn't contain more objects or IDL directives.
     // Comments after JSON are allowed.
     EXPECT(kTokenEof);
+  }
   return NoError();
 }
 

tests/test.cpp

@@ -1402,6 +1402,49 @@ void NativeInlineTableVectorTest() {
   TEST_ASSERT(unpacked.t == test.t);
 }
 
+void DoNotRequireEofTest(const std::string& tests_data_path) {
+  std::string schemafile;
+  bool ok = flatbuffers::LoadFile(
+      (tests_data_path + "monster_test.fbs").c_str(), false, &schemafile);
+  TEST_EQ(ok, true);
+  auto include_test_path =
+      flatbuffers::ConCatPathFileName(tests_data_path, "include_test");
+  const char *include_directories[] = { tests_data_path.c_str(),
+                                        include_test_path.c_str(), nullptr };
+  flatbuffers::IDLOptions opt;
+  opt.require_json_eof = false;
+  flatbuffers::Parser parser(opt);
+  ok = parser.Parse(schemafile.c_str(), include_directories);
+  TEST_EQ(ok, true);
+  
+  const char *str = R"(This string contains two monsters, the first one is {
+      "name": "Blob",
+      "hp": 5
+    }
+    and the second one is {
+      "name": "Imp",
+      "hp": 10
+    }
+  )";
+  const char *tableStart = std::strchr(str, '{');
+  ok = parser.ParseJson(tableStart);
+  TEST_EQ(ok, true);
+
+  const Monster *monster = GetMonster(parser.builder_.GetBufferPointer());
+  TEST_EQ_STR(monster->name()->c_str(), "Blob");
+  TEST_EQ(monster->hp(), 5);
+  
+  tableStart += parser.BytesConsumed();
+
+  tableStart = std::strchr(tableStart + 1, '{');
+  ok = parser.ParseJson(tableStart);
+  TEST_EQ(ok, true);
+
+  monster = GetMonster(parser.builder_.GetBufferPointer());
+  TEST_EQ_STR(monster->name()->c_str(), "Imp");
+  TEST_EQ(monster->hp(), 10);
+}
+
 int FlatBufferTests(const std::string &tests_data_path) {
   // Run our various test suites:
 
@@ -1448,6 +1491,7 @@ int FlatBufferTests(const std::string &tests_data_path) {
   TestMonsterExtraFloats(tests_data_path);
   ParseIncorrectMonsterJsonTest(tests_data_path);
   FixedLengthArraySpanTest(tests_data_path);
+  DoNotRequireEofTest(tests_data_path);
 #endif
 
   UtilConvertCase();