From a4ff275d9b204c5c6fac917bfe1cb9455caa9e94 Mon Sep 17 00:00:00 2001
From: Alex Ames <alexander.ames@gmail.com>
Date: Thu, 3 Nov 2022 08:57:46 -0700
Subject: [PATCH] Added option to not requires an EoF token when parsing JSON
 (#7620)

Previously when parsing a JSON representation of a Flatbuffer, the
parser required that the input string contain one and only one root
table. This change adds a flag that removes that requirement, so that
if a Flatbuffer table is embedded in some larger string the parser will
simply stop parsing once it reaches the end of the root table, and does
not validate that it has reached the end of the string.

This change also adds a BytesConsumed function, which returns the number
of bytes the parser consumed. This is useful if the table embedded in
some larger string that is being parsed, and that outer parser needs to
know how many bytes the table was so that it can step over it.
---
 include/flatbuffers/idl.h |  5 +++++
 src/idl_parser.cpp        | 12 ++++++++---
 tests/test.cpp            | 44 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h
index 562bb420a..fd2da8bfa 100644
--- a/include/flatbuffers/idl.h
+++ b/include/flatbuffers/idl.h
@@ -642,6 +642,7 @@ struct IDLOptions {
   bool json_nested_legacy_flatbuffers;
   bool ts_flat_file;
   bool no_leak_private_annotations;
+  bool require_json_eof;
 
   // Possible options for the more general generator below.
   enum Language {
@@ -743,6 +744,7 @@ struct IDLOptions {
         json_nested_legacy_flatbuffers(false),
         ts_flat_file(false),
         no_leak_private_annotations(false),
+        require_json_eof(true),
         mini_reflect(IDLOptions::kNone),
         require_explicit_ids(false),
         rust_serialize(false),
@@ -905,6 +907,9 @@ class Parser : public ParserState {
 
   bool ParseJson(const char *json, const char *json_filename = nullptr);
 
+  // Returns the number of characters were consumed when parsing a JSON string.
+  std::ptrdiff_t BytesConsumed() const;
+   
   // Set the root type. May override the one set in the schema.
   bool SetRootType(const char *name);
 
diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp
index ef4a5bc96..caa26ba0e 100644
--- a/src/idl_parser.cpp
+++ b/src/idl_parser.cpp
@@ -3264,6 +3264,10 @@ bool Parser::ParseJson(const char *json, const char *json_filename) {
   return done;
 }
 
+std::ptrdiff_t Parser::BytesConsumed() const {
+  return std::distance(source_, cursor_);
+}
+
 CheckedError Parser::StartParseFile(const char *source,
                                     const char *source_filename) {
   file_being_parsed_ = source_filename ? source_filename : "";
@@ -3601,9 +3605,11 @@ CheckedError Parser::DoParseJson() {
                                                : nullptr);
     }
   }
-  // Check that JSON file doesn't contain more objects or IDL directives.
-  // Comments after JSON are allowed.
-  EXPECT(kTokenEof);
+  if (opts.require_json_eof) {
+    // Check that JSON file doesn't contain more objects or IDL directives.
+    // Comments after JSON are allowed.
+    EXPECT(kTokenEof);
+  }
   return NoError();
 }
 
diff --git a/tests/test.cpp b/tests/test.cpp
index 8c5c02710..6e5dbc4e3 100644
--- a/tests/test.cpp
+++ b/tests/test.cpp
@@ -1402,6 +1402,49 @@ void NativeInlineTableVectorTest() {
   TEST_ASSERT(unpacked.t == test.t);
 }
 
+void DoNotRequireEofTest(const std::string& tests_data_path) {
+  std::string schemafile;
+  bool ok = flatbuffers::LoadFile(
+      (tests_data_path + "monster_test.fbs").c_str(), false, &schemafile);
+  TEST_EQ(ok, true);
+  auto include_test_path =
+      flatbuffers::ConCatPathFileName(tests_data_path, "include_test");
+  const char *include_directories[] = { tests_data_path.c_str(),
+                                        include_test_path.c_str(), nullptr };
+  flatbuffers::IDLOptions opt;
+  opt.require_json_eof = false;
+  flatbuffers::Parser parser(opt);
+  ok = parser.Parse(schemafile.c_str(), include_directories);
+  TEST_EQ(ok, true);
+  
+  const char *str = R"(This string contains two monsters, the first one is {
+      "name": "Blob",
+      "hp": 5
+    }
+    and the second one is {
+      "name": "Imp",
+      "hp": 10
+    }
+  )";
+  const char *tableStart = std::strchr(str, '{');
+  ok = parser.ParseJson(tableStart);
+  TEST_EQ(ok, true);
+
+  const Monster *monster = GetMonster(parser.builder_.GetBufferPointer());
+  TEST_EQ_STR(monster->name()->c_str(), "Blob");
+  TEST_EQ(monster->hp(), 5);
+  
+  tableStart += parser.BytesConsumed();
+
+  tableStart = std::strchr(tableStart + 1, '{');
+  ok = parser.ParseJson(tableStart);
+  TEST_EQ(ok, true);
+
+  monster = GetMonster(parser.builder_.GetBufferPointer());
+  TEST_EQ_STR(monster->name()->c_str(), "Imp");
+  TEST_EQ(monster->hp(), 10);
+}
+
 int FlatBufferTests(const std::string &tests_data_path) {
   // Run our various test suites:
 
@@ -1448,6 +1491,7 @@ int FlatBufferTests(const std::string &tests_data_path) {
   TestMonsterExtraFloats(tests_data_path);
   ParseIncorrectMonsterJsonTest(tests_data_path);
   FixedLengthArraySpanTest(tests_data_path);
+  DoNotRequireEofTest(tests_data_path);
 #endif
 
   UtilConvertCase();