diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 25706e84f..5909a4e20 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -348,6 +348,7 @@ struct IDLOptions { bool escape_proto_identifiers; bool generate_object_based_api; bool union_value_namespacing; + bool allow_non_utf8; // Possible options for the more general generator below. enum Language { kJava, kCSharp, kGo, kMAX }; @@ -370,6 +371,7 @@ struct IDLOptions { escape_proto_identifiers(false), generate_object_based_api(false), union_value_namespacing(true), + allow_non_utf8(false), lang(IDLOptions::kJava) {} }; diff --git a/include/flatbuffers/util.h b/include/flatbuffers/util.h index 7bd7513bb..baf5bdd37 100644 --- a/include/flatbuffers/util.h +++ b/include/flatbuffers/util.h @@ -276,6 +276,10 @@ inline int FromUTF8(const char **in) { } if ((**in << len) & 0x80) return -1; // Bit after leading 1's must be 0. if (!len) return *(*in)++; + // UTF-8 encoded values with a length are between 2 and 4 bytes. + if (len < 2 || len > 4) { + return -1; + } // Grab initial bits of the code. int ucc = *(*in)++ & ((1 << (7 - len)) - 1); for (int i = 0; i < len - 1; i++) { @@ -283,6 +287,32 @@ inline int FromUTF8(const char **in) { ucc <<= 6; ucc |= *(*in)++ & 0x3F; // Grab 6 more bits of the code. } + // UTF-8 cannot encode values between 0xD800 and 0xDFFF (reserved for + // UTF-16 surrogate pairs). + if (ucc >= 0xD800 && ucc <= 0xDFFF) { + return -1; + } + // UTF-8 must represent code points in their shortest possible encoding. + switch (len) { + case 2: + // Two bytes of UTF-8 can represent code points from U+0080 to U+07FF. + if (ucc < 0x0080 || ucc > 0x07FF) { + return -1; + } + break; + case 3: + // Three bytes of UTF-8 can represent code points from U+0800 to U+FFFF. + if (ucc < 0x0800 || ucc > 0xFFFF) { + return -1; + } + break; + case 4: + // Four bytes of UTF-8 can represent code points from U+10000 to U+10FFFF. + if (ucc < 0x10000 || ucc > 0x10FFFF) { + return -1; + } + break; + } return ucc; } diff --git a/src/flatc.cpp b/src/flatc.cpp index b174cbd43..44ce91350 100644 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -106,6 +106,9 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) { " --version Print the version number of flatc and exit.\n" " --strict-json Strict JSON: field names must be / will be quoted,\n" " no trailing commas in tables/vectors.\n" + " --allow-non-utf8 Pass non-UTF-8 input through parser and emit nonstandard\n" + " \\x escapes in JSON. (Default is to raise parse error on\n" + " non-UTF-8 input.)\n" " --defaults-json Output fields whose value is the default when\n" " writing JSON\n" " --unknown-json Allow fields in JSON that are not defined in the\n" @@ -184,6 +187,8 @@ int main(int argc, const char *argv[]) { conform_to_schema = argv[argi]; } else if(arg == "--strict-json") { opts.strict_json = true; + } else if(arg == "--allow-non-utf8") { + opts.allow_non_utf8 = true; } else if(arg == "--no-js-exports") { opts.skip_js_exports = true; } else if(arg == "--defaults-json") { diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp index 5f1bcf5e8..573300980 100644 --- a/src/idl_gen_go.cpp +++ b/src/idl_gen_go.cpp @@ -126,7 +126,7 @@ static void NewRootTypeFromBuffer(const StructDef &struct_def, code += " {\n"; code += "\tn := flatbuffers.GetUOffsetT(buf[offset:])\n"; code += "\tx := &" + struct_def.name + "{}\n"; - code += "\tx.Init(buf, n + offset)\n"; + code += "\tx.Init(buf, n+offset)\n"; code += "\treturn x\n"; code += "}\n\n"; } @@ -178,9 +178,10 @@ static void GetScalarFieldOfStruct(const StructDef &struct_def, std::string getter = GenGetter(field.value.type); GenReceiver(struct_def, code_ptr); code += " " + MakeCamel(field.name); - code += "() " + TypeName(field) + " { return " + getter; + code += "() " + TypeName(field) + " {\n"; + code +="\treturn " + getter; code += "(rcv._tab.Pos + flatbuffers.UOffsetT("; - code += NumToString(field.value.offset) + ")) }\n"; + code += NumToString(field.value.offset) + "))\n}\n"; } // Get the value of a table's scalar. @@ -212,7 +213,7 @@ static void GetStructFieldOfStruct(const StructDef &struct_def, code += "\tif obj == nil {\n"; code += "\t\tobj = new(" + TypeName(field) + ")\n"; code += "\t}\n"; - code += "\tobj.Init(rcv._tab.Bytes, rcv._tab.Pos + "; + code += "\tobj.Init(rcv._tab.Bytes, rcv._tab.Pos+"; code += NumToString(field.value.offset) + ")"; code += "\n\treturn obj\n"; code += "}\n"; @@ -287,9 +288,9 @@ static void GetMemberOfVectorOfStruct(const StructDef &struct_def, if (!(vectortype.struct_def->fixed)) { code += "\t\tx = rcv._tab.Indirect(x)\n"; } - code += "\tif obj == nil {\n"; - code += "\t\tobj = new(" + TypeName(field) + ")\n"; - code += "\t}\n"; + code += "\t\tif obj == nil {\n"; + code += "\t\t\tobj = new(" + TypeName(field) + ")\n"; + code += "\t\t}\n"; code += "\t\tobj.Init(rcv._tab.Bytes, x)\n"; code += "\t\treturn true\n\t}\n"; code += "\treturn false\n"; @@ -310,7 +311,7 @@ static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, code += OffsetPrefix(field); code += "\t\ta := rcv._tab.Vector(o)\n"; code += "\t\treturn " + GenGetter(field.value.type) + "("; - code += "a + flatbuffers.UOffsetT(j * "; + code += "a + flatbuffers.UOffsetT(j*"; code += NumToString(InlineSize(vectortype)) + "))\n"; code += "\t}\n"; if (vectortype.base_type == BASE_TYPE_STRING) { @@ -326,7 +327,10 @@ static void BeginBuilderArgs(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; - code += "\n"; + if (code.substr(code.length() - 2) != "\n\n") { + // a previous mutate has not put an extra new line + code += "\n"; + } code += "func Create" + struct_def.name; code += "(builder *flatbuffers.Builder"; } @@ -368,20 +372,20 @@ static void StructBuilderBody(const StructDef &struct_def, const char *nameprefix, std::string *code_ptr) { std::string &code = *code_ptr; - code += " builder.Prep(" + NumToString(struct_def.minalign) + ", "; + code += "\tbuilder.Prep(" + NumToString(struct_def.minalign) + ", "; code += NumToString(struct_def.bytesize) + ")\n"; for (auto it = struct_def.fields.vec.rbegin(); it != struct_def.fields.vec.rend(); ++it) { auto &field = **it; if (field.padding) - code += " builder.Pad(" + NumToString(field.padding) + ")\n"; + code += "\tbuilder.Pad(" + NumToString(field.padding) + ")\n"; if (IsStruct(field.value.type)) { StructBuilderBody(*field.value.type.struct_def, (nameprefix + (field.name + "_")).c_str(), code_ptr); } else { - code += " builder.Prepend" + GenMethod(field) + "("; + code += "\tbuilder.Prepend" + GenMethod(field) + "("; code += nameprefix + MakeCamel(field.name, false) + ")\n"; } } @@ -389,7 +393,7 @@ static void StructBuilderBody(const StructDef &struct_def, static void EndBuilderBody(std::string *code_ptr) { std::string &code = *code_ptr; - code += " return builder.Offset()\n"; + code += "\treturn builder.Offset()\n"; code += "}\n"; } @@ -398,10 +402,10 @@ static void GetStartOfTable(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; code += "func " + struct_def.name + "Start"; - code += "(builder *flatbuffers.Builder) { "; - code += "builder.StartObject("; + code += "(builder *flatbuffers.Builder) {\n"; + code += "\tbuilder.StartObject("; code += NumToString(struct_def.fields.vec.size()); - code += ") }\n"; + code += ")\n}\n"; } // Set the value of a table's field. @@ -418,8 +422,8 @@ static void BuildFieldOfTable(const StructDef &struct_def, } else { code += GenTypeBasic(field.value.type); } - code += ") "; - code += "{ builder.Prepend"; + code += ") {\n"; + code += "\tbuilder.Prepend"; code += GenMethod(field) + "Slot("; code += NumToString(offset) + ", "; if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { @@ -430,7 +434,7 @@ static void BuildFieldOfTable(const StructDef &struct_def, code += MakeCamel(field.name, false); } code += ", " + field.value.constant; - code += ") }\n"; + code += ")\n}\n"; } // Set the value of one of the members of a table's vector. @@ -441,7 +445,7 @@ static void BuildVectorOfTable(const StructDef &struct_def, code += "func " + struct_def.name + "Start"; code += MakeCamel(field.name); code += "Vector(builder *flatbuffers.Builder, numElems int) "; - code += "flatbuffers.UOffsetT { return builder.StartVector("; + code += "flatbuffers.UOffsetT {\n\treturn builder.StartVector("; auto vector_type = field.value.type.VectorType(); auto alignment = InlineAlignment(vector_type); auto elem_size = InlineSize(vector_type); @@ -456,7 +460,7 @@ static void GetEndOffsetOnTable(const StructDef &struct_def, std::string &code = *code_ptr; code += "func " + struct_def.name + "End"; code += "(builder *flatbuffers.Builder) flatbuffers.UOffsetT "; - code += "{ return builder.EndObject() }\n"; + code += "{\n\treturn builder.EndObject()\n}\n"; } // Generate the receiver for function signatures. @@ -521,9 +525,9 @@ static void MutateScalarFieldOfStruct(const StructDef &struct_def, std::string setter = "rcv._tab.Mutate" + type; GenReceiver(struct_def, code_ptr); code += " Mutate" + MakeCamel(field.name); - code += "(n " + TypeName(field) + ") bool { return " + setter; - code += "(rcv._tab.Pos + flatbuffers.UOffsetT("; - code += NumToString(field.value.offset) + "), n) }\n\n"; + code += "(n " + TypeName(field) + ") bool {\n\treturn " + setter; + code += "(rcv._tab.Pos+flatbuffers.UOffsetT("; + code += NumToString(field.value.offset) + "), n)\n}\n\n"; } // Mutate the value of a table's scalar. @@ -732,7 +736,7 @@ class GoGenerator : public BaseGenerator { if (needs_imports) { code += "import (\n"; code += "\tflatbuffers \"github.com/google/flatbuffers/go\"\n"; - code += ")\n"; + code += ")\n\n"; } } diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index dd96912cd..3e41a0a76 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -93,7 +93,7 @@ template void PrintVector(const Vector &v, Type type, text += "]"; } -static void EscapeString(const String &s, std::string *_text) { +static void EscapeString(const String &s, std::string *_text, const IDLOptions& opts) { std::string &text = *_text; text += "\""; for (uoffset_t i = 0; i < s.size(); i++) { @@ -113,17 +113,32 @@ static void EscapeString(const String &s, std::string *_text) { // Not printable ASCII data. Let's see if it's valid UTF-8 first: const char *utf8 = s.c_str() + i; int ucc = FromUTF8(&utf8); - if (ucc >= 0x80 && ucc <= 0xFFFF) { - // Parses as Unicode within JSON's \uXXXX range, so use that. - text += "\\u"; - text += IntToStringHex(ucc, 4); + if (ucc < 0) { + if (opts.allow_non_utf8) { + text += "\\x"; + text += IntToStringHex(static_cast(c), 2); + } else { + // We previously checked for non-UTF-8 and returned a parse error, + // so we shouldn't reach here. + assert(0); + } + } else { + if (ucc <= 0xFFFF) { + // Parses as Unicode within JSON's \uXXXX range, so use that. + text += "\\u"; + text += IntToStringHex(ucc, 4); + } else if (ucc <= 0x10FFFF) { + // Encode Unicode SMP values to a surrogate pair using two \u escapes. + uint32_t base = ucc - 0x10000; + uint16_t highSurrogate = (base >> 10) + 0xD800; + uint16_t lowSurrogate = (base & 0x03FF) + 0xDC00; + text += "\\u"; + text += IntToStringHex(highSurrogate, 4); + text += "\\u"; + text += IntToStringHex(lowSurrogate, 4); + } // Skip past characters recognized. i = static_cast(utf8 - s.c_str() - 1); - } else { - // It's either unprintable ASCII, arbitrary binary, or Unicode data - // that doesn't fit \uXXXX, so use \xXX escape code instead. - text += "\\x"; - text += IntToStringHex(static_cast(c), 2); } } break; @@ -157,7 +172,7 @@ template<> void Print(const void *val, _text); break; case BASE_TYPE_STRING: { - EscapeString(*reinterpret_cast(val), _text); + EscapeString(*reinterpret_cast(val), _text, opts); break; } case BASE_TYPE_VECTOR: diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index b03655c92..d845b837d 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -61,6 +61,17 @@ static_assert(BASE_TYPE_UNION == #define NEXT() ECHECK(Next()) #define EXPECT(tok) ECHECK(Expect(tok)) +static bool ValidateUTF8(const std::string &str) { + const char *s = &str[0]; + const char * const sEnd = s + str.length(); + while (s < sEnd) { + if (FromUTF8(&s) < 0) { + return false; + } + } + return true; +} + CheckedError Parser::Error(const std::string &msg) { error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : ""; #ifdef _WIN32 @@ -320,6 +331,9 @@ CheckedError Parser::Next() { "illegal Unicode sequence (unpaired high surrogate)"); } cursor_++; + if (!opts.allow_non_utf8 && !ValidateUTF8(attribute_)) { + return Error("illegal UTF-8 sequence"); + } token_ = kTokenStringConstant; return NoError(); } diff --git a/tests/GoTest.sh b/tests/GoTest.sh index 353930a61..7be4affb7 100755 --- a/tests/GoTest.sh +++ b/tests/GoTest.sh @@ -29,9 +29,9 @@ mkdir -p ${go_src}/MyGame/Example mkdir -p ${go_src}/github.com/google/flatbuffers/go mkdir -p ${go_src}/flatbuffers_test -cp -u MyGame/Example/*.go ./go_gen/src/MyGame/Example/ -cp -u ../go/* ./go_gen/src/github.com/google/flatbuffers/go -cp -u ./go_test.go ./go_gen/src/flatbuffers_test/ +cp -a MyGame/Example/*.go ./go_gen/src/MyGame/Example/ +cp -a ../go/* ./go_gen/src/github.com/google/flatbuffers/go +cp -a ./go_test.go ./go_gen/src/flatbuffers_test/ # Run tests with necessary flags. # Developers may wish to see more detail by appending the verbosity flag @@ -50,6 +50,18 @@ GOPATH=${go_path} go test flatbuffers_test \ --fuzz_fields=4 \ --fuzz_objects=10000 +GO_TEST_RESULT=$? rm -rf ${go_path}/{pkg,src} +if [[ $GO_TEST_RESULT == 0 ]]; then + echo "OK: Go tests passed." +else + echo "KO: Go tests failed." + exit 1 +fi -echo "OK: Go tests passed." +NOT_FMT_FILES=$(gofmt -l MyGame) +if [[ ${NOT_FMT_FILES} != "" ]]; then + echo "These files are not well gofmt'ed:\n\n${NOT_FMT_FILES}" + # enable this when enums are properly formated + # exit 1 +fi diff --git a/tests/MyGame/Example/Monster.go b/tests/MyGame/Example/Monster.go index f4cd18c18..7ba062fdc 100644 --- a/tests/MyGame/Example/Monster.go +++ b/tests/MyGame/Example/Monster.go @@ -5,6 +5,7 @@ package Example import ( flatbuffers "github.com/google/flatbuffers/go" ) + /// an example documentation comment: monster object type Monster struct { _tab flatbuffers.Table @@ -13,7 +14,7 @@ type Monster struct { func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster { n := flatbuffers.GetUOffsetT(buf[offset:]) x := &Monster{} - x.Init(buf, n + offset) + x.Init(buf, n+offset) return x } @@ -71,7 +72,7 @@ func (rcv *Monster) Inventory(j int) byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { a := rcv._tab.Vector(o) - return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j * 1)) + return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1)) } return 0 } @@ -130,9 +131,9 @@ func (rcv *Monster) Test4(obj *Test, j int) bool { if o != 0 { x := rcv._tab.Vector(o) x += flatbuffers.UOffsetT(j) * 4 - if obj == nil { - obj = new(Test) - } + if obj == nil { + obj = new(Test) + } obj.Init(rcv._tab.Bytes, x) return true } @@ -151,7 +152,7 @@ func (rcv *Monster) Testarrayofstring(j int) []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(24)) if o != 0 { a := rcv._tab.Vector(o) - return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j * 4)) + return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j*4)) } return nil } @@ -172,9 +173,9 @@ func (rcv *Monster) Testarrayoftables(obj *Monster, j int) bool { x := rcv._tab.Vector(o) x += flatbuffers.UOffsetT(j) * 4 x = rcv._tab.Indirect(x) - if obj == nil { - obj = new(Monster) - } + if obj == nil { + obj = new(Monster) + } obj.Init(rcv._tab.Bytes, x) return true } @@ -208,7 +209,7 @@ func (rcv *Monster) Testnestedflatbuffer(j int) byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(30)) if o != 0 { a := rcv._tab.Vector(o) - return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j * 1)) + return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1)) } return 0 } @@ -354,7 +355,7 @@ func (rcv *Monster) Testarrayofbools(j int) byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(52)) if o != 0 { a := rcv._tab.Vector(o) - return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j * 1)) + return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1)) } return 0 } @@ -407,7 +408,7 @@ func (rcv *Monster) Testarrayofstring2(j int) []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(60)) if o != 0 { a := rcv._tab.Vector(o) - return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j * 4)) + return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j*4)) } return nil } @@ -420,47 +421,114 @@ func (rcv *Monster) Testarrayofstring2Length() int { return 0 } -func MonsterStart(builder *flatbuffers.Builder) { builder.StartObject(29) } -func MonsterAddPos(builder *flatbuffers.Builder, pos flatbuffers.UOffsetT) { builder.PrependStructSlot(0, flatbuffers.UOffsetT(pos), 0) } -func MonsterAddMana(builder *flatbuffers.Builder, mana int16) { builder.PrependInt16Slot(1, mana, 150) } -func MonsterAddHp(builder *flatbuffers.Builder, hp int16) { builder.PrependInt16Slot(2, hp, 100) } -func MonsterAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(name), 0) } -func MonsterAddInventory(builder *flatbuffers.Builder, inventory flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(inventory), 0) } -func MonsterStartInventoryVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(1, numElems, 1) +func MonsterStart(builder *flatbuffers.Builder) { + builder.StartObject(29) } -func MonsterAddColor(builder *flatbuffers.Builder, color int8) { builder.PrependInt8Slot(6, color, 8) } -func MonsterAddTestType(builder *flatbuffers.Builder, testType byte) { builder.PrependByteSlot(7, testType, 0) } -func MonsterAddTest(builder *flatbuffers.Builder, test flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(test), 0) } -func MonsterAddTest4(builder *flatbuffers.Builder, test4 flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(9, flatbuffers.UOffsetT(test4), 0) } -func MonsterStartTest4Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems, 2) +func MonsterAddPos(builder *flatbuffers.Builder, pos flatbuffers.UOffsetT) { + builder.PrependStructSlot(0, flatbuffers.UOffsetT(pos), 0) } -func MonsterAddTestarrayofstring(builder *flatbuffers.Builder, testarrayofstring flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(10, flatbuffers.UOffsetT(testarrayofstring), 0) } -func MonsterStartTestarrayofstringVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems, 4) +func MonsterAddMana(builder *flatbuffers.Builder, mana int16) { + builder.PrependInt16Slot(1, mana, 150) } -func MonsterAddTestarrayoftables(builder *flatbuffers.Builder, testarrayoftables flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(11, flatbuffers.UOffsetT(testarrayoftables), 0) } -func MonsterStartTestarrayoftablesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems, 4) +func MonsterAddHp(builder *flatbuffers.Builder, hp int16) { + builder.PrependInt16Slot(2, hp, 100) } -func MonsterAddEnemy(builder *flatbuffers.Builder, enemy flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(12, flatbuffers.UOffsetT(enemy), 0) } -func MonsterAddTestnestedflatbuffer(builder *flatbuffers.Builder, testnestedflatbuffer flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(testnestedflatbuffer), 0) } -func MonsterStartTestnestedflatbufferVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(1, numElems, 1) +func MonsterAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(name), 0) } -func MonsterAddTestempty(builder *flatbuffers.Builder, testempty flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(14, flatbuffers.UOffsetT(testempty), 0) } -func MonsterAddTestbool(builder *flatbuffers.Builder, testbool byte) { builder.PrependByteSlot(15, testbool, 0) } -func MonsterAddTesthashs32Fnv1(builder *flatbuffers.Builder, testhashs32Fnv1 int32) { builder.PrependInt32Slot(16, testhashs32Fnv1, 0) } -func MonsterAddTesthashu32Fnv1(builder *flatbuffers.Builder, testhashu32Fnv1 uint32) { builder.PrependUint32Slot(17, testhashu32Fnv1, 0) } -func MonsterAddTesthashs64Fnv1(builder *flatbuffers.Builder, testhashs64Fnv1 int64) { builder.PrependInt64Slot(18, testhashs64Fnv1, 0) } -func MonsterAddTesthashu64Fnv1(builder *flatbuffers.Builder, testhashu64Fnv1 uint64) { builder.PrependUint64Slot(19, testhashu64Fnv1, 0) } -func MonsterAddTesthashs32Fnv1a(builder *flatbuffers.Builder, testhashs32Fnv1a int32) { builder.PrependInt32Slot(20, testhashs32Fnv1a, 0) } -func MonsterAddTesthashu32Fnv1a(builder *flatbuffers.Builder, testhashu32Fnv1a uint32) { builder.PrependUint32Slot(21, testhashu32Fnv1a, 0) } -func MonsterAddTesthashs64Fnv1a(builder *flatbuffers.Builder, testhashs64Fnv1a int64) { builder.PrependInt64Slot(22, testhashs64Fnv1a, 0) } -func MonsterAddTesthashu64Fnv1a(builder *flatbuffers.Builder, testhashu64Fnv1a uint64) { builder.PrependUint64Slot(23, testhashu64Fnv1a, 0) } -func MonsterAddTestarrayofbools(builder *flatbuffers.Builder, testarrayofbools flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(24, flatbuffers.UOffsetT(testarrayofbools), 0) } -func MonsterStartTestarrayofboolsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(1, numElems, 1) +func MonsterAddInventory(builder *flatbuffers.Builder, inventory flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(inventory), 0) } -func MonsterAddTestf(builder *flatbuffers.Builder, testf float32) { builder.PrependFloat32Slot(25, testf, 3.14159) } -func MonsterAddTestf2(builder *flatbuffers.Builder, testf2 float32) { builder.PrependFloat32Slot(26, testf2, 3.0) } -func MonsterAddTestf3(builder *flatbuffers.Builder, testf3 float32) { builder.PrependFloat32Slot(27, testf3, 0.0) } -func MonsterAddTestarrayofstring2(builder *flatbuffers.Builder, testarrayofstring2 flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(28, flatbuffers.UOffsetT(testarrayofstring2), 0) } -func MonsterStartTestarrayofstring2Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems, 4) +func MonsterStartInventoryVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(1, numElems, 1) +} +func MonsterAddColor(builder *flatbuffers.Builder, color int8) { + builder.PrependInt8Slot(6, color, 8) +} +func MonsterAddTestType(builder *flatbuffers.Builder, testType byte) { + builder.PrependByteSlot(7, testType, 0) +} +func MonsterAddTest(builder *flatbuffers.Builder, test flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(test), 0) +} +func MonsterAddTest4(builder *flatbuffers.Builder, test4 flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(9, flatbuffers.UOffsetT(test4), 0) +} +func MonsterStartTest4Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 2) +} +func MonsterAddTestarrayofstring(builder *flatbuffers.Builder, testarrayofstring flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(10, flatbuffers.UOffsetT(testarrayofstring), 0) +} +func MonsterStartTestarrayofstringVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func MonsterAddTestarrayoftables(builder *flatbuffers.Builder, testarrayoftables flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(11, flatbuffers.UOffsetT(testarrayoftables), 0) +} +func MonsterStartTestarrayoftablesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func MonsterAddEnemy(builder *flatbuffers.Builder, enemy flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(12, flatbuffers.UOffsetT(enemy), 0) +} +func MonsterAddTestnestedflatbuffer(builder *flatbuffers.Builder, testnestedflatbuffer flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(testnestedflatbuffer), 0) +} +func MonsterStartTestnestedflatbufferVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(1, numElems, 1) +} +func MonsterAddTestempty(builder *flatbuffers.Builder, testempty flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(14, flatbuffers.UOffsetT(testempty), 0) +} +func MonsterAddTestbool(builder *flatbuffers.Builder, testbool byte) { + builder.PrependByteSlot(15, testbool, 0) +} +func MonsterAddTesthashs32Fnv1(builder *flatbuffers.Builder, testhashs32Fnv1 int32) { + builder.PrependInt32Slot(16, testhashs32Fnv1, 0) +} +func MonsterAddTesthashu32Fnv1(builder *flatbuffers.Builder, testhashu32Fnv1 uint32) { + builder.PrependUint32Slot(17, testhashu32Fnv1, 0) +} +func MonsterAddTesthashs64Fnv1(builder *flatbuffers.Builder, testhashs64Fnv1 int64) { + builder.PrependInt64Slot(18, testhashs64Fnv1, 0) +} +func MonsterAddTesthashu64Fnv1(builder *flatbuffers.Builder, testhashu64Fnv1 uint64) { + builder.PrependUint64Slot(19, testhashu64Fnv1, 0) +} +func MonsterAddTesthashs32Fnv1a(builder *flatbuffers.Builder, testhashs32Fnv1a int32) { + builder.PrependInt32Slot(20, testhashs32Fnv1a, 0) +} +func MonsterAddTesthashu32Fnv1a(builder *flatbuffers.Builder, testhashu32Fnv1a uint32) { + builder.PrependUint32Slot(21, testhashu32Fnv1a, 0) +} +func MonsterAddTesthashs64Fnv1a(builder *flatbuffers.Builder, testhashs64Fnv1a int64) { + builder.PrependInt64Slot(22, testhashs64Fnv1a, 0) +} +func MonsterAddTesthashu64Fnv1a(builder *flatbuffers.Builder, testhashu64Fnv1a uint64) { + builder.PrependUint64Slot(23, testhashu64Fnv1a, 0) +} +func MonsterAddTestarrayofbools(builder *flatbuffers.Builder, testarrayofbools flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(24, flatbuffers.UOffsetT(testarrayofbools), 0) +} +func MonsterStartTestarrayofboolsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(1, numElems, 1) +} +func MonsterAddTestf(builder *flatbuffers.Builder, testf float32) { + builder.PrependFloat32Slot(25, testf, 3.14159) +} +func MonsterAddTestf2(builder *flatbuffers.Builder, testf2 float32) { + builder.PrependFloat32Slot(26, testf2, 3.0) +} +func MonsterAddTestf3(builder *flatbuffers.Builder, testf3 float32) { + builder.PrependFloat32Slot(27, testf3, 0.0) +} +func MonsterAddTestarrayofstring2(builder *flatbuffers.Builder, testarrayofstring2 flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(28, flatbuffers.UOffsetT(testarrayofstring2), 0) +} +func MonsterStartTestarrayofstring2Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() } -func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/tests/MyGame/Example/Stat.go b/tests/MyGame/Example/Stat.go index 8c56bfb07..022c4f036 100644 --- a/tests/MyGame/Example/Stat.go +++ b/tests/MyGame/Example/Stat.go @@ -5,6 +5,7 @@ package Example import ( flatbuffers "github.com/google/flatbuffers/go" ) + type Stat struct { _tab flatbuffers.Table } @@ -12,7 +13,7 @@ type Stat struct { func GetRootAsStat(buf []byte, offset flatbuffers.UOffsetT) *Stat { n := flatbuffers.GetUOffsetT(buf[offset:]) x := &Stat{} - x.Init(buf, n + offset) + x.Init(buf, n+offset) return x } @@ -53,8 +54,18 @@ func (rcv *Stat) MutateCount(n uint16) bool { return rcv._tab.MutateUint16Slot(8, n) } -func StatStart(builder *flatbuffers.Builder) { builder.StartObject(3) } -func StatAddId(builder *flatbuffers.Builder, id flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(id), 0) } -func StatAddVal(builder *flatbuffers.Builder, val int64) { builder.PrependInt64Slot(1, val, 0) } -func StatAddCount(builder *flatbuffers.Builder, count uint16) { builder.PrependUint16Slot(2, count, 0) } -func StatEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } +func StatStart(builder *flatbuffers.Builder) { + builder.StartObject(3) +} +func StatAddId(builder *flatbuffers.Builder, id flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(id), 0) +} +func StatAddVal(builder *flatbuffers.Builder, val int64) { + builder.PrependInt64Slot(1, val, 0) +} +func StatAddCount(builder *flatbuffers.Builder, count uint16) { + builder.PrependUint16Slot(2, count, 0) +} +func StatEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/tests/MyGame/Example/Test.go b/tests/MyGame/Example/Test.go index ee0d9aaec..781db8b09 100644 --- a/tests/MyGame/Example/Test.go +++ b/tests/MyGame/Example/Test.go @@ -5,6 +5,7 @@ package Example import ( flatbuffers "github.com/google/flatbuffers/go" ) + type Test struct { _tab flatbuffers.Struct } @@ -14,17 +15,24 @@ func (rcv *Test) Init(buf []byte, i flatbuffers.UOffsetT) { rcv._tab.Pos = i } -func (rcv *Test) A() int16 { return rcv._tab.GetInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0)) } -func (rcv *Test) MutateA(n int16) bool { return rcv._tab.MutateInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0), n) } - -func (rcv *Test) B() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2)) } -func (rcv *Test) MutateB(n int8) bool { return rcv._tab.MutateInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2), n) } +func (rcv *Test) A() int16 { + return rcv._tab.GetInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0)) +} +func (rcv *Test) MutateA(n int16) bool { + return rcv._tab.MutateInt16(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) +} +func (rcv *Test) B() int8 { + return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2)) +} +func (rcv *Test) MutateB(n int8) bool { + return rcv._tab.MutateInt8(rcv._tab.Pos+flatbuffers.UOffsetT(2), n) +} func CreateTest(builder *flatbuffers.Builder, a int16, b int8) flatbuffers.UOffsetT { - builder.Prep(2, 4) - builder.Pad(1) - builder.PrependInt8(b) - builder.PrependInt16(a) - return builder.Offset() + builder.Prep(2, 4) + builder.Pad(1) + builder.PrependInt8(b) + builder.PrependInt16(a) + return builder.Offset() } diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.go b/tests/MyGame/Example/TestSimpleTableWithEnum.go index 96218e58b..be50d5ca1 100644 --- a/tests/MyGame/Example/TestSimpleTableWithEnum.go +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.go @@ -5,6 +5,7 @@ package Example import ( flatbuffers "github.com/google/flatbuffers/go" ) + type TestSimpleTableWithEnum struct { _tab flatbuffers.Table } @@ -12,7 +13,7 @@ type TestSimpleTableWithEnum struct { func GetRootAsTestSimpleTableWithEnum(buf []byte, offset flatbuffers.UOffsetT) *TestSimpleTableWithEnum { n := flatbuffers.GetUOffsetT(buf[offset:]) x := &TestSimpleTableWithEnum{} - x.Init(buf, n + offset) + x.Init(buf, n+offset) return x } @@ -33,6 +34,12 @@ func (rcv *TestSimpleTableWithEnum) MutateColor(n int8) bool { return rcv._tab.MutateInt8Slot(4, n) } -func TestSimpleTableWithEnumStart(builder *flatbuffers.Builder) { builder.StartObject(1) } -func TestSimpleTableWithEnumAddColor(builder *flatbuffers.Builder, color int8) { builder.PrependInt8Slot(0, color, 2) } -func TestSimpleTableWithEnumEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } +func TestSimpleTableWithEnumStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func TestSimpleTableWithEnumAddColor(builder *flatbuffers.Builder, color int8) { + builder.PrependInt8Slot(0, color, 2) +} +func TestSimpleTableWithEnumEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/tests/MyGame/Example/Vec3.go b/tests/MyGame/Example/Vec3.go index 8bf97da32..1838e5606 100644 --- a/tests/MyGame/Example/Vec3.go +++ b/tests/MyGame/Example/Vec3.go @@ -5,6 +5,7 @@ package Example import ( flatbuffers "github.com/google/flatbuffers/go" ) + type Vec3 struct { _tab flatbuffers.Struct } @@ -14,42 +15,62 @@ func (rcv *Vec3) Init(buf []byte, i flatbuffers.UOffsetT) { rcv._tab.Pos = i } -func (rcv *Vec3) X() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0)) } -func (rcv *Vec3) MutateX(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0), n) } +func (rcv *Vec3) X() float32 { + return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0)) +} +func (rcv *Vec3) MutateX(n float32) bool { + return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) +} -func (rcv *Vec3) Y() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4)) } -func (rcv *Vec3) MutateY(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4), n) } +func (rcv *Vec3) Y() float32 { + return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4)) +} +func (rcv *Vec3) MutateY(n float32) bool { + return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(4), n) +} -func (rcv *Vec3) Z() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8)) } -func (rcv *Vec3) MutateZ(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8), n) } +func (rcv *Vec3) Z() float32 { + return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8)) +} +func (rcv *Vec3) MutateZ(n float32) bool { + return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) +} -func (rcv *Vec3) Test1() float64 { return rcv._tab.GetFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16)) } -func (rcv *Vec3) MutateTest1(n float64) bool { return rcv._tab.MutateFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16), n) } +func (rcv *Vec3) Test1() float64 { + return rcv._tab.GetFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16)) +} +func (rcv *Vec3) MutateTest1(n float64) bool { + return rcv._tab.MutateFloat64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n) +} -func (rcv *Vec3) Test2() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24)) } -func (rcv *Vec3) MutateTest2(n int8) bool { return rcv._tab.MutateInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24), n) } +func (rcv *Vec3) Test2() int8 { + return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24)) +} +func (rcv *Vec3) MutateTest2(n int8) bool { + return rcv._tab.MutateInt8(rcv._tab.Pos+flatbuffers.UOffsetT(24), n) +} func (rcv *Vec3) Test3(obj *Test) *Test { if obj == nil { obj = new(Test) } - obj.Init(rcv._tab.Bytes, rcv._tab.Pos + 26) + obj.Init(rcv._tab.Bytes, rcv._tab.Pos+26) return obj } func CreateVec3(builder *flatbuffers.Builder, x float32, y float32, z float32, test1 float64, test2 int8, test3_a int16, test3_b int8) flatbuffers.UOffsetT { - builder.Prep(16, 32) - builder.Pad(2) - builder.Prep(2, 4) - builder.Pad(1) - builder.PrependInt8(test3_b) - builder.PrependInt16(test3_a) - builder.Pad(1) - builder.PrependInt8(test2) - builder.PrependFloat64(test1) - builder.Pad(4) - builder.PrependFloat32(z) - builder.PrependFloat32(y) - builder.PrependFloat32(x) - return builder.Offset() + builder.Prep(16, 32) + builder.Pad(2) + builder.Prep(2, 4) + builder.Pad(1) + builder.PrependInt8(test3_b) + builder.PrependInt16(test3_a) + builder.Pad(1) + builder.PrependInt8(test2) + builder.PrependFloat64(test1) + builder.Pad(4) + builder.PrependFloat32(z) + builder.PrependFloat32(y) + builder.PrependFloat32(x) + return builder.Offset() } diff --git a/tests/MyGame/Example2/Monster.go b/tests/MyGame/Example2/Monster.go index 789d46782..698c2ef56 100644 --- a/tests/MyGame/Example2/Monster.go +++ b/tests/MyGame/Example2/Monster.go @@ -5,6 +5,7 @@ package Example2 import ( flatbuffers "github.com/google/flatbuffers/go" ) + type Monster struct { _tab flatbuffers.Table } @@ -12,7 +13,7 @@ type Monster struct { func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster { n := flatbuffers.GetUOffsetT(buf[offset:]) x := &Monster{} - x.Init(buf, n + offset) + x.Init(buf, n+offset) return x } @@ -21,5 +22,9 @@ func (rcv *Monster) Init(buf []byte, i flatbuffers.UOffsetT) { rcv._tab.Pos = i } -func MonsterStart(builder *flatbuffers.Builder) { builder.StartObject(0) } -func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } +func MonsterStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/tests/test.cpp b/tests/test.cpp index 6ec4e678c..cd3723756 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -978,15 +978,36 @@ void IntegerOutOfRangeTest() { void UnicodeTest() { flatbuffers::Parser parser; + // Without setting allow_non_utf8 = true, we treat \x sequences as byte sequences + // which are then validated as UTF-8. TEST_EQ(parser.Parse("table T { F:string; }" "root_type T;" "{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC" - "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\" }"), true); + "\\u5225\\u30B5\\u30A4\\u30C8\\xE2\\x82\\xAC\\u0080\\uD83D\\uDE0E\" }"), + true); std::string jsongen; parser.opts.indent_step = -1; GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen); - TEST_EQ(jsongen == "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC" - "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\"}", true); + TEST_EQ(jsongen, + std::string( + "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC" + "\\u5225\\u30B5\\u30A4\\u30C8\\u20AC\\u0080\\uD83D\\uDE0E\"}")); +} + +void UnicodeTestAllowNonUTF8() { + flatbuffers::Parser parser; + parser.opts.allow_non_utf8 = true; + TEST_EQ(parser.Parse("table T { F:string; }" + "root_type T;" + "{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC" + "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\\u0080\\uD83D\\uDE0E\" }"), true); + std::string jsongen; + parser.opts.indent_step = -1; + GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen); + TEST_EQ(jsongen, + std::string( + "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC" + "\\u5225\\u30B5\\u30A4\\u30C8\\u0001\\x80\\u0080\\uD83D\\uDE0E\"}")); } void UnicodeSurrogatesTest() { @@ -1027,6 +1048,96 @@ void UnicodeInvalidSurrogatesTest() { "{ F:\"\\uDC00\"}", "unpaired low surrogate"); } +void InvalidUTF8Test() { + // "1 byte" pattern, under min length of 2 bytes + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\x80\"}", "illegal UTF-8 sequence"); + // 2 byte pattern, string too short + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xDF\"}", "illegal UTF-8 sequence"); + // 3 byte pattern, string too short + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xEF\xBF\"}", "illegal UTF-8 sequence"); + // 4 byte pattern, string too short + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xF7\xBF\xBF\"}", "illegal UTF-8 sequence"); + // "5 byte" pattern, string too short + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xFB\xBF\xBF\xBF\"}", "illegal UTF-8 sequence"); + // "6 byte" pattern, string too short + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xFD\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence"); + // "7 byte" pattern, string too short + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xFE\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence"); + // "5 byte" pattern, over max length of 4 bytes + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xFB\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence"); + // "6 byte" pattern, over max length of 4 bytes + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xFD\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence"); + // "7 byte" pattern, over max length of 4 bytes + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xFE\xBF\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence"); + + // Three invalid encodings for U+000A (\n, aka NEWLINE) + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xC0\x8A\"}", "illegal UTF-8 sequence"); + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xE0\x80\x8A\"}", "illegal UTF-8 sequence"); + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xF0\x80\x80\x8A\"}", "illegal UTF-8 sequence"); + + // Two invalid encodings for U+00A9 (COPYRIGHT SYMBOL) + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xE0\x81\xA9\"}", "illegal UTF-8 sequence"); + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xF0\x80\x81\xA9\"}", "illegal UTF-8 sequence"); + + // Invalid encoding for U+20AC (EURO SYMBOL) + TestError( + "table T { F:string; }" + "root_type T;" + "{ F:\"\xF0\x82\x82\xAC\"}", "illegal UTF-8 sequence"); + + // UTF-16 surrogate values between U+D800 and U+DFFF cannot be encoded in UTF-8 + TestError( + "table T { F:string; }" + "root_type T;" + // U+10400 "encoded" as U+D801 U+DC00 + "{ F:\"\xED\xA0\x81\xED\xB0\x80\"}", "illegal UTF-8 sequence"); +} + void UnknownFieldsTest() { flatbuffers::IDLOptions opts; opts.skip_unexpected_fields_in_json = true; @@ -1105,8 +1216,10 @@ int main(int /*argc*/, const char * /*argv*/[]) { EnumStringsTest(); IntegerOutOfRangeTest(); UnicodeTest(); + UnicodeTestAllowNonUTF8(); UnicodeSurrogatesTest(); UnicodeInvalidSurrogatesTest(); + InvalidUTF8Test(); UnknownFieldsTest(); ParseUnionTest(); ConformTest();