From 90baa1444b59af408e5363b284d4333d35835e8d Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <6349682+vaind@users.noreply.github.com> Date: Wed, 15 Sep 2021 17:50:57 +0200 Subject: [PATCH] Dart: binary lists (typed_data) (#6839) * Dart - add eager mode to Uint8ListReader * Dart - add Int8ListReader * Dart - use binary reader where useful * Dart - test binary list reader laziness * Dart - update generated code --- dart/lib/flat_buffers.dart | 72 +++++++++++++++++-- dart/lib/src/reference.dart | 3 +- dart/test/flat_buffers_test.dart | 18 +++-- ...onster_test_my_game.example_generated.dart | 20 +++--- src/idl_gen_dart.cpp | 8 +++ ...onster_test_my_game.example_generated.dart | 20 +++--- 6 files changed, 108 insertions(+), 33 deletions(-) diff --git a/dart/lib/flat_buffers.dart b/dart/lib/flat_buffers.dart index 215ee5109..22bd788b6 100644 --- a/dart/lib/flat_buffers.dart +++ b/dart/lib/flat_buffers.dart @@ -155,7 +155,8 @@ class Builder { } } - /// Calculate the finished buffer size (aligned).@pragma('vm:prefer-inline') + /// Calculate the finished buffer size (aligned). + @pragma('vm:prefer-inline') int size() => _tail + ((-_tail) & (_maxAlign - 1)); /// Add the [field] with the given boolean [value]. The field is not added if @@ -1170,11 +1171,17 @@ class Uint16Reader extends Reader { int read(BufferContext bc, int offset) => bc._getUint16(offset); } -/// Reader of lists of unsigned 8-bit integer values. -/// -/// The returned unmodifiable lists lazily read values on access. +/// Reader of unmodifiable binary data (a list of unsigned 8-bit integers). class Uint8ListReader extends Reader> { - const Uint8ListReader(); + /// Enables lazy reading of the list + /// + /// If true, the returned unmodifiable list lazily reads bytes on access. + /// Therefore, the underlying buffer must not change while accessing the list. + /// + /// If false, reads the whole list immediately as an Uint8List. + final bool lazy; + + const Uint8ListReader({this.lazy = true}); @override @pragma('vm:prefer-inline') @@ -1182,8 +1189,18 @@ class Uint8ListReader extends Reader> { @override @pragma('vm:prefer-inline') - List read(BufferContext bc, int offset) => - _FbUint8List(bc, bc.derefObject(offset)); + List read(BufferContext bc, int offset) { + final listOffset = bc.derefObject(offset); + if (lazy) return _FbUint8List(bc, listOffset); + + final length = bc._getUint32(listOffset); + final result = Uint8List(length); + var pos = listOffset + _sizeofUint32; + for (var i = 0; i < length; i++, pos++) { + result[i] = bc._getUint8(pos); + } + return result; + } } /// The reader of unsigned 8-bit integers. @@ -1199,6 +1216,38 @@ class Uint8Reader extends Reader { int read(BufferContext bc, int offset) => bc._getUint8(offset); } +/// Reader of unmodifiable binary data (a list of signed 8-bit integers). +class Int8ListReader extends Reader> { + /// Enables lazy reading of the list + /// + /// If true, the returned unmodifiable list lazily reads bytes on access. + /// Therefore, the underlying buffer must not change while accessing the list. + /// + /// If false, reads the whole list immediately as an Uint8List. + final bool lazy; + + const Int8ListReader({this.lazy = true}); + + @override + @pragma('vm:prefer-inline') + int get size => _sizeofUint32; + + @override + @pragma('vm:prefer-inline') + List read(BufferContext bc, int offset) { + final listOffset = bc.derefObject(offset); + if (lazy) return _FbUint8List(bc, listOffset); + + final length = bc._getUint32(listOffset); + final result = Int8List(length); + var pos = listOffset + _sizeofUint32; + for (var i = 0; i < length; i++, pos++) { + result[i] = bc._getInt8(pos); + } + return result; + } +} + /// The list backed by 64-bit values - Uint64 length and Float64. class _FbFloat64List extends _FbList { _FbFloat64List(BufferContext bc, int offset) : super(bc, offset); @@ -1286,6 +1335,15 @@ class _FbUint8List extends _FbList { int operator [](int i) => bc._getUint8(offset + 4 + i); } +/// List backed by 8-bit signed integers. +class _FbInt8List extends _FbList { + _FbInt8List(BufferContext bc, int offset) : super(bc, offset); + + @override + @pragma('vm:prefer-inline') + int operator [](int i) => bc._getInt8(offset + 4 + i); +} + /// List backed by 8-bit unsigned integers. class _FbBoolList extends _FbList { _FbBoolList(BufferContext bc, int offset) : super(bc, offset); diff --git a/dart/lib/src/reference.dart b/dart/lib/src/reference.dart index 89752fa02..e52d0b709 100644 --- a/dart/lib/src/reference.dart +++ b/dart/lib/src/reference.dart @@ -373,8 +373,7 @@ class Reference { return null; } - int _diffKeys( - List input, int index, int indirectOffset, int byteWidth) { + int _diffKeys(List input, int index, int indirectOffset, int byteWidth) { final keyOffset = indirectOffset + index * byteWidth; final keyIndirectOffset = keyOffset - _readUInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth)); diff --git a/dart/test/flat_buffers_test.dart b/dart/test/flat_buffers_test.dart index 55a3b6c12..b670b452f 100644 --- a/dart/test/flat_buffers_test.dart +++ b/dart/test/flat_buffers_test.dart @@ -658,15 +658,25 @@ class BuilderTest { List byteList; { Builder builder = Builder(initialSize: 0); - int offset = builder.writeListUint8([1, 2, 3, 4, 0x9A]); + int offset = builder.writeListUint8([1, 2, 3, 4, 0x9A, 0xFA]); builder.finish(offset); byteList = builder.buffer; } // read and verify BufferContext buf = BufferContext.fromBytes(byteList); - List items = const Uint8ListReader().read(buf, 0); - expect(items, hasLength(5)); - expect(items, orderedEquals([1, 2, 3, 4, 0x9A])); + const buffOffset = 8; // 32-bit offset to the list, + 32-bit length + for (final lazy in [true, false]) { + List items = Uint8ListReader(lazy: lazy).read(buf, 0); + expect(items, hasLength(6)); + expect(items, orderedEquals([1, 2, 3, 4, 0x9A, 0xFA])); + + // overwrite the buffer to verify the laziness + buf.buffer.setUint8(buffOffset + 1, 99); + expect(items, orderedEquals([1, lazy ? 99 : 2, 3, 4, 0x9A, 0xFA])); + + // restore the previous value for the next loop + buf.buffer.setUint8(buffOffset + 1, 2); + } } void test_reset() { diff --git a/dart/test/monster_test_my_game.example_generated.dart b/dart/test/monster_test_my_game.example_generated.dart index 69c918a29..869f1e6d8 100644 --- a/dart/test/monster_test_my_game.example_generated.dart +++ b/dart/test/monster_test_my_game.example_generated.dart @@ -1039,7 +1039,7 @@ class Monster { int get mana => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 6, 150); int get hp => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 8, 100); String? get name => const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 10); - List? get inventory => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 14); + List? get inventory => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 14); Color get color => Color.fromValue(const fb.Uint8Reader().vTableGet(_bc, _bcOffset, 16, 8)); AnyTypeId? get testType => AnyTypeId._createOrNull(const fb.Uint8Reader().vTableGetNullable(_bc, _bcOffset, 18)); dynamic get test { @@ -1056,7 +1056,7 @@ class Monster { /// multiline too List? get testarrayoftables => const fb.ListReader(Monster.reader).vTableGetNullable(_bc, _bcOffset, 26); Monster? get enemy => Monster.reader.vTableGetNullable(_bc, _bcOffset, 28); - List? get testnestedflatbuffer => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 30); + List? get testnestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 30); Stat? get testempty => Stat.reader.vTableGetNullable(_bc, _bcOffset, 32); bool get testbool => const fb.BoolReader().vTableGet(_bc, _bcOffset, 34, false); int get testhashs32Fnv1 => const fb.Int32Reader().vTableGet(_bc, _bcOffset, 36, 0); @@ -1073,7 +1073,7 @@ class Monster { double get testf3 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 58, 0.0); List? get testarrayofstring2 => const fb.ListReader(fb.StringReader()).vTableGetNullable(_bc, _bcOffset, 60); List? get testarrayofsortedstruct => const fb.ListReader(Ability.reader).vTableGetNullable(_bc, _bcOffset, 62); - List? get flex => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 64); + List? get flex => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 64); List? get test5 => const fb.ListReader(Test.reader).vTableGetNullable(_bc, _bcOffset, 66); List? get vectorOfLongs => const fb.ListReader(fb.Int64Reader()).vTableGetNullable(_bc, _bcOffset, 68); List? get vectorOfDoubles => const fb.ListReader(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 70); @@ -1106,7 +1106,7 @@ class Monster { } List? get vectorOfEnums => const fb.ListReader(Color.reader).vTableGetNullable(_bc, _bcOffset, 98); Race get signedEnum => Race.fromValue(const fb.Int8Reader().vTableGet(_bc, _bcOffset, 100, -1)); - List? get testrequirednestedflatbuffer => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 102); + List? get testrequirednestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 102); List? get scalarKeySortedTables => const fb.ListReader(Stat.reader).vTableGetNullable(_bc, _bcOffset, 104); @override @@ -1119,7 +1119,7 @@ class Monster { mana: mana, hp: hp, name: name, - inventory: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 14), + inventory: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 14), color: color, testType: testType, test: test, @@ -1127,7 +1127,7 @@ class Monster { testarrayofstring: const fb.ListReader(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24), testarrayoftables: testarrayoftables?.map((e) => e.unpack()).toList(), enemy: enemy?.unpack(), - testnestedflatbuffer: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 30), + testnestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 30), testempty: testempty?.unpack(), testbool: testbool, testhashs32Fnv1: testhashs32Fnv1, @@ -1144,7 +1144,7 @@ class Monster { testf3: testf3, testarrayofstring2: const fb.ListReader(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 60), testarrayofsortedstruct: testarrayofsortedstruct?.map((e) => e.unpack()).toList(), - flex: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 64), + flex: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 64), test5: test5?.map((e) => e.unpack()).toList(), vectorOfLongs: const fb.ListReader(fb.Int64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 68), vectorOfDoubles: const fb.ListReader(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 70), @@ -1163,7 +1163,7 @@ class Monster { anyAmbiguous: anyAmbiguous, vectorOfEnums: const fb.ListReader(Color.reader, lazy: false).vTableGetNullable(_bc, _bcOffset, 98), signedEnum: signedEnum, - testrequirednestedflatbuffer: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 102), + testrequirednestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 102), scalarKeySortedTables: scalarKeySortedTables?.map((e) => e.unpack()).toList()); static int pack(fb.Builder fbBuilder, MonsterT? object) { @@ -1914,7 +1914,7 @@ class TypeAliases { int get u64 => const fb.Uint64Reader().vTableGet(_bc, _bcOffset, 18, 0); double get f32 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 20, 0.0); double get f64 => const fb.Float64Reader().vTableGet(_bc, _bcOffset, 22, 0.0); - List? get v8 => const fb.ListReader(fb.Int8Reader()).vTableGetNullable(_bc, _bcOffset, 24); + List? get v8 => const fb.Int8ListReader().vTableGetNullable(_bc, _bcOffset, 24); List? get vf64 => const fb.ListReader(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 26); @override @@ -1933,7 +1933,7 @@ class TypeAliases { u64: u64, f32: f32, f64: f64, - v8: const fb.ListReader(fb.Int8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24), + v8: const fb.Int8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 24), vf64: const fb.ListReader(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 26)); static int pack(fb.Builder fbBuilder, TypeAliasesT? object) { diff --git a/src/idl_gen_dart.cpp b/src/idl_gen_dart.cpp index 7e57dabb6..5fde2d859 100644 --- a/src/idl_gen_dart.cpp +++ b/src/idl_gen_dart.cpp @@ -335,6 +335,14 @@ class DartGenerator : public BaseGenerator { if (type.base_type == BASE_TYPE_BOOL) { return prefix + ".BoolReader()"; } else if (IsVector(type)) { + if (!type.VectorType().enum_def) { + if (type.VectorType().base_type == BASE_TYPE_CHAR) { + return prefix + ".Int8ListReader(" + (lazy ? ")" : "lazy: false)"); + } + if (type.VectorType().base_type == BASE_TYPE_UCHAR) { + return prefix + ".Uint8ListReader(" + (lazy ? ")" : "lazy: false)"); + } + } return prefix + ".ListReader<" + GenDartTypeName(type.VectorType(), current_namespace, def) + ">(" + GenReaderTypeName(type.VectorType(), current_namespace, def, true, diff --git a/tests/monster_test_my_game.example_generated.dart b/tests/monster_test_my_game.example_generated.dart index 69c918a29..869f1e6d8 100644 --- a/tests/monster_test_my_game.example_generated.dart +++ b/tests/monster_test_my_game.example_generated.dart @@ -1039,7 +1039,7 @@ class Monster { int get mana => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 6, 150); int get hp => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 8, 100); String? get name => const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 10); - List? get inventory => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 14); + List? get inventory => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 14); Color get color => Color.fromValue(const fb.Uint8Reader().vTableGet(_bc, _bcOffset, 16, 8)); AnyTypeId? get testType => AnyTypeId._createOrNull(const fb.Uint8Reader().vTableGetNullable(_bc, _bcOffset, 18)); dynamic get test { @@ -1056,7 +1056,7 @@ class Monster { /// multiline too List? get testarrayoftables => const fb.ListReader(Monster.reader).vTableGetNullable(_bc, _bcOffset, 26); Monster? get enemy => Monster.reader.vTableGetNullable(_bc, _bcOffset, 28); - List? get testnestedflatbuffer => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 30); + List? get testnestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 30); Stat? get testempty => Stat.reader.vTableGetNullable(_bc, _bcOffset, 32); bool get testbool => const fb.BoolReader().vTableGet(_bc, _bcOffset, 34, false); int get testhashs32Fnv1 => const fb.Int32Reader().vTableGet(_bc, _bcOffset, 36, 0); @@ -1073,7 +1073,7 @@ class Monster { double get testf3 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 58, 0.0); List? get testarrayofstring2 => const fb.ListReader(fb.StringReader()).vTableGetNullable(_bc, _bcOffset, 60); List? get testarrayofsortedstruct => const fb.ListReader(Ability.reader).vTableGetNullable(_bc, _bcOffset, 62); - List? get flex => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 64); + List? get flex => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 64); List? get test5 => const fb.ListReader(Test.reader).vTableGetNullable(_bc, _bcOffset, 66); List? get vectorOfLongs => const fb.ListReader(fb.Int64Reader()).vTableGetNullable(_bc, _bcOffset, 68); List? get vectorOfDoubles => const fb.ListReader(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 70); @@ -1106,7 +1106,7 @@ class Monster { } List? get vectorOfEnums => const fb.ListReader(Color.reader).vTableGetNullable(_bc, _bcOffset, 98); Race get signedEnum => Race.fromValue(const fb.Int8Reader().vTableGet(_bc, _bcOffset, 100, -1)); - List? get testrequirednestedflatbuffer => const fb.ListReader(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 102); + List? get testrequirednestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 102); List? get scalarKeySortedTables => const fb.ListReader(Stat.reader).vTableGetNullable(_bc, _bcOffset, 104); @override @@ -1119,7 +1119,7 @@ class Monster { mana: mana, hp: hp, name: name, - inventory: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 14), + inventory: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 14), color: color, testType: testType, test: test, @@ -1127,7 +1127,7 @@ class Monster { testarrayofstring: const fb.ListReader(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24), testarrayoftables: testarrayoftables?.map((e) => e.unpack()).toList(), enemy: enemy?.unpack(), - testnestedflatbuffer: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 30), + testnestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 30), testempty: testempty?.unpack(), testbool: testbool, testhashs32Fnv1: testhashs32Fnv1, @@ -1144,7 +1144,7 @@ class Monster { testf3: testf3, testarrayofstring2: const fb.ListReader(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 60), testarrayofsortedstruct: testarrayofsortedstruct?.map((e) => e.unpack()).toList(), - flex: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 64), + flex: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 64), test5: test5?.map((e) => e.unpack()).toList(), vectorOfLongs: const fb.ListReader(fb.Int64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 68), vectorOfDoubles: const fb.ListReader(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 70), @@ -1163,7 +1163,7 @@ class Monster { anyAmbiguous: anyAmbiguous, vectorOfEnums: const fb.ListReader(Color.reader, lazy: false).vTableGetNullable(_bc, _bcOffset, 98), signedEnum: signedEnum, - testrequirednestedflatbuffer: const fb.ListReader(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 102), + testrequirednestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 102), scalarKeySortedTables: scalarKeySortedTables?.map((e) => e.unpack()).toList()); static int pack(fb.Builder fbBuilder, MonsterT? object) { @@ -1914,7 +1914,7 @@ class TypeAliases { int get u64 => const fb.Uint64Reader().vTableGet(_bc, _bcOffset, 18, 0); double get f32 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 20, 0.0); double get f64 => const fb.Float64Reader().vTableGet(_bc, _bcOffset, 22, 0.0); - List? get v8 => const fb.ListReader(fb.Int8Reader()).vTableGetNullable(_bc, _bcOffset, 24); + List? get v8 => const fb.Int8ListReader().vTableGetNullable(_bc, _bcOffset, 24); List? get vf64 => const fb.ListReader(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 26); @override @@ -1933,7 +1933,7 @@ class TypeAliases { u64: u64, f32: f32, f64: f64, - v8: const fb.ListReader(fb.Int8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24), + v8: const fb.Int8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 24), vf64: const fb.ListReader(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 26)); static int pack(fb.Builder fbBuilder, TypeAliasesT? object) {