From 8d5e424c6549b904504f17ddd14b76bb3cdbe8d0 Mon Sep 17 00:00:00 2001 From: Derek Bailey Date: Mon, 28 Oct 2019 09:30:31 -0700 Subject: [PATCH] Add ByteBuffer copy for vector of bytes in Java (#5587) --- .../google/flatbuffers/FlatBufferBuilder.java | 32 ++++ src/idl_gen_general.cpp | 74 +++++---- tests/JavaTest.java | 145 ++++++++++++++++-- tests/MyGame/Example/Monster.cs | 1 + tests/MyGame/Example/Monster.java | 12 +- tests/MyGame/Example/TypeAliases.java | 3 +- tests/union_vector/Movie.cs | 1 + 7 files changed, 221 insertions(+), 47 deletions(-) diff --git a/java/com/google/flatbuffers/FlatBufferBuilder.java b/java/com/google/flatbuffers/FlatBufferBuilder.java index 7341b015d..e5e3967ac 100644 --- a/java/com/google/flatbuffers/FlatBufferBuilder.java +++ b/java/com/google/flatbuffers/FlatBufferBuilder.java @@ -572,6 +572,38 @@ public class FlatBufferBuilder { return endVector(); } + /** + * Create a byte array in the buffer. + * + * @param arr a source array with data. + * @param offset the offset in the source array to start copying from. + * @param length the number of bytes to copy from the source array. + * @return The offset in the buffer where the encoded array starts. + */ + public int createByteVector(byte[] arr, int offset, int length) { + startVector(1, length, 1); + bb.position(space -= length); + bb.put(arr, offset, length); + return endVector(); + } + + /** + * Create a byte array in the buffer. + * + * The source {@link ByteBuffer} position is advanced by {@link ByteBuffer#remaining()} places + * after this call. + * + * @param byteBuffer A source {@link ByteBuffer} with data. + * @return The offset in the buffer where the encoded array starts. + */ + public int createByteVector(ByteBuffer byteBuffer) { + int length = byteBuffer.remaining(); + startVector(1, length, 1); + bb.position(space -= length); + bb.put(byteBuffer); + return endVector(); + } + /// @cond FLATBUFFERS_INTERNAL /** * Should not be accessing the final buffer before it is finished. diff --git a/src/idl_gen_general.cpp b/src/idl_gen_general.cpp index 30564bf84..03ffb8810 100644 --- a/src/idl_gen_general.cpp +++ b/src/idl_gen_general.cpp @@ -1513,44 +1513,58 @@ class GeneralGenerator : public BaseGenerator { auto alignment = InlineAlignment(vector_type); auto elem_size = InlineSize(vector_type); if (!IsStruct(vector_type)) { - // Generate a method to create a vector from a Java array. - code += " public static " + GenVectorOffsetType() + " "; - code += FunctionStart('C') + "reate"; - code += MakeCamel(field.name); - code += "Vector(FlatBufferBuilder builder, "; - code += GenTypeBasic(vector_type) + "[] data) "; - code += "{ builder." + FunctionStart('S') + "tartVector("; - code += NumToString(elem_size); - code += ", data." + FunctionStart('L') + "ength, "; - code += NumToString(alignment); - code += "); for (int i = data."; - code += FunctionStart('L') + "ength - 1; i >= 0; i--) builder."; - code += FunctionStart('A') + "dd"; - code += GenMethod(vector_type); - code += "("; - code += SourceCastBasic(vector_type, false); - code += "data[i]"; - if (lang_.language == IDLOptions::kCSharp && - (vector_type.base_type == BASE_TYPE_STRUCT || - vector_type.base_type == BASE_TYPE_STRING)) - code += ".Value"; - code += "); return "; - code += "builder." + FunctionStart('E') + "ndVector(); }\n"; - // For C#, include a block copy method signature. - // Skip if the vector is of enums, because builder.Add - // throws an exception when supplied an enum array. - if (lang_.language == IDLOptions::kCSharp && - !IsEnum(vector_type)) { + // generate a method to create a vector from a java array. + if (lang_.language == IDLOptions::kJava && + (vector_type.base_type == BASE_TYPE_CHAR || + vector_type.base_type == BASE_TYPE_UCHAR)) { + // Handle byte[] and ByteBuffers separately for Java code += " public static " + GenVectorOffsetType() + " "; code += FunctionStart('C') + "reate"; code += MakeCamel(field.name); - code += "VectorBlock(FlatBufferBuilder builder, "; + code += "Vector(FlatBufferBuilder builder, byte[] data) "; + code += "{ return builder.createByteVector(data); }\n"; + + code += " public static " + GenVectorOffsetType() + " "; + code += FunctionStart('C') + "reate"; + code += MakeCamel(field.name); + code += "Vector(FlatBufferBuilder builder, ByteBuffer data) "; + code += "{ return builder.createByteVector(data); }\n"; + } else { + code += " public static " + GenVectorOffsetType() + " "; + code += FunctionStart('C') + "reate"; + code += MakeCamel(field.name); + code += "Vector(FlatBufferBuilder builder, "; code += GenTypeBasic(vector_type) + "[] data) "; code += "{ builder." + FunctionStart('S') + "tartVector("; code += NumToString(elem_size); code += ", data." + FunctionStart('L') + "ength, "; code += NumToString(alignment); - code += "); builder.Add(data); return builder.EndVector(); }\n"; + code += "); for (int i = data."; + code += FunctionStart('L') + "ength - 1; i >= 0; i--) builder."; + code += FunctionStart('A') + "dd"; + code += GenMethod(vector_type); + code += "("; + code += SourceCastBasic(vector_type, false); + code += "data[i]"; + if (lang_.language == IDLOptions::kCSharp && + (vector_type.base_type == BASE_TYPE_STRUCT || + vector_type.base_type == BASE_TYPE_STRING)) + code += ".Value"; + code += "); return "; + code += "builder." + FunctionStart('E') + "ndVector(); }\n"; + // For C#, include a block copy method signature. + if (lang_.language == IDLOptions::kCSharp) { + code += " public static " + GenVectorOffsetType() + " "; + code += FunctionStart('C') + "reate"; + code += MakeCamel(field.name); + code += "VectorBlock(FlatBufferBuilder builder, "; + code += GenTypeBasic(vector_type) + "[] data) "; + code += "{ builder." + FunctionStart('S') + "tartVector("; + code += NumToString(elem_size); + code += ", data." + FunctionStart('L') + "ength, "; + code += NumToString(alignment); + code += "); builder.Add(data); return builder.EndVector(); }\n"; + } } } // Generate a method to start a vector, data to be added manually diff --git a/tests/JavaTest.java b/tests/JavaTest.java index 79fee1ead..9c11910d5 100644 --- a/tests/JavaTest.java +++ b/tests/JavaTest.java @@ -14,26 +14,27 @@ * limitations under the License. */ -import java.util.Arrays; -import java.math.BigInteger; -import java.io.*; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; -import java.util.Map; -import java.util.HashMap; +import static com.google.flatbuffers.Constants.*; + import MyGame.Example.*; +import MyGame.MonsterExtra; import NamespaceA.*; import NamespaceA.NamespaceB.*; import com.google.flatbuffers.ByteBufferUtil; -import static com.google.flatbuffers.Constants.*; -import com.google.flatbuffers.FlatBufferBuilder; import com.google.flatbuffers.ByteVector; -import com.google.flatbuffers.FlexBuffersBuilder; +import com.google.flatbuffers.FlatBufferBuilder; import com.google.flatbuffers.FlexBuffers; +import com.google.flatbuffers.FlexBuffersBuilder; import com.google.flatbuffers.StringVector; import com.google.flatbuffers.UnionVector; -import MyGame.MonsterExtra; +import java.io.*; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; class JavaTest { public static void main(String[] args) { @@ -88,6 +89,8 @@ class JavaTest { TestFlexBuffers(); + TestVectorOfBytes(); + System.out.println("FlatBuffers test: completed successfully"); } @@ -955,6 +958,124 @@ class JavaTest { testFlexBuferEmpty(); } + static void TestVectorOfBytes() { + FlatBufferBuilder fbb = new FlatBufferBuilder(16); + int str = fbb.createString("ByteMonster"); + byte[] data = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + int offset = Monster.createInventoryVector(fbb, data); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + int monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject.inventoryLength(), data.length); + TestEq(monsterObject.inventory(4), (int) data[4]); + TestEq(ByteBuffer.wrap(data), monsterObject.inventoryAsByteBuffer()); + + fbb.clear(); + ByteBuffer bb = ByteBuffer.wrap(data); + offset = fbb.createByteVector(bb); + str = fbb.createString("ByteMonster"); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject2 = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject2.inventoryLength(), data.length); + for (int i = 0; i < data.length; i++) { + TestEq(monsterObject2.inventory(i), (int) bb.get(i)); + } + + fbb.clear(); + offset = fbb.createByteVector(data, 3, 4); + str = fbb.createString("ByteMonster"); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject3 = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject3.inventoryLength(), 4); + TestEq(monsterObject3.inventory(0), (int) data[3]); + + fbb.clear(); + bb = ByteBuffer.wrap(data); + offset = Monster.createInventoryVector(fbb, bb); + str = fbb.createString("ByteMonster"); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject4 = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject4.inventoryLength(), data.length); + TestEq(monsterObject4.inventory(8), (int) 8); + + fbb.clear(); + byte[] largeData = new byte[1024]; + offset = fbb.createByteVector(largeData); + str = fbb.createString("ByteMonster"); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject5 = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject5.inventoryLength(), largeData.length); + TestEq(monsterObject5.inventory(25), (int) largeData[25]); + + fbb.clear(); + bb = ByteBuffer.wrap(largeData); + bb.position(512); + ByteBuffer bb2 = bb.slice(); + TestEq(bb2.arrayOffset(), 512); + offset = fbb.createByteVector(bb2); + str = fbb.createString("ByteMonster"); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject6 = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject6.inventoryLength(), 512); + TestEq(monsterObject6.inventory(0), (int) largeData[512]); + + fbb.clear(); + bb = ByteBuffer.wrap(largeData); + bb.limit(256); + offset = fbb.createByteVector(bb); + str = fbb.createString("ByteMonster"); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject7 = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject7.inventoryLength(), 256); + + fbb.clear(); + bb = ByteBuffer.allocateDirect(2048); + offset = fbb.createByteVector(bb); + str = fbb.createString("ByteMonster"); + Monster.startMonster(fbb); + Monster.addName(fbb, str); + Monster.addInventory(fbb, offset); + monster1 = Monster.endMonster(fbb); + Monster.finishMonsterBuffer(fbb, monster1); + Monster monsterObject8 = Monster.getRootAsMonster(fbb.dataBuffer()); + + TestEq(monsterObject8.inventoryLength(), 2048); + } + static void TestEq(T a, T b) { if (!a.equals(b)) { System.out.println("" + a.getClass().getName() + " " + b.getClass().getName()); diff --git a/tests/MyGame/Example/Monster.cs b/tests/MyGame/Example/Monster.cs index c84d15f91..36d6c2dde 100644 --- a/tests/MyGame/Example/Monster.cs +++ b/tests/MyGame/Example/Monster.cs @@ -285,6 +285,7 @@ public struct Monster : IFlatbufferObject public static void AddAnyAmbiguous(FlatBufferBuilder builder, int anyAmbiguousOffset) { builder.AddOffset(46, anyAmbiguousOffset, 0); } public static void AddVectorOfEnums(FlatBufferBuilder builder, VectorOffset vectorOfEnumsOffset) { builder.AddOffset(47, vectorOfEnumsOffset.Value, 0); } public static VectorOffset CreateVectorOfEnumsVector(FlatBufferBuilder builder, MyGame.Example.Color[] data) { builder.StartVector(1, data.Length, 1); for (int i = data.Length - 1; i >= 0; i--) builder.AddByte((byte)data[i]); return builder.EndVector(); } + public static VectorOffset CreateVectorOfEnumsVectorBlock(FlatBufferBuilder builder, MyGame.Example.Color[] data) { builder.StartVector(1, data.Length, 1); builder.Add(data); return builder.EndVector(); } public static void StartVectorOfEnumsVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(1, numElems, 1); } public static void AddSignedEnum(FlatBufferBuilder builder, MyGame.Example.Race signedEnum) { builder.AddSbyte(48, (sbyte)signedEnum, -1); } public static Offset EndMonster(FlatBufferBuilder builder) { diff --git a/tests/MyGame/Example/Monster.java b/tests/MyGame/Example/Monster.java index 961108d34..07c4e41b2 100644 --- a/tests/MyGame/Example/Monster.java +++ b/tests/MyGame/Example/Monster.java @@ -204,7 +204,8 @@ public final class Monster extends Table { public static void addHp(FlatBufferBuilder builder, short hp) { builder.addShort(2, hp, 100); } public static void addName(FlatBufferBuilder builder, int nameOffset) { builder.addOffset(3, nameOffset, 0); } public static void addInventory(FlatBufferBuilder builder, int inventoryOffset) { builder.addOffset(5, inventoryOffset, 0); } - public static int createInventoryVector(FlatBufferBuilder builder, byte[] data) { builder.startVector(1, data.length, 1); for (int i = data.length - 1; i >= 0; i--) builder.addByte(data[i]); return builder.endVector(); } + public static int createInventoryVector(FlatBufferBuilder builder, byte[] data) { return builder.createByteVector(data); } + public static int createInventoryVector(FlatBufferBuilder builder, ByteBuffer data) { return builder.createByteVector(data); } public static void startInventoryVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); } public static void addColor(FlatBufferBuilder builder, int color) { builder.addByte(6, (byte)color, (byte)8); } public static void addTestType(FlatBufferBuilder builder, byte testType) { builder.addByte(7, testType, 0); } @@ -219,7 +220,8 @@ public final class Monster extends Table { public static void startTestarrayoftablesVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } public static void addEnemy(FlatBufferBuilder builder, int enemyOffset) { builder.addOffset(12, enemyOffset, 0); } public static void addTestnestedflatbuffer(FlatBufferBuilder builder, int testnestedflatbufferOffset) { builder.addOffset(13, testnestedflatbufferOffset, 0); } - public static int createTestnestedflatbufferVector(FlatBufferBuilder builder, byte[] data) { builder.startVector(1, data.length, 1); for (int i = data.length - 1; i >= 0; i--) builder.addByte(data[i]); return builder.endVector(); } + public static int createTestnestedflatbufferVector(FlatBufferBuilder builder, byte[] data) { return builder.createByteVector(data); } + public static int createTestnestedflatbufferVector(FlatBufferBuilder builder, ByteBuffer data) { return builder.createByteVector(data); } public static void startTestnestedflatbufferVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); } public static void addTestempty(FlatBufferBuilder builder, int testemptyOffset) { builder.addOffset(14, testemptyOffset, 0); } public static void addTestbool(FlatBufferBuilder builder, boolean testbool) { builder.addBoolean(15, testbool, false); } @@ -243,7 +245,8 @@ public final class Monster extends Table { public static void addTestarrayofsortedstruct(FlatBufferBuilder builder, int testarrayofsortedstructOffset) { builder.addOffset(29, testarrayofsortedstructOffset, 0); } public static void startTestarrayofsortedstructVector(FlatBufferBuilder builder, int numElems) { builder.startVector(8, numElems, 4); } public static void addFlex(FlatBufferBuilder builder, int flexOffset) { builder.addOffset(30, flexOffset, 0); } - public static int createFlexVector(FlatBufferBuilder builder, byte[] data) { builder.startVector(1, data.length, 1); for (int i = data.length - 1; i >= 0; i--) builder.addByte(data[i]); return builder.endVector(); } + public static int createFlexVector(FlatBufferBuilder builder, byte[] data) { return builder.createByteVector(data); } + public static int createFlexVector(FlatBufferBuilder builder, ByteBuffer data) { return builder.createByteVector(data); } public static void startFlexVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); } public static void addTest5(FlatBufferBuilder builder, int test5Offset) { builder.addOffset(31, test5Offset, 0); } public static void startTest5Vector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 2); } @@ -277,7 +280,8 @@ public final class Monster extends Table { public static void addAnyAmbiguousType(FlatBufferBuilder builder, byte anyAmbiguousType) { builder.addByte(45, anyAmbiguousType, 0); } public static void addAnyAmbiguous(FlatBufferBuilder builder, int anyAmbiguousOffset) { builder.addOffset(46, anyAmbiguousOffset, 0); } public static void addVectorOfEnums(FlatBufferBuilder builder, int vectorOfEnumsOffset) { builder.addOffset(47, vectorOfEnumsOffset, 0); } - public static int createVectorOfEnumsVector(FlatBufferBuilder builder, byte[] data) { builder.startVector(1, data.length, 1); for (int i = data.length - 1; i >= 0; i--) builder.addByte(data[i]); return builder.endVector(); } + public static int createVectorOfEnumsVector(FlatBufferBuilder builder, byte[] data) { return builder.createByteVector(data); } + public static int createVectorOfEnumsVector(FlatBufferBuilder builder, ByteBuffer data) { return builder.createByteVector(data); } public static void startVectorOfEnumsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); } public static void addSignedEnum(FlatBufferBuilder builder, byte signedEnum) { builder.addByte(48, signedEnum, -1); } public static int endMonster(FlatBufferBuilder builder) { diff --git a/tests/MyGame/Example/TypeAliases.java b/tests/MyGame/Example/TypeAliases.java index 0c7df009a..178a0f925 100644 --- a/tests/MyGame/Example/TypeAliases.java +++ b/tests/MyGame/Example/TypeAliases.java @@ -91,7 +91,8 @@ public final class TypeAliases extends Table { public static void addF32(FlatBufferBuilder builder, float f32) { builder.addFloat(8, f32, 0.0f); } public static void addF64(FlatBufferBuilder builder, double f64) { builder.addDouble(9, f64, 0.0); } public static void addV8(FlatBufferBuilder builder, int v8Offset) { builder.addOffset(10, v8Offset, 0); } - public static int createV8Vector(FlatBufferBuilder builder, byte[] data) { builder.startVector(1, data.length, 1); for (int i = data.length - 1; i >= 0; i--) builder.addByte(data[i]); return builder.endVector(); } + public static int createV8Vector(FlatBufferBuilder builder, byte[] data) { return builder.createByteVector(data); } + public static int createV8Vector(FlatBufferBuilder builder, ByteBuffer data) { return builder.createByteVector(data); } public static void startV8Vector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); } public static void addVf64(FlatBufferBuilder builder, int vf64Offset) { builder.addOffset(11, vf64Offset, 0); } public static int createVf64Vector(FlatBufferBuilder builder, double[] data) { builder.startVector(8, data.length, 8); for (int i = data.length - 1; i >= 0; i--) builder.addDouble(data[i]); return builder.endVector(); } diff --git a/tests/union_vector/Movie.cs b/tests/union_vector/Movie.cs index 4d67d336b..d300b44ba 100644 --- a/tests/union_vector/Movie.cs +++ b/tests/union_vector/Movie.cs @@ -49,6 +49,7 @@ public struct Movie : IFlatbufferObject public static void AddMainCharacter(FlatBufferBuilder builder, int mainCharacterOffset) { builder.AddOffset(1, mainCharacterOffset, 0); } public static void AddCharactersType(FlatBufferBuilder builder, VectorOffset charactersTypeOffset) { builder.AddOffset(2, charactersTypeOffset.Value, 0); } public static VectorOffset CreateCharactersTypeVector(FlatBufferBuilder builder, Character[] data) { builder.StartVector(1, data.Length, 1); for (int i = data.Length - 1; i >= 0; i--) builder.AddByte((byte)data[i]); return builder.EndVector(); } + public static VectorOffset CreateCharactersTypeVectorBlock(FlatBufferBuilder builder, Character[] data) { builder.StartVector(1, data.Length, 1); builder.Add(data); return builder.EndVector(); } public static void StartCharactersTypeVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(1, numElems, 1); } public static void AddCharacters(FlatBufferBuilder builder, VectorOffset charactersOffset) { builder.AddOffset(3, charactersOffset.Value, 0); } public static VectorOffset CreateCharactersVector(FlatBufferBuilder builder, int[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i]); return builder.EndVector(); }