You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "abandy (via GitHub)" <gi...@apache.org> on 2023/06/10 18:21:25 UTC
[GitHub] [arrow] abandy commented on a diff in pull request #35985: GH-35969: [Swift] use ArrowType instead of ArrowType.info and add binary, time32 and time64 types
abandy commented on code in PR #35985:
URL: https://github.com/apache/arrow/pull/35985#discussion_r1225504522
##########
swift/Arrow/Tests/ArrowTests/IPCTests.swift:
##########
@@ -209,4 +212,163 @@ final class IPCFileReaderTests: XCTestCase {
throw error
}
}
+
+ func testSchemaInMemoryToFromStream() throws {
+ //read existing file
+ let schema = makeSchema()
+ let arrowWriter = ArrowWriter()
+ let writerInfo = ArrowWriter.Info(.schema, schema: schema)
+ switch arrowWriter.toStream(writerInfo) {
+ case .success(let writeData):
+ let arrowReader = ArrowReader()
+ switch arrowReader.fromStream(writeData) {
+ case .success(let result):
+ XCTAssertNotNil(result.schema)
+ let schema = result.schema!
+ XCTAssertEqual(schema.fields.count, 3)
+ XCTAssertEqual(schema.fields[0].name, "col1")
+ XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8)
+ XCTAssertEqual(schema.fields[1].name, "col2")
+ XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
+ XCTAssertEqual(schema.fields[2].name, "col3")
+ XCTAssertEqual(schema.fields[2].type.info, ArrowType.ArrowDate32)
+ case.failure(let error):
+ throw error
+ }
+ case .failure(let error):
+ throw error
+ }
+ }
+
+ func makeBinaryDataset() throws -> (ArrowSchema, RecordBatch) {
+ let schemaBuilder = ArrowSchema.Builder();
+ let schema = schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true)
+ .addField("col2", type: ArrowType(ArrowType.ArrowBinary), isNullable: false)
+ .finish()
+
+ let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder();
+ uint8Builder.append(10)
+ uint8Builder.append(22)
+ uint8Builder.append(33)
+ uint8Builder.append(44)
+ let binaryBuilder = try ArrowArrayBuilders.loadBinaryArrayBuilder();
+ binaryBuilder.append("test10".data(using: .utf8))
+ binaryBuilder.append("test22".data(using: .utf8))
+ binaryBuilder.append("test33".data(using: .utf8))
+ binaryBuilder.append("test44".data(using: .utf8))
+
+ let intHolder = ArrowArrayHolder(try uint8Builder.finish())
+ let binaryHolder = ArrowArrayHolder(try binaryBuilder.finish())
+ let result = RecordBatch.Builder()
+ .addColumn("col1", arrowArray: intHolder)
+ .addColumn("col2", arrowArray: binaryHolder)
+ .finish()
+ switch result {
+ case .success(let recordBatch):
+ return (schema, recordBatch)
+ case .failure(let error):
+ throw error
+ }
+ }
+
+ func makeTimeDataset() throws -> (ArrowSchema, RecordBatch) {
+ let schemaBuilder = ArrowSchema.Builder();
+ let schema = schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true)
+ .addField("col2", type: ArrowTypeTime64(.Microseconds), isNullable: false)
+ .addField("col3", type: ArrowTypeTime32(.Milliseconds), isNullable: false)
+ .finish()
+
+ let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder();
+ uint8Builder.append(10)
+ uint8Builder.append(22)
+ uint8Builder.append(33)
+ uint8Builder.append(44)
+ let time64Builder = try ArrowArrayBuilders.loadTime64ArrayBuilder(.Nanoseconds);
+ time64Builder.append(12345678)
+ time64Builder.append(1)
+ time64Builder.append(nil)
+ time64Builder.append(98765432)
+ let time32Builder = try ArrowArrayBuilders.loadTime32ArrayBuilder(.Milliseconds);
+ time32Builder.append(1)
+ time32Builder.append(2)
+ time32Builder.append(nil)
+ time32Builder.append(3)
+
+ let intHolder = ArrowArrayHolder(try uint8Builder.finish())
+ let time64Holder = ArrowArrayHolder(try time64Builder.finish())
+ let time32Holder = ArrowArrayHolder(try time32Builder.finish())
+ let result = RecordBatch.Builder()
+ .addColumn("col1", arrowArray: intHolder)
+ .addColumn("col2", arrowArray: time64Holder)
+ .addColumn("col3", arrowArray: time32Holder)
Review Comment:
Good idea, I will update.
##########
swift/Arrow/Tests/ArrowTests/IPCTests.swift:
##########
@@ -209,4 +212,163 @@ final class IPCFileReaderTests: XCTestCase {
throw error
}
}
+
+ func testSchemaInMemoryToFromStream() throws {
+ //read existing file
+ let schema = makeSchema()
+ let arrowWriter = ArrowWriter()
+ let writerInfo = ArrowWriter.Info(.schema, schema: schema)
+ switch arrowWriter.toStream(writerInfo) {
+ case .success(let writeData):
+ let arrowReader = ArrowReader()
+ switch arrowReader.fromStream(writeData) {
+ case .success(let result):
+ XCTAssertNotNil(result.schema)
+ let schema = result.schema!
+ XCTAssertEqual(schema.fields.count, 3)
+ XCTAssertEqual(schema.fields[0].name, "col1")
+ XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8)
+ XCTAssertEqual(schema.fields[1].name, "col2")
+ XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
+ XCTAssertEqual(schema.fields[2].name, "col3")
+ XCTAssertEqual(schema.fields[2].type.info, ArrowType.ArrowDate32)
+ case.failure(let error):
+ throw error
+ }
+ case .failure(let error):
+ throw error
+ }
+ }
+
+ func makeBinaryDataset() throws -> (ArrowSchema, RecordBatch) {
+ let schemaBuilder = ArrowSchema.Builder();
+ let schema = schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true)
+ .addField("col2", type: ArrowType(ArrowType.ArrowBinary), isNullable: false)
+ .finish()
+
+ let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder();
+ uint8Builder.append(10)
+ uint8Builder.append(22)
+ uint8Builder.append(33)
+ uint8Builder.append(44)
+ let binaryBuilder = try ArrowArrayBuilders.loadBinaryArrayBuilder();
+ binaryBuilder.append("test10".data(using: .utf8))
+ binaryBuilder.append("test22".data(using: .utf8))
+ binaryBuilder.append("test33".data(using: .utf8))
+ binaryBuilder.append("test44".data(using: .utf8))
+
+ let intHolder = ArrowArrayHolder(try uint8Builder.finish())
+ let binaryHolder = ArrowArrayHolder(try binaryBuilder.finish())
+ let result = RecordBatch.Builder()
+ .addColumn("col1", arrowArray: intHolder)
+ .addColumn("col2", arrowArray: binaryHolder)
+ .finish()
+ switch result {
+ case .success(let recordBatch):
+ return (schema, recordBatch)
+ case .failure(let error):
+ throw error
+ }
+ }
+
+ func makeTimeDataset() throws -> (ArrowSchema, RecordBatch) {
+ let schemaBuilder = ArrowSchema.Builder();
+ let schema = schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true)
+ .addField("col2", type: ArrowTypeTime64(.Microseconds), isNullable: false)
+ .addField("col3", type: ArrowTypeTime32(.Milliseconds), isNullable: false)
+ .finish()
+
+ let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder();
+ uint8Builder.append(10)
+ uint8Builder.append(22)
+ uint8Builder.append(33)
+ uint8Builder.append(44)
+ let time64Builder = try ArrowArrayBuilders.loadTime64ArrayBuilder(.Nanoseconds);
+ time64Builder.append(12345678)
+ time64Builder.append(1)
+ time64Builder.append(nil)
+ time64Builder.append(98765432)
+ let time32Builder = try ArrowArrayBuilders.loadTime32ArrayBuilder(.Milliseconds);
+ time32Builder.append(1)
+ time32Builder.append(2)
+ time32Builder.append(nil)
+ time32Builder.append(3)
+
+ let intHolder = ArrowArrayHolder(try uint8Builder.finish())
Review Comment:
Will change.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org