[GitHub] [arrow-adbc] lidavidm commented on a diff in pull request #605: feat(java/driver/flight-sql): implement getObjects

["lidavidm (via GitHub)" <gi...@apache.org>]

lidavidm commented on code in PR #605:
URL: https://github.com/apache/arrow-adbc/pull/605#discussion_r1174923488


##########
java/driver/flight-sql/src/main/java/org/apache/arrow/adbc/driver/flightsql/ObjectMetadataBuilder.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.adbc.driver.flightsql;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.core.StandardSchemas;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.complex.writer.VarCharWriter;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+final class ObjectMetadataBuilder {
+
+  private final FlightSqlClient client;
+  private final VectorSchemaRoot root;
+  private final VarCharVector adbcCatalogNames;
+  private final UnionListWriter adbcCatalogDbSchemasWriter;
+  private final BaseWriter.StructWriter adbcCatalogDbSchemasStructWriter;
+  private final BaseWriter.ListWriter adbcCatalogDbSchemaTablesWriter;
+  private final VarCharWriter adbcCatalogDbSchemaNameWriter;
+  private final BaseWriter.StructWriter adbcTablesStructWriter;
+  private final VarCharWriter adbcTableNameWriter;
+  private final VarCharWriter adbcTableTypeWriter;
+  private final BaseWriter.ListWriter adbcTableColumnsWriter;
+  private final BufferAllocator allocator;
+  private final AdbcConnection.GetObjectsDepth depth;
+  private final String catalogPattern;
+  private final String dbSchemaPattern;
+  private final String tableNamePattern;
+  private final String[] tableTypes;
+  private final String columnNamePattern;
+
+  ObjectMetadataBuilder(
+      BufferAllocator allocator,
+      FlightSqlClient client,
+      final AdbcConnection.GetObjectsDepth depth,
+      final String catalogPattern,
+      final String dbSchemaPattern,
+      final String tableNamePattern,
+      final String[] tableTypes,
+      final String columnNamePattern) {
+    this.allocator = allocator;
+    this.client = client;
+    this.depth = depth;
+    this.catalogPattern = catalogPattern;
+    this.dbSchemaPattern = dbSchemaPattern;
+    this.tableNamePattern = tableNamePattern;
+    this.tableTypes = tableTypes;
+    this.columnNamePattern = columnNamePattern;
+    this.root = VectorSchemaRoot.create(StandardSchemas.GET_OBJECTS_SCHEMA, allocator);
+    this.adbcCatalogNames = (VarCharVector) root.getVector(0);
+    this.adbcCatalogDbSchemasWriter = ((ListVector) root.getVector(1)).getWriter();
+    this.adbcCatalogDbSchemasStructWriter = adbcCatalogDbSchemasWriter.struct();
+    this.adbcCatalogDbSchemaTablesWriter =
+        adbcCatalogDbSchemasStructWriter.list("db_schema_tables");
+    this.adbcCatalogDbSchemaNameWriter = adbcCatalogDbSchemasStructWriter.varChar("db_schema_name");
+    this.adbcTablesStructWriter = adbcCatalogDbSchemaTablesWriter.struct();
+    this.adbcTableNameWriter = adbcTablesStructWriter.varChar("table_name");
+    this.adbcTableTypeWriter = adbcTablesStructWriter.varChar("table_type");
+    this.adbcTableColumnsWriter = adbcTablesStructWriter.list("table_columns");
+  }
+
+  private void writeVarChar(VarCharWriter writer, String value) {
+    byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+    try (ArrowBuf tempBuf = allocator.buffer(bytes.length)) {
+      tempBuf.setBytes(0, bytes, 0, bytes.length);
+      writer.writeVarChar(0, bytes.length, tempBuf);
+    }
+  }
+
+  private boolean patternMatched(String name, String pattern) {
+    if (pattern == null) {
+      return true;
+    }
+
+    return name.matches(pattern.replace("_", ".").replace("%", ".*"));
+  }
+
+  VectorSchemaRoot build() throws AdbcException {
+    // TODO Catalogs and schemas that don't contain tables are being left out

Review Comment:
   Would you mind filing an issue and then updating this to be `TODO(apache/arrow-adbc#NNN): ...`?



##########
java/driver/validation/src/main/java/org/apache/arrow/adbc/driver/testsuite/AbstractTransactionTest.java:
##########
@@ -147,7 +147,7 @@ void commit() throws Exception {
   }
 
   @Test
-  void enableAutoCommitAlsoCommits() throws Exception {
+  public void enableAutoCommitAlsoCommits() throws Exception {

Review Comment:
   nit: are these necessary? JUnit 5 doesn't require public visibility anymore.



##########
java/driver/flight-sql/src/main/java/org/apache/arrow/adbc/driver/flightsql/ObjectMetadataBuilder.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.adbc.driver.flightsql;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.core.StandardSchemas;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.complex.writer.VarCharWriter;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+final class ObjectMetadataBuilder {
+
+  private final FlightSqlClient client;
+  private final VectorSchemaRoot root;
+  private final VarCharVector adbcCatalogNames;
+  private final UnionListWriter adbcCatalogDbSchemasWriter;
+  private final BaseWriter.StructWriter adbcCatalogDbSchemasStructWriter;
+  private final BaseWriter.ListWriter adbcCatalogDbSchemaTablesWriter;
+  private final VarCharWriter adbcCatalogDbSchemaNameWriter;
+  private final BaseWriter.StructWriter adbcTablesStructWriter;
+  private final VarCharWriter adbcTableNameWriter;
+  private final VarCharWriter adbcTableTypeWriter;
+  private final BaseWriter.ListWriter adbcTableColumnsWriter;
+  private final BufferAllocator allocator;
+  private final AdbcConnection.GetObjectsDepth depth;
+  private final String catalogPattern;
+  private final String dbSchemaPattern;
+  private final String tableNamePattern;
+  private final String[] tableTypes;
+  private final String columnNamePattern;
+
+  ObjectMetadataBuilder(
+      BufferAllocator allocator,
+      FlightSqlClient client,
+      final AdbcConnection.GetObjectsDepth depth,
+      final String catalogPattern,
+      final String dbSchemaPattern,
+      final String tableNamePattern,
+      final String[] tableTypes,
+      final String columnNamePattern) {
+    this.allocator = allocator;
+    this.client = client;
+    this.depth = depth;
+    this.catalogPattern = catalogPattern;
+    this.dbSchemaPattern = dbSchemaPattern;
+    this.tableNamePattern = tableNamePattern;
+    this.tableTypes = tableTypes;
+    this.columnNamePattern = columnNamePattern;
+    this.root = VectorSchemaRoot.create(StandardSchemas.GET_OBJECTS_SCHEMA, allocator);
+    this.adbcCatalogNames = (VarCharVector) root.getVector(0);
+    this.adbcCatalogDbSchemasWriter = ((ListVector) root.getVector(1)).getWriter();
+    this.adbcCatalogDbSchemasStructWriter = adbcCatalogDbSchemasWriter.struct();
+    this.adbcCatalogDbSchemaTablesWriter =
+        adbcCatalogDbSchemasStructWriter.list("db_schema_tables");
+    this.adbcCatalogDbSchemaNameWriter = adbcCatalogDbSchemasStructWriter.varChar("db_schema_name");
+    this.adbcTablesStructWriter = adbcCatalogDbSchemaTablesWriter.struct();
+    this.adbcTableNameWriter = adbcTablesStructWriter.varChar("table_name");
+    this.adbcTableTypeWriter = adbcTablesStructWriter.varChar("table_type");
+    this.adbcTableColumnsWriter = adbcTablesStructWriter.list("table_columns");
+  }
+
+  private void writeVarChar(VarCharWriter writer, String value) {
+    byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+    try (ArrowBuf tempBuf = allocator.buffer(bytes.length)) {
+      tempBuf.setBytes(0, bytes, 0, bytes.length);
+      writer.writeVarChar(0, bytes.length, tempBuf);
+    }
+  }
+
+  private boolean patternMatched(String name, String pattern) {
+    if (pattern == null) {
+      return true;
+    }
+
+    return name.matches(pattern.replace("_", ".").replace("%", ".*"));
+  }
+
+  VectorSchemaRoot build() throws AdbcException {
+    // TODO Catalogs and schemas that don't contain tables are being left out
+    FlightInfo info;
+    if (depth == AdbcConnection.GetObjectsDepth.CATALOGS) {
+      info = client.getCatalogs();
+    } else if (depth == AdbcConnection.GetObjectsDepth.DB_SCHEMAS) {
+      info = client.getSchemas(null, dbSchemaPattern);
+    } else {
+      info =
+          client.getTables(
+              null, // TODO pattern match later during processing
+              dbSchemaPattern,
+              tableNamePattern,
+              tableTypes == null ? null : Arrays.asList(tableTypes),
+              depth == AdbcConnection.GetObjectsDepth.ALL);
+    }
+
+    byte[] lastCatalogAdded = null;
+    byte[] lastDbSchemaAdded = null;
+    int catalogIndex = 0;
+
+    for (FlightEndpoint endpoint : info.getEndpoints()) {

Review Comment:
   What other drivers have done is factor out the reader logic from the statement so that it doesn't need to be replicated here. Though maybe it's simple enough that it isn't a big deal.



##########
java/driver/flight-sql/src/main/java/org/apache/arrow/adbc/driver/flightsql/ObjectMetadataBuilder.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.adbc.driver.flightsql;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.core.StandardSchemas;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.complex.writer.VarCharWriter;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+final class ObjectMetadataBuilder {
+
+  private final FlightSqlClient client;
+  private final VectorSchemaRoot root;
+  private final VarCharVector adbcCatalogNames;
+  private final UnionListWriter adbcCatalogDbSchemasWriter;
+  private final BaseWriter.StructWriter adbcCatalogDbSchemasStructWriter;
+  private final BaseWriter.ListWriter adbcCatalogDbSchemaTablesWriter;
+  private final VarCharWriter adbcCatalogDbSchemaNameWriter;
+  private final BaseWriter.StructWriter adbcTablesStructWriter;
+  private final VarCharWriter adbcTableNameWriter;
+  private final VarCharWriter adbcTableTypeWriter;
+  private final BaseWriter.ListWriter adbcTableColumnsWriter;
+  private final BufferAllocator allocator;
+  private final AdbcConnection.GetObjectsDepth depth;
+  private final String catalogPattern;
+  private final String dbSchemaPattern;
+  private final String tableNamePattern;
+  private final String[] tableTypes;
+  private final String columnNamePattern;
+
+  ObjectMetadataBuilder(
+      BufferAllocator allocator,
+      FlightSqlClient client,
+      final AdbcConnection.GetObjectsDepth depth,
+      final String catalogPattern,
+      final String dbSchemaPattern,
+      final String tableNamePattern,
+      final String[] tableTypes,
+      final String columnNamePattern) {
+    this.allocator = allocator;
+    this.client = client;
+    this.depth = depth;
+    this.catalogPattern = catalogPattern;
+    this.dbSchemaPattern = dbSchemaPattern;
+    this.tableNamePattern = tableNamePattern;
+    this.tableTypes = tableTypes;
+    this.columnNamePattern = columnNamePattern;
+    this.root = VectorSchemaRoot.create(StandardSchemas.GET_OBJECTS_SCHEMA, allocator);
+    this.adbcCatalogNames = (VarCharVector) root.getVector(0);
+    this.adbcCatalogDbSchemasWriter = ((ListVector) root.getVector(1)).getWriter();
+    this.adbcCatalogDbSchemasStructWriter = adbcCatalogDbSchemasWriter.struct();
+    this.adbcCatalogDbSchemaTablesWriter =
+        adbcCatalogDbSchemasStructWriter.list("db_schema_tables");
+    this.adbcCatalogDbSchemaNameWriter = adbcCatalogDbSchemasStructWriter.varChar("db_schema_name");
+    this.adbcTablesStructWriter = adbcCatalogDbSchemaTablesWriter.struct();
+    this.adbcTableNameWriter = adbcTablesStructWriter.varChar("table_name");
+    this.adbcTableTypeWriter = adbcTablesStructWriter.varChar("table_type");
+    this.adbcTableColumnsWriter = adbcTablesStructWriter.list("table_columns");
+  }
+
+  private void writeVarChar(VarCharWriter writer, String value) {
+    byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+    try (ArrowBuf tempBuf = allocator.buffer(bytes.length)) {
+      tempBuf.setBytes(0, bytes, 0, bytes.length);
+      writer.writeVarChar(0, bytes.length, tempBuf);
+    }
+  }
+
+  private boolean patternMatched(String name, String pattern) {
+    if (pattern == null) {
+      return true;
+    }
+
+    return name.matches(pattern.replace("_", ".").replace("%", ".*"));
+  }
+
+  VectorSchemaRoot build() throws AdbcException {
+    // TODO Catalogs and schemas that don't contain tables are being left out
+    FlightInfo info;
+    if (depth == AdbcConnection.GetObjectsDepth.CATALOGS) {
+      info = client.getCatalogs();
+    } else if (depth == AdbcConnection.GetObjectsDepth.DB_SCHEMAS) {
+      info = client.getSchemas(null, dbSchemaPattern);
+    } else {
+      info =
+          client.getTables(
+              null, // TODO pattern match later during processing
+              dbSchemaPattern,
+              tableNamePattern,
+              tableTypes == null ? null : Arrays.asList(tableTypes),
+              depth == AdbcConnection.GetObjectsDepth.ALL);
+    }
+
+    byte[] lastCatalogAdded = null;
+    byte[] lastDbSchemaAdded = null;
+    int catalogIndex = 0;
+
+    for (FlightEndpoint endpoint : info.getEndpoints()) {
+      FlightStream stream = client.getStream(endpoint.getTicket());
+      while (stream.next()) {
+        try (VectorSchemaRoot res = stream.getRoot()) {
+          VarCharVector catalogVector = (VarCharVector) res.getVector(0);
+
+          for (int i = 0; i < res.getRowCount(); i++) {
+            byte[] catalog = catalogVector.get(i);
+
+            if (i == 0 || lastCatalogAdded != catalog) {
+              if (catalog == null) {
+                adbcCatalogNames.setNull(catalogIndex);
+              } else {
+                adbcCatalogNames.setSafe(catalogIndex, catalog);
+              }
+              if (depth == AdbcConnection.GetObjectsDepth.CATALOGS) {
+                adbcCatalogDbSchemasWriter.writeNull();
+              } else {
+                if (catalogIndex != 0) {
+                  adbcCatalogDbSchemasWriter.endList();
+                }
+                adbcCatalogDbSchemasWriter.startList();
+                lastDbSchemaAdded = null;
+              }
+              catalogIndex++;
+              lastCatalogAdded = catalog;
+            }
+
+            if (depth != AdbcConnection.GetObjectsDepth.CATALOGS) {
+              VarCharVector dbSchemaVector = (VarCharVector) res.getVector(1);
+              byte[] dbSchema = dbSchemaVector.get(i);
+
+              if (!Arrays.equals(lastDbSchemaAdded, dbSchema)) {
+                if (i != 0) {
+                  adbcCatalogDbSchemaTablesWriter.endList();
+                  adbcCatalogDbSchemasStructWriter.end();
+                }
+                adbcCatalogDbSchemasStructWriter.start();
+                writeVarChar(
+                    adbcCatalogDbSchemaNameWriter, new String(dbSchema, StandardCharsets.UTF_8));
+                if (depth == AdbcConnection.GetObjectsDepth.DB_SCHEMAS) {
+                  adbcCatalogDbSchemaTablesWriter.writeNull();
+                } else {
+                  adbcCatalogDbSchemaTablesWriter.startList();
+                }
+
+                lastDbSchemaAdded = dbSchema;
+              }
+            }
+
+            if (depth != AdbcConnection.GetObjectsDepth.CATALOGS
+                && depth != AdbcConnection.GetObjectsDepth.DB_SCHEMAS) {
+              VarCharVector tableNameVector = (VarCharVector) res.getVector(2);
+              VarCharVector tableTypeVector = (VarCharVector) res.getVector(3);
+
+              adbcTablesStructWriter.start();
+              writeVarChar(
+                  adbcTableNameWriter, new String(tableNameVector.get(i), StandardCharsets.UTF_8));
+              writeVarChar(
+                  adbcTableTypeWriter, new String(tableTypeVector.get(i), StandardCharsets.UTF_8));
+
+              if (depth == AdbcConnection.GetObjectsDepth.ALL) {
+                VarBinaryVector tableSchemaVector = (VarBinaryVector) res.getVector(4);
+                Schema schema;
+
+                try {
+                  schema =
+                      MessageSerializer.deserializeSchema(
+                          new ReadChannel(
+                              Channels.newChannel(
+                                  new ByteArrayInputStream(tableSchemaVector.get(i)))));
+                } catch (IOException e) {
+                  throw new RuntimeException(e);
+                }
+
+                adbcTableColumnsWriter.startList();
+
+                for (int y = 0; y < schema.getFields().size(); y++) {
+                  Field field = schema.getFields().get(y);
+                  if (patternMatched(field.getName(), columnNamePattern)) {
+                    adbcTableColumnsWriter.struct().start();
+                    writeVarChar(
+                        adbcTableColumnsWriter.struct().varChar("column_name"), field.getName());
+                    adbcTableColumnsWriter.struct().integer("ordinal_position").writeInt(y + 1);
+                    adbcTableColumnsWriter.struct().end();
+                  }
+                }
+                adbcTableColumnsWriter.endList();
+              }
+
+              adbcTablesStructWriter.end();
+            }
+          }
+
+          if (depth != AdbcConnection.GetObjectsDepth.CATALOGS) {
+            adbcCatalogDbSchemaTablesWriter.endList();
+            adbcCatalogDbSchemasStructWriter.end();
+            adbcCatalogDbSchemasWriter.endList();
+          }
+        }
+      }
+    }
+
+    this.root.setRowCount(catalogIndex);
+    return root;

Review Comment:
   nit: inconsistency between 'this.root' and 'root'



##########
java/driver/flight-sql/src/main/java/org/apache/arrow/adbc/driver/flightsql/ObjectMetadataBuilder.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.adbc.driver.flightsql;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.core.StandardSchemas;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.complex.writer.VarCharWriter;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+final class ObjectMetadataBuilder {
+
+  private final FlightSqlClient client;
+  private final VectorSchemaRoot root;
+  private final VarCharVector adbcCatalogNames;
+  private final UnionListWriter adbcCatalogDbSchemasWriter;
+  private final BaseWriter.StructWriter adbcCatalogDbSchemasStructWriter;
+  private final BaseWriter.ListWriter adbcCatalogDbSchemaTablesWriter;
+  private final VarCharWriter adbcCatalogDbSchemaNameWriter;
+  private final BaseWriter.StructWriter adbcTablesStructWriter;
+  private final VarCharWriter adbcTableNameWriter;
+  private final VarCharWriter adbcTableTypeWriter;
+  private final BaseWriter.ListWriter adbcTableColumnsWriter;
+  private final BufferAllocator allocator;
+  private final AdbcConnection.GetObjectsDepth depth;
+  private final String catalogPattern;
+  private final String dbSchemaPattern;
+  private final String tableNamePattern;
+  private final String[] tableTypes;
+  private final String columnNamePattern;
+
+  ObjectMetadataBuilder(
+      BufferAllocator allocator,
+      FlightSqlClient client,
+      final AdbcConnection.GetObjectsDepth depth,
+      final String catalogPattern,
+      final String dbSchemaPattern,
+      final String tableNamePattern,
+      final String[] tableTypes,
+      final String columnNamePattern) {
+    this.allocator = allocator;
+    this.client = client;
+    this.depth = depth;
+    this.catalogPattern = catalogPattern;
+    this.dbSchemaPattern = dbSchemaPattern;
+    this.tableNamePattern = tableNamePattern;
+    this.tableTypes = tableTypes;
+    this.columnNamePattern = columnNamePattern;
+    this.root = VectorSchemaRoot.create(StandardSchemas.GET_OBJECTS_SCHEMA, allocator);
+    this.adbcCatalogNames = (VarCharVector) root.getVector(0);
+    this.adbcCatalogDbSchemasWriter = ((ListVector) root.getVector(1)).getWriter();
+    this.adbcCatalogDbSchemasStructWriter = adbcCatalogDbSchemasWriter.struct();
+    this.adbcCatalogDbSchemaTablesWriter =
+        adbcCatalogDbSchemasStructWriter.list("db_schema_tables");
+    this.adbcCatalogDbSchemaNameWriter = adbcCatalogDbSchemasStructWriter.varChar("db_schema_name");
+    this.adbcTablesStructWriter = adbcCatalogDbSchemaTablesWriter.struct();
+    this.adbcTableNameWriter = adbcTablesStructWriter.varChar("table_name");
+    this.adbcTableTypeWriter = adbcTablesStructWriter.varChar("table_type");
+    this.adbcTableColumnsWriter = adbcTablesStructWriter.list("table_columns");
+  }
+
+  private void writeVarChar(VarCharWriter writer, String value) {
+    byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+    try (ArrowBuf tempBuf = allocator.buffer(bytes.length)) {
+      tempBuf.setBytes(0, bytes, 0, bytes.length);
+      writer.writeVarChar(0, bytes.length, tempBuf);
+    }
+  }
+
+  private boolean patternMatched(String name, String pattern) {
+    if (pattern == null) {
+      return true;
+    }
+
+    return name.matches(pattern.replace("_", ".").replace("%", ".*"));

Review Comment:
   It would be good to escape the pattern beforehand (e.g. Pattern.quote), and pre-compile the pattern, though that will complicate the code.



##########
java/driver/flight-sql/src/main/java/org/apache/arrow/adbc/driver/flightsql/ObjectMetadataBuilder.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.adbc.driver.flightsql;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.core.StandardSchemas;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.complex.writer.VarCharWriter;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+final class ObjectMetadataBuilder {
+
+  private final FlightSqlClient client;
+  private final VectorSchemaRoot root;
+  private final VarCharVector adbcCatalogNames;
+  private final UnionListWriter adbcCatalogDbSchemasWriter;
+  private final BaseWriter.StructWriter adbcCatalogDbSchemasStructWriter;
+  private final BaseWriter.ListWriter adbcCatalogDbSchemaTablesWriter;
+  private final VarCharWriter adbcCatalogDbSchemaNameWriter;
+  private final BaseWriter.StructWriter adbcTablesStructWriter;
+  private final VarCharWriter adbcTableNameWriter;
+  private final VarCharWriter adbcTableTypeWriter;
+  private final BaseWriter.ListWriter adbcTableColumnsWriter;
+  private final BufferAllocator allocator;
+  private final AdbcConnection.GetObjectsDepth depth;
+  private final String catalogPattern;
+  private final String dbSchemaPattern;
+  private final String tableNamePattern;
+  private final String[] tableTypes;
+  private final String columnNamePattern;
+
+  ObjectMetadataBuilder(
+      BufferAllocator allocator,
+      FlightSqlClient client,
+      final AdbcConnection.GetObjectsDepth depth,
+      final String catalogPattern,
+      final String dbSchemaPattern,
+      final String tableNamePattern,
+      final String[] tableTypes,
+      final String columnNamePattern) {
+    this.allocator = allocator;
+    this.client = client;
+    this.depth = depth;
+    this.catalogPattern = catalogPattern;
+    this.dbSchemaPattern = dbSchemaPattern;
+    this.tableNamePattern = tableNamePattern;
+    this.tableTypes = tableTypes;
+    this.columnNamePattern = columnNamePattern;
+    this.root = VectorSchemaRoot.create(StandardSchemas.GET_OBJECTS_SCHEMA, allocator);
+    this.adbcCatalogNames = (VarCharVector) root.getVector(0);
+    this.adbcCatalogDbSchemasWriter = ((ListVector) root.getVector(1)).getWriter();
+    this.adbcCatalogDbSchemasStructWriter = adbcCatalogDbSchemasWriter.struct();
+    this.adbcCatalogDbSchemaTablesWriter =
+        adbcCatalogDbSchemasStructWriter.list("db_schema_tables");
+    this.adbcCatalogDbSchemaNameWriter = adbcCatalogDbSchemasStructWriter.varChar("db_schema_name");
+    this.adbcTablesStructWriter = adbcCatalogDbSchemaTablesWriter.struct();
+    this.adbcTableNameWriter = adbcTablesStructWriter.varChar("table_name");
+    this.adbcTableTypeWriter = adbcTablesStructWriter.varChar("table_type");
+    this.adbcTableColumnsWriter = adbcTablesStructWriter.list("table_columns");
+  }
+
+  private void writeVarChar(VarCharWriter writer, String value) {
+    byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+    try (ArrowBuf tempBuf = allocator.buffer(bytes.length)) {
+      tempBuf.setBytes(0, bytes, 0, bytes.length);
+      writer.writeVarChar(0, bytes.length, tempBuf);
+    }
+  }

Review Comment:
   I wonder why we don't just upstream this (at some point). This bit of code gets written so much for no gain...



##########
java/driver/flight-sql/src/main/java/org/apache/arrow/adbc/driver/flightsql/ObjectMetadataBuilder.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.adbc.driver.flightsql;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.core.StandardSchemas;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.sql.FlightSqlClient;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.complex.writer.VarCharWriter;
+import org.apache.arrow.vector.ipc.ReadChannel;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+final class ObjectMetadataBuilder {
+
+  private final FlightSqlClient client;
+  private final VectorSchemaRoot root;
+  private final VarCharVector adbcCatalogNames;
+  private final UnionListWriter adbcCatalogDbSchemasWriter;
+  private final BaseWriter.StructWriter adbcCatalogDbSchemasStructWriter;
+  private final BaseWriter.ListWriter adbcCatalogDbSchemaTablesWriter;
+  private final VarCharWriter adbcCatalogDbSchemaNameWriter;
+  private final BaseWriter.StructWriter adbcTablesStructWriter;
+  private final VarCharWriter adbcTableNameWriter;
+  private final VarCharWriter adbcTableTypeWriter;
+  private final BaseWriter.ListWriter adbcTableColumnsWriter;
+  private final BufferAllocator allocator;
+  private final AdbcConnection.GetObjectsDepth depth;
+  private final String catalogPattern;
+  private final String dbSchemaPattern;
+  private final String tableNamePattern;
+  private final String[] tableTypes;
+  private final String columnNamePattern;
+
+  ObjectMetadataBuilder(
+      BufferAllocator allocator,
+      FlightSqlClient client,
+      final AdbcConnection.GetObjectsDepth depth,
+      final String catalogPattern,
+      final String dbSchemaPattern,
+      final String tableNamePattern,
+      final String[] tableTypes,
+      final String columnNamePattern) {
+    this.allocator = allocator;
+    this.client = client;
+    this.depth = depth;
+    this.catalogPattern = catalogPattern;
+    this.dbSchemaPattern = dbSchemaPattern;
+    this.tableNamePattern = tableNamePattern;
+    this.tableTypes = tableTypes;
+    this.columnNamePattern = columnNamePattern;
+    this.root = VectorSchemaRoot.create(StandardSchemas.GET_OBJECTS_SCHEMA, allocator);
+    this.adbcCatalogNames = (VarCharVector) root.getVector(0);
+    this.adbcCatalogDbSchemasWriter = ((ListVector) root.getVector(1)).getWriter();
+    this.adbcCatalogDbSchemasStructWriter = adbcCatalogDbSchemasWriter.struct();
+    this.adbcCatalogDbSchemaTablesWriter =
+        adbcCatalogDbSchemasStructWriter.list("db_schema_tables");
+    this.adbcCatalogDbSchemaNameWriter = adbcCatalogDbSchemasStructWriter.varChar("db_schema_name");
+    this.adbcTablesStructWriter = adbcCatalogDbSchemaTablesWriter.struct();
+    this.adbcTableNameWriter = adbcTablesStructWriter.varChar("table_name");
+    this.adbcTableTypeWriter = adbcTablesStructWriter.varChar("table_type");
+    this.adbcTableColumnsWriter = adbcTablesStructWriter.list("table_columns");
+  }
+
+  private void writeVarChar(VarCharWriter writer, String value) {
+    byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+    try (ArrowBuf tempBuf = allocator.buffer(bytes.length)) {
+      tempBuf.setBytes(0, bytes, 0, bytes.length);
+      writer.writeVarChar(0, bytes.length, tempBuf);
+    }
+  }
+
+  private boolean patternMatched(String name, String pattern) {
+    if (pattern == null) {
+      return true;
+    }
+
+    return name.matches(pattern.replace("_", ".").replace("%", ".*"));
+  }
+
+  VectorSchemaRoot build() throws AdbcException {
+    // TODO Catalogs and schemas that don't contain tables are being left out
+    FlightInfo info;
+    if (depth == AdbcConnection.GetObjectsDepth.CATALOGS) {
+      info = client.getCatalogs();
+    } else if (depth == AdbcConnection.GetObjectsDepth.DB_SCHEMAS) {
+      info = client.getSchemas(null, dbSchemaPattern);
+    } else {
+      info =
+          client.getTables(
+              null, // TODO pattern match later during processing
+              dbSchemaPattern,
+              tableNamePattern,
+              tableTypes == null ? null : Arrays.asList(tableTypes),
+              depth == AdbcConnection.GetObjectsDepth.ALL);
+    }
+
+    byte[] lastCatalogAdded = null;
+    byte[] lastDbSchemaAdded = null;
+    int catalogIndex = 0;
+
+    for (FlightEndpoint endpoint : info.getEndpoints()) {
+      FlightStream stream = client.getStream(endpoint.getTicket());
+      while (stream.next()) {
+        try (VectorSchemaRoot res = stream.getRoot()) {
+          VarCharVector catalogVector = (VarCharVector) res.getVector(0);

Review Comment:
   Validate the schema somewhere before casting?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org