You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vi...@apache.org on 2021/09/06 03:46:36 UTC
[drill] branch master updated: DRILL-7995: Add Oracle HDFS lib to
query OCI Object Storage (#2311)
This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 41c2dcc DRILL-7995: Add Oracle HDFS lib to query OCI Object Storage (#2311)
41c2dcc is described below
commit 41c2dcc498afcbac8aa584c4180445ebf62b34ff
Author: Vitalii Diravka <vi...@apache.org>
AuthorDate: Mon Sep 6 06:46:28 2021 +0300
DRILL-7995: Add Oracle HDFS lib to query OCI Object Storage (#2311)
* DRILL-7995: Add Oracle HDFS lib to query OCI Object Storage
- add OCI OS dependency with exclusions
- update GlassFish Jersey dependencies version
- small refactoring in ViewExpansionContext
- add jersey-hk2 dependency (this injection framework was excluded from Jersey in latest versions)
- decrease jdbc-all
- update default plugins in bootstrap-storage-plugins.json
- udpate core-site-example.xml
- rename OCI OS Storage plugin "os" -> "oci"
---
distribution/pom.xml | 25 +++++++++
.../src/main/resources/core-site-example.xml | 43 ++++++++++++++++
exec/java-exec/pom.xml | 24 +++++----
.../drill/exec/ops/ViewExpansionContext.java | 5 +-
.../main/resources/bootstrap-storage-plugins.json | 59 ++++++++++++++++++++++
exec/jdbc-all/pom.xml | 6 ++-
exec/jdbc/pom.xml | 2 +-
pom.xml | 8 ++-
8 files changed, 157 insertions(+), 15 deletions(-)
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 0c5a5fc..7c3e69b 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -32,6 +32,7 @@
<properties>
<aws.java.sdk.version>1.11.375</aws.java.sdk.version>
+ <oci.hdfs.version>3.3.0.7.0.1</oci.hdfs.version>
</properties>
<dependencies>
@@ -147,6 +148,30 @@
<version>${aws.java.sdk.version}</version>
</dependency>
<dependency>
+ <groupId>com.oracle.oci.sdk</groupId>
+ <artifactId>oci-hdfs-connector</artifactId>
+ <version>${oci.hdfs.version}</version> <!--this version corresponds to the Drill Hadoop version-->
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<exclusions>
diff --git a/distribution/src/main/resources/core-site-example.xml b/distribution/src/main/resources/core-site-example.xml
index 3ce2e29..0261658 100644
--- a/distribution/src/main/resources/core-site-example.xml
+++ b/distribution/src/main/resources/core-site-example.xml
@@ -130,4 +130,47 @@
</property>
-->
+ <!--
+ HDFS Connector for Object Storage:
+ https://docs.oracle.com/en-us/iaas/Content/API/SDKDocs/hdfsconnector.htm
+ -->
+ <!--
+ <property>
+ <name>fs.oci.client.hostname</name>
+ <value>https://objectstorage.us-ashburn-1.oraclecloud.com</value>
+ <description>
+ The URL of the host endpoint. For example, https://www.example.com.
+ </description>
+ </property>
+ <property>
+ <name>fs.oci.client.auth.tenantId</name>
+ <value>ocid1.tenancy.oc1..exampleuniqueID</value>
+ <description>
+ The OCID of your tenancy. To get the value, see Required Keys and OCIDs:
+ https://docs.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm
+ </description>
+ </property>
+ <property>
+ <name>fs.oci.client.auth.userId</name>
+ <value>ocid1.user.oc1..exampleuniqueID</value>
+ <description>
+ The OCID of the user calling the API. To get the value, see Required Keys and OCIDs.
+ </description>
+ </property>
+ <property>
+ <name>fs.oci.client.auth.fingerprint</name>
+ <value>20:3b:97:13:55:1c:5b:0d:d3:37:d8:50:4e:c5:3a:34</value>
+ <description>
+ The fingerprint for the key pair being used. To get the value, see Required Keys and OCIDs.
+ </description>
+ </property>
+ <property>
+ <name>fs.oci.client.auth.pemfilepath</name>
+ <value>/opt/drill/conf/oci_api_key.pem</value>
+ <description>
+ The full path and file name of the private key used for authentication. The file should be on the local file system.
+ </description>
+ </property>
+ -->
+
</configuration>
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a022c7a..2660f60 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -198,6 +198,20 @@
<artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
+ <groupId>org.glassfish.jersey.ext</groupId>
+ <artifactId>jersey-mvc-freemarker</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>jersey-hk2</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>${jackson.version}</version>
@@ -212,16 +226,6 @@
<artifactId>jackson-dataformat-hocon</artifactId>
</dependency>
<dependency>
- <groupId>org.glassfish.jersey.ext</groupId>
- <artifactId>jersey-mvc-freemarker</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>${calcite.groupId}</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java
index 6b4191c..07fbae3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java
@@ -31,8 +31,9 @@ import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
/**
* Contains context information about view expansion(s) in a query. Part of {@link org.apache.drill.exec.ops
* .QueryContext}. Before expanding a view into its definition, as part of the
- * {@link org.apache.drill.exec.planner.logical.DrillViewTable#toRel(ToRelContext, RelOptTable)}, first a
- * {@link ViewExpansionToken} is requested from ViewExpansionContext through {@link #reserveViewExpansionToken(String)}.
+ * {@link org.apache.drill.exec.planner.logical.DrillViewTable#toRel(org.apache.calcite.plan.RelOptTable.ToRelContext,
+ * org.apache.calcite.plan.RelOptTable)}, first a {@link ViewExpansionToken} is requested from ViewExpansionContext
+ * through {@link #reserveViewExpansionToken(String)}.
* Once view expansion is complete, a token is released through {@link ViewExpansionToken#release()}. A view definition
* itself may contain zero or more views for expanding those nested views also a token is obtained.
*
diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
index 81f63f2..f171fc2 100644
--- a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -111,6 +111,65 @@
},
"enabled" : false
},
+ "oci" : {
+ "type" : "file",
+ "connection": "oci://{bucket_name}@{namespace_name}",
+ "config" : {
+ "fs.oci.client.hostname": "HOSTNAME",
+ "fs.oci.client.auth.tenantId": "TENANT_ID",
+ "fs.oci.client.auth.userId": "USER_ID",
+ "fs.oci.client.auth.fingerprint": "FINGERPRINT",
+ "fs.oci.client.auth.pemfilepath": "PEM_FILEPATH"
+ },
+ "workspaces" : {
+ "root" : {
+ "location" : "/",
+ "writable" : false
+ },
+ "tmp" : {
+ "location" : "/tmp",
+ "writable" : true
+ }
+ },
+ "formats" : {
+ "psv" : {
+ "type" : "text",
+ "extensions" : [ "tbl" ],
+ "fieldDelimiter" : "|"
+ },
+ "csv" : {
+ "type" : "text",
+ "extensions" : [ "csv" ],
+ "fieldDelimiter" : ","
+ },
+ "tsv" : {
+ "type" : "text",
+ "extensions" : [ "tsv" ],
+ "fieldDelimiter" : "\t"
+ },
+ "parquet" : {
+ "type" : "parquet"
+ },
+ "json" : {
+ "type" : "json",
+ "extensions" : [ "json" ]
+ },
+ "avro" : {
+ "type" : "avro"
+ },
+ "sequencefile" : {
+ "type" : "sequencefile",
+ "extensions" : [ "seq" ]
+ },
+ "csvh" : {
+ "type" : "text",
+ "extensions" : [ "csvh" ],
+ "fieldDelimiter" : ",",
+ "extractHeader" : true
+ }
+ },
+ "enabled" : false
+ },
"cp" : {
"type" : "file",
"connection" : "classpath:///",
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 7506d94..a69c4aa 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -81,6 +81,10 @@
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-common</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-jetty-servlet</artifactId>
</exclusion>
@@ -544,7 +548,7 @@
This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users.
</message>
- <maxsize>46900000</maxsize>
+ <maxsize>46600000</maxsize>
<minsize>15000000</minsize>
<files>
<file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file>
diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml
index d6d259d..052ab10 100644
--- a/exec/jdbc/pom.xml
+++ b/exec/jdbc/pom.xml
@@ -102,7 +102,7 @@
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
- <version>2.0.1.Final</version>
+ <version>${javax.validation.api}</version>
</dependency>
</dependencies>
diff --git a/pom.xml b/pom.xml
index 00cd1e3..9894041 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,7 +94,8 @@
<avro.version>1.9.1</avro.version>
<metrics.version>4.0.2</metrics.version>
<jetty.version>9.4.41.v20210516</jetty.version>
- <jersey.version>2.25.1</jersey.version>
+ <jersey.version>2.34</jersey.version>
+ <javax.validation.api>2.0.1.Final</javax.validation.api>
<asm.version>7.3.1</asm.version>
<excludedGroups />
<memoryMb>4000</memoryMb>
@@ -2933,6 +2934,11 @@
<artifactId>jersey-container-servlet</artifactId>
<version>${jersey.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>jersey-hk2</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
<!--/GlassFish Jersey dependecies-->
</dependencies>
</dependencyManagement>