You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2019/01/29 12:13:00 UTC
[carbondata] branch master updated: [HOTFIX] Upgraded jars to work
S3 with presto
This is an automated email from the ASF dual-hosted git repository.
kumarvishal09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 3f63f91 [HOTFIX] Upgraded jars to work S3 with presto
3f63f91 is described below
commit 3f63f91915d5da9d94a5c912b5415f230be64c07
Author: ravipesala <ra...@gmail.com>
AuthorDate: Sun Jan 27 15:12:29 2019 +0530
[HOTFIX] Upgraded jars to work S3 with presto
There is a duplicate jar aws-java-sdk and low version jars avoid connecting to S3 in presto. Those jars are upgraded in this PR and updated doc.
This closes #3110
---
.../statusmanager/SegmentUpdateStatusManager.java | 3 ++-
docs/presto-guide.md | 18 ++++-----------
integration/presto/pom.xml | 27 ++++++++--------------
3 files changed, 16 insertions(+), 32 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index c5f5f74..a02e903 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -52,6 +52,7 @@ import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
+import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
/**
@@ -655,7 +656,7 @@ public class SegmentUpdateStatusManager {
// get the updated status file identifier from the table status.
String tableUpdateStatusIdentifier = getUpdatedStatusIdentifier();
- if (null == tableUpdateStatusIdentifier) {
+ if (StringUtils.isEmpty(tableUpdateStatusIdentifier)) {
return new SegmentUpdateDetails[0];
}
diff --git a/docs/presto-guide.md b/docs/presto-guide.md
index 054f29f..7389bc6 100644
--- a/docs/presto-guide.md
+++ b/docs/presto-guide.md
@@ -254,23 +254,15 @@ Now you can use the Presto CLI on the coordinator to query data sources in the c
```
Required properties
- fs.s3a.access.key={value}
- fs.s3a.secret.key={value}
+ hive.s3.aws-access-key={value}
+ hive.s3.aws-secret-key={value}
Optional properties
- fs.s3a.endpoint={value}
+ hive.s3.endpoint={value}
```
- * In case you want to query carbonstore on s3 using S3 api put following additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties
- ```
- fs.s3.awsAccessKeyId={value}
- fs.s3.awsSecretAccessKey={value}
- ```
- * In case You want to query carbonstore on s3 using S3N api put following additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties
- ```
- fs.s3n.awsAccessKeyId={value}
- fs.s3n.awsSecretAccessKey={value}
- ```
+
+ Please refer <a>https://prestodb.io/docs/current/connector/hive.html</a> for more details on S3 integration.
### Generate CarbonData file
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index d69515d..8a9c06d 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -32,6 +32,7 @@
<properties>
<presto.version>0.210</presto.version>
+ <httpcore.version>4.4.9</httpcore.version>
<dev.path>${basedir}/../../dev</dev.path>
<jacoco.append>true</jacoco.append>
</properties>
@@ -376,7 +377,7 @@
<dependency>
<groupId>com.facebook.presto.hadoop</groupId>
<artifactId>hadoop-apache2</artifactId>
- <version>2.7.3-1</version>
+ <version>2.7.4-3</version>
<exclusions>
<exclusion>
<groupId>org.antlr</groupId>
@@ -522,23 +523,8 @@
<artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk</artifactId>
- <version>1.7.4</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -560,6 +546,11 @@
<artifactId>httpcore</artifactId>
<version>${httpcore.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.5</version>
+ </dependency>
</dependencies>
<build>