You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/02/03 17:17:40 UTC
[incubator-iceberg] branch spark-3 updated: Bump Apache spark to
3.0.0-preview2 (#743)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch spark-3
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/spark-3 by this push:
new 88c3c06 Bump Apache spark to 3.0.0-preview2 (#743)
88c3c06 is described below
commit 88c3c063404aa52ea407e23b683d35155892acfb
Author: yujiantao <30...@users.noreply.github.com>
AuthorDate: Tue Feb 4 01:17:33 2020 +0800
Bump Apache spark to 3.0.0-preview2 (#743)
---
build.gradle | 1 +
.../iceberg/spark/source/SparkBatchWrite.java | 16 +-
.../iceberg/spark/source/SparkStreamingWrite.java | 5 +-
.../spark/source/TestForwardCompatibility.java | 2 +-
.../spark/source/TestStructuredStreaming.java | 9 +-
.../apache/iceberg/spark/source/TestTables.java | 2 +-
versions.lock | 293 ++++++++++++---------
versions.props | 4 +-
8 files changed, 192 insertions(+), 140 deletions(-)
diff --git a/build.gradle b/build.gradle
index cc0f666..1e11666 100644
--- a/build.gradle
+++ b/build.gradle
@@ -242,6 +242,7 @@ project(':iceberg-spark') {
compileOnly "org.apache.avro:avro"
compileOnly("org.apache.spark:spark-hive_2.12") {
exclude group: 'org.apache.avro', module: 'avro'
+ exclude group: 'org.pentaho' // missing dependency
}
testCompile "org.apache.hadoop:hadoop-hdfs::tests"
diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchWrite.java b/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchWrite.java
index b0c1ded..b1e1fa5 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchWrite.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchWrite.java
@@ -62,6 +62,7 @@ import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.connector.write.BatchWrite;
import org.apache.spark.sql.connector.write.DataWriter;
import org.apache.spark.sql.connector.write.DataWriterFactory;
+import org.apache.spark.sql.connector.write.PhysicalWriteInfo;
import org.apache.spark.sql.connector.write.WriterCommitMessage;
import org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
@@ -127,7 +128,7 @@ class SparkBatchWrite implements BatchWrite {
}
@Override
- public WriterFactory createBatchWriterFactory() {
+ public WriterFactory createBatchWriterFactory(PhysicalWriteInfo info) {
return new WriterFactory(
table.spec(), format, table.locationProvider(), table.properties(), fileIo, encryptionManager, targetFileSize,
dsSchema);
@@ -493,6 +494,12 @@ class SparkBatchWrite implements BatchWrite {
public void write(InternalRow row) throws IOException {
writeInternal(row);
}
+
+ @Override
+ public void close() {
+ // close is called after abort or commit is called. Both abort and commit will close
+ // the current data file so there is no more cleanup to do here.
+ }
}
private static class PartitionedWriter extends BaseWriter {
@@ -533,5 +540,12 @@ class SparkBatchWrite implements BatchWrite {
writeInternal(row);
}
+
+
+ @Override
+ public void close() {
+ // close is called after abort or commit is called. Both abort and commit will close
+ // the current data file so there is no more cleanup to do here.
+ }
}
}
diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/SparkStreamingWrite.java b/spark/src/main/java/org/apache/iceberg/spark/source/SparkStreamingWrite.java
index c07c202..03042a4 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/source/SparkStreamingWrite.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/source/SparkStreamingWrite.java
@@ -28,6 +28,7 @@ import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotUpdate;
import org.apache.iceberg.Table;
import org.apache.iceberg.expressions.Expressions;
+import org.apache.spark.sql.connector.write.PhysicalWriteInfo;
import org.apache.spark.sql.connector.write.WriterCommitMessage;
import org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory;
import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
@@ -52,9 +53,9 @@ public class SparkStreamingWrite extends SparkBatchWrite implements StreamingWri
}
@Override
- public StreamingDataWriterFactory createStreamingWriterFactory() {
+ public StreamingDataWriterFactory createStreamingWriterFactory(PhysicalWriteInfo info) {
// the writer factory works for both batch and streaming
- return createBatchWriterFactory();
+ return createBatchWriterFactory(info);
}
@Override
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
index 5c6640e..85ac3e9 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java
@@ -125,7 +125,7 @@ public class TestForwardCompatibility {
}
@Test
- public void testSparkStreamingWriteFailsUnknownTransform() throws IOException {
+ public void testSparkStreamingWriteFailsUnknownTransform() throws Exception {
File parent = temp.newFolder("avro");
File location = new File(parent, "test");
File dataFolder = new File(location, "data");
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java
index b32ce7f..4d4fecb 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java
@@ -22,7 +22,6 @@ package org.apache.iceberg.spark.source;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import java.io.File;
-import java.io.IOException;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.PartitionSpec;
@@ -79,7 +78,7 @@ public class TestStructuredStreaming {
}
@Test
- public void testStreamingWriteAppendMode() throws IOException {
+ public void testStreamingWriteAppendMode() throws Exception {
File parent = temp.newFolder("parquet");
File location = new File(parent, "test-table");
File checkpoint = new File(parent, "checkpoint");
@@ -139,7 +138,7 @@ public class TestStructuredStreaming {
}
@Test
- public void testStreamingWriteCompleteMode() throws IOException {
+ public void testStreamingWriteCompleteMode() throws Exception {
File parent = temp.newFolder("parquet");
File location = new File(parent, "test-table");
File checkpoint = new File(parent, "checkpoint");
@@ -200,7 +199,7 @@ public class TestStructuredStreaming {
}
@Test
- public void testStreamingWriteCompleteModeWithProjection() throws IOException {
+ public void testStreamingWriteCompleteModeWithProjection() throws Exception {
File parent = temp.newFolder("parquet");
File location = new File(parent, "test-table");
File checkpoint = new File(parent, "checkpoint");
@@ -261,7 +260,7 @@ public class TestStructuredStreaming {
}
@Test
- public void testStreamingWriteUpdateMode() throws IOException {
+ public void testStreamingWriteUpdateMode() throws Exception {
exceptionRule.expect(StreamingQueryException.class);
// The following error message to verify is issued by
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestTables.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestTables.java
index 92f8e29..879aa16 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestTables.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestTables.java
@@ -19,6 +19,7 @@
package org.apache.iceberg.spark.source;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import java.io.File;
import java.util.Map;
@@ -37,7 +38,6 @@ import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.io.LocationProvider;
import org.apache.iceberg.io.OutputFile;
-import parquet.Preconditions;
// TODO: Use the copy of this from core.
class TestTables {
diff --git a/versions.lock b/versions.lock
index 6010faa..947f5fb 100644
--- a/versions.lock
+++ b/versions.lock
@@ -1,36 +1,42 @@
# Run ./gradlew --write-locks to regenerate this file
-ant:ant:1.6.5 (1 constraints: bb0a41d8)
-antlr:antlr:2.7.7 (2 constraints: 36167e02)
+ant:ant:1.6.5 (2 constraints: 2d1539e1)
aopalliance:aopalliance:1.0 (1 constraints: 170a83ac)
-asm:asm:3.1 (2 constraints: 4f19c3c6)
-com.carrotsearch:hppc:0.7.2 (1 constraints: f70cda14)
+asm:asm:3.1 (3 constraints: 251fd7ad)
+asm:asm-commons:3.1 (1 constraints: 9c0f1f7a)
+asm:asm-tree:3.1 (1 constraints: 2307035c)
+ch.qos.logback:logback-classic:1.0.9 (3 constraints: d3258a88)
+ch.qos.logback:logback-core:1.0.9 (4 constraints: dd32435a)
+co.cask.tephra:tephra-api:0.6.0 (3 constraints: 0828ded1)
+co.cask.tephra:tephra-core:0.6.0 (2 constraints: 831cd90d)
+co.cask.tephra:tephra-hbase-compat-1.0:0.6.0 (1 constraints: 370d6920)
com.clearspring.analytics:stream:2.9.6 (1 constraints: 230de736)
com.esotericsoftware:kryo-shaded:4.0.2 (2 constraints: b8134fa6)
com.esotericsoftware:minlog:1.3.0 (1 constraints: 670e7c4f)
-com.fasterxml.jackson.core:jackson-annotations:2.10.0 (4 constraints: 884807a8)
-com.fasterxml.jackson.core:jackson-core:2.10.0 (5 constraints: 60499d20)
-com.fasterxml.jackson.core:jackson-databind:2.10.0 (8 constraints: 297bcc44)
+com.fasterxml.jackson.core:jackson-annotations:2.10.0 (5 constraints: 1f57675b)
+com.fasterxml.jackson.core:jackson-core:2.10.0 (6 constraints: f85730ec)
+com.fasterxml.jackson.core:jackson-databind:2.10.0 (10 constraints: 1996cd43)
com.fasterxml.jackson.module:jackson-module-paranamer:2.10.0 (1 constraints: 02163516)
com.fasterxml.jackson.module:jackson-module-scala_2.12:2.10.0 (1 constraints: 450d1044)
com.github.ben-manes.caffeine:caffeine:2.7.0 (1 constraints: 0b050a36)
-com.github.luben:zstd-jni:1.4.3-1 (1 constraints: 780d8f51)
+com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:0.1.2 (1 constraints: e90b08f3)
+com.github.luben:zstd-jni:1.4.4-3 (1 constraints: 7b0d9451)
com.github.spotbugs:spotbugs-annotations:3.1.9 (1 constraints: 8d0d3128)
-com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (1 constraints: 6d05ab40)
-com.google.code.findbugs:jsr305:3.0.2 (10 constraints: 6b88df04)
-com.google.code.gson:gson:2.2.4 (1 constraints: 8c0d3f2f)
+com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (5 constraints: 693c9a5d)
+com.google.code.findbugs:jsr305:3.0.2 (14 constraints: f2bc5352)
+com.google.code.gson:gson:2.2.4 (5 constraints: 3b40b359)
com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 161a2544)
com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714)
com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4)
-com.google.guava:guava:28.0-jre (23 constraints: cc5c2ea0)
+com.google.guava:guava:28.0-jre (29 constraints: 0dae25de)
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918)
-com.google.inject:guice:3.0 (6 constraints: 6873914c)
+com.google.inject:guice:3.0 (8 constraints: 2c93366d)
+com.google.inject.extensions:guice-assistedinject:3.0 (1 constraints: 250b42ce)
com.google.inject.extensions:guice-servlet:3.0 (11 constraints: a9d50a2b)
com.google.j2objc:j2objc-annotations:1.3 (1 constraints: b809eda0)
-com.google.protobuf:protobuf-java:2.5.0 (15 constraints: f80eac0f)
-com.googlecode.javaewah:JavaEWAH:0.3.2 (1 constraints: ea0dfc42)
+com.google.protobuf:protobuf-java:3.0.0-beta-1 (20 constraints: 9253e608)
com.jamesmurty.utils:java-xmlbuilder:0.4 (1 constraints: e40aa5ca)
com.jcraft:jsch:0.1.54 (1 constraints: be0df13c)
-com.jolbox:bonecp:0.8.0.RELEASE (2 constraints: b22109f9)
+com.jolbox:bonecp:0.8.0.RELEASE (1 constraints: 680fc193)
com.ning:compress-lzf:1.0.3 (1 constraints: 160dc436)
com.sun.jersey:jersey-client:1.9 (4 constraints: 65529ed7)
com.sun.jersey:jersey-core:1.9 (9 constraints: ec8f4404)
@@ -38,90 +44,105 @@ com.sun.jersey:jersey-json:1.9 (5 constraints: 945f2f90)
com.sun.jersey:jersey-server:1.9 (4 constraints: ef373c01)
com.sun.jersey.contribs:jersey-guice:1.9 (4 constraints: 65529ed7)
com.sun.xml.bind:jaxb-impl:2.2.3-1 (1 constraints: 330c2404)
+com.tdunning:json:1.8 (1 constraints: 910b72db)
com.thoughtworks.paranamer:paranamer:2.8 (4 constraints: 68364d1c)
com.twitter:chill-java:0.9.3 (2 constraints: a916ae6f)
com.twitter:chill_2.12:0.9.3 (2 constraints: 141bd1c3)
-com.twitter:parquet-hadoop-bundle:1.6.0 (3 constraints: 7d265724)
com.univocity:univocity-parsers:2.8.3 (2 constraints: a01b9ef6)
+com.zaxxer:HikariCP:2.5.1 (1 constraints: 390d7120)
commons-beanutils:commons-beanutils:1.7.0 (1 constraints: da0e635f)
commons-beanutils:commons-beanutils-core:1.8.0 (1 constraints: 1d134124)
-commons-cli:commons-cli:1.2 (8 constraints: 9467c282)
-commons-codec:commons-codec:1.10 (18 constraints: d1eef4f2)
-commons-collections:commons-collections:3.2.2 (4 constraints: 42476934)
+commons-cli:commons-cli:1.2 (8 constraints: 40657789)
+commons-codec:commons-codec:1.10 (21 constraints: ba140706)
+commons-collections:commons-collections:3.2.2 (4 constraints: 8047e302)
commons-configuration:commons-configuration:1.6 (1 constraints: 2d0d5c14)
commons-daemon:commons-daemon:1.0.13 (1 constraints: d50c811c)
-commons-dbcp:commons-dbcp:1.4 (3 constraints: 9029e0e4)
+commons-dbcp:commons-dbcp:1.4 (2 constraints: d6191af9)
commons-digester:commons-digester:1.8 (1 constraints: bf1228fe)
-commons-el:commons-el:1.0 (1 constraints: fb077074)
+commons-el:commons-el:1.0 (2 constraints: ad11e7f0)
commons-httpclient:commons-httpclient:3.1 (4 constraints: e62ccf7f)
-commons-io:commons-io:2.4 (6 constraints: 4a568049)
-commons-lang:commons-lang:2.6 (19 constraints: 5f0d34d8)
-commons-logging:commons-logging:1.2 (27 constraints: f2ab390b)
+commons-io:commons-io:2.4 (7 constraints: 2a61d030)
+commons-lang:commons-lang:2.6 (20 constraints: 651a744c)
+commons-logging:commons-logging:1.2 (22 constraints: b160d978)
commons-net:commons-net:3.1 (3 constraints: 3e223661)
-commons-pool:commons-pool:1.6 (4 constraints: e336ab5e)
+commons-pool:commons-pool:1.6 (3 constraints: c626e5cb)
dk.brics.automaton:automaton:1.11-8 (1 constraints: 92088a8d)
hsqldb:hsqldb:1.8.0.10 (1 constraints: f008499f)
io.airlift:aircompressor:0.10 (1 constraints: 090a9fb2)
-io.dropwizard.metrics:metrics-core:3.2.6 (6 constraints: 955e05c1)
-io.dropwizard.metrics:metrics-graphite:3.2.6 (1 constraints: 1d0dd736)
-io.dropwizard.metrics:metrics-json:3.2.6 (1 constraints: 1d0dd736)
-io.dropwizard.metrics:metrics-jvm:3.2.6 (1 constraints: 1d0dd736)
+io.dropwizard.metrics:metrics-core:4.1.1 (10 constraints: 6a9f308c)
+io.dropwizard.metrics:metrics-graphite:4.1.1 (1 constraints: 180dd436)
+io.dropwizard.metrics:metrics-jmx:4.1.1 (1 constraints: 180dd436)
+io.dropwizard.metrics:metrics-json:4.1.1 (2 constraints: 01196712)
+io.dropwizard.metrics:metrics-jvm:4.1.1 (2 constraints: 01196712)
io.netty:netty:3.10.6.Final (8 constraints: 91a1ee0f)
-io.netty:netty-all:4.1.42.Final (3 constraints: d031f725)
-jakarta.annotation:jakarta.annotation-api:1.3.4 (4 constraints: 083a0f79)
-jakarta.ws.rs:jakarta.ws.rs-api:2.1.5 (5 constraints: 8764845f)
-javax.activation:activation:1.1.1 (1 constraints: 150dc536)
+io.netty:netty-all:4.1.42.Final (4 constraints: aa40f259)
+it.unimi.dsi:fastutil:6.5.6 (1 constraints: 910b3ce5)
+jakarta.annotation:jakarta.annotation-api:1.3.5 (4 constraints: 0a3a4179)
+jakarta.validation:jakarta.validation-api:2.0.2 (1 constraints: fd10b6c3)
+jakarta.ws.rs:jakarta.ws.rs-api:2.1.6 (5 constraints: 8c647f61)
+javax.activation:activation:1.1.1 (3 constraints: b12367a0)
javax.inject:javax.inject:1 (2 constraints: b018a173)
-javax.jdo:jdo-api:3.0.1 (2 constraints: 4c1dcc1a)
+javax.jdo:jdo-api:3.0.1 (1 constraints: 350d6720)
+javax.mail:mail:1.4.1 (1 constraints: fc0fe399)
javax.servlet:javax.servlet-api:3.1.0 (1 constraints: 160dce36)
-javax.servlet:servlet-api:2.5 (7 constraints: c87e5aa1)
+javax.servlet:jsp-api:2.0 (1 constraints: 0b0aa0a7)
+javax.servlet:servlet-api:2.5 (9 constraints: f991a6d2)
javax.servlet.jsp:jsp-api:2.1 (1 constraints: 290d5a14)
javax.transaction:jta:1.1 (1 constraints: 9f07d96b)
-javax.validation:validation-api:2.0.1.Final (1 constraints: 14133a30)
+javax.transaction:transaction-api:1.1 (1 constraints: 0a0b64c9)
javax.xml.bind:jaxb-api:2.2.11 (6 constraints: a069fd48)
-javolution:javolution:5.5.1 (1 constraints: f00d1a43)
-jline:jline:2.12 (3 constraints: 98208776)
-joda-time:joda-time:2.9.9 (5 constraints: c33279e6)
-log4j:apache-log4j-extras:1.2.17 (4 constraints: 3f36b1af)
-log4j:log4j:1.2.17 (12 constraints: 24ab8929)
+javolution:javolution:5.5.1 (1 constraints: 3c0d8020)
+jline:jline:2.12 (3 constraints: 7c21b2cb)
+joda-time:joda-time:2.10.5 (5 constraints: 1f34d3f9)
+junit:junit:4.12 (5 constraints: 8b39da91)
+log4j:log4j:1.2.17 (11 constraints: 61a1efd6)
+net.hydromatic:eigenbase-properties:1.1.5 (1 constraints: 5f0daf2c)
net.java.dev.jets3t:jets3t:0.9.0 (2 constraints: ec152b22)
net.razorvine:pyrolite:4.30 (1 constraints: eb0cc229)
net.sf.kosmosfs:kfs:0.3 (1 constraints: fd077074)
-net.sf.opencsv:opencsv:2.3 (2 constraints: a218daa5)
+net.sf.opencsv:opencsv:2.3 (1 constraints: 170b02ce)
net.sf.py4j:py4j:0.10.8.1 (1 constraints: aa0d2f5f)
-org.antlr:ST4:4.0.4 (3 constraints: 5521e4e4)
-org.antlr:antlr-runtime:3.4 (6 constraints: b84229a1)
+org.antlr:ST4:4.0.4 (2 constraints: 69130686)
+org.antlr:antlr-runtime:3.5.2 (4 constraints: 31260d06)
org.antlr:antlr4-runtime:4.7.1 (1 constraints: da0e9d7c)
-org.antlr:stringtemplate:3.2.1 (1 constraints: c10a3bc6)
-org.apache.ant:ant:1.9.1 (3 constraints: a721ed14)
+org.apache.ant:ant:1.9.1 (3 constraints: 262660e7)
org.apache.ant:ant-launcher:1.9.1 (1 constraints: 69082485)
-org.apache.arrow:arrow-format:0.12.0 (1 constraints: 210ded21)
-org.apache.arrow:arrow-memory:0.12.0 (1 constraints: 210ded21)
-org.apache.arrow:arrow-vector:0.12.0 (1 constraints: 010f7d8b)
+org.apache.arrow:arrow-format:0.15.1 (1 constraints: 250df721)
+org.apache.arrow:arrow-memory:0.15.1 (1 constraints: 250df721)
+org.apache.arrow:arrow-vector:0.15.1 (1 constraints: 050f878b)
org.apache.avro:avro:1.8.2 (4 constraints: 3d2eebf3)
org.apache.avro:avro-ipc:1.8.2 (1 constraints: f90b5bf4)
org.apache.avro:avro-mapred:1.8.2 (2 constraints: 3c1a8487)
-org.apache.commons:commons-compress:1.8.1 (6 constraints: 274bbeb0)
+org.apache.calcite:calcite-core:1.10.0 (2 constraints: 2c19fe15)
+org.apache.calcite:calcite-druid:1.10.0 (1 constraints: 340b16e2)
+org.apache.calcite:calcite-linq4j:1.10.0 (2 constraints: 821ba1bd)
+org.apache.calcite.avatica:avatica:1.8.0 (3 constraints: 3a26a801)
+org.apache.calcite.avatica:avatica-metrics:1.8.0 (1 constraints: 960e635d)
+org.apache.commons:commons-compress:1.9 (5 constraints: 893cc0b4)
org.apache.commons:commons-crypto:1.0.0 (2 constraints: 3c1ea6bf)
-org.apache.commons:commons-lang3:3.9 (5 constraints: 583e671a)
+org.apache.commons:commons-lang3:3.9 (9 constraints: 7e6fc79d)
org.apache.commons:commons-math3:3.4.1 (2 constraints: a21afc90)
org.apache.commons:commons-text:1.6 (1 constraints: bb0cd11c)
+org.apache.curator:apache-curator:2.7.1 (2 constraints: c718e2d6)
org.apache.curator:curator-client:2.7.1 (2 constraints: 6a1d2734)
-org.apache.curator:curator-framework:2.7.1 (4 constraints: 4937d02c)
+org.apache.curator:curator-framework:2.7.1 (5 constraints: 0845a22f)
org.apache.curator:curator-recipes:2.7.1 (2 constraints: a91ada91)
-org.apache.derby:derby:10.12.1.1 (3 constraints: a02cf182)
+org.apache.derby:derby:10.12.1.1 (2 constraints: cb1b0c38)
org.apache.directory.api:api-asn1-api:1.0.0-M20 (1 constraints: 3d163b13)
org.apache.directory.api:api-util:1.0.0-M20 (1 constraints: 3d163b13)
org.apache.directory.server:apacheds-i18n:2.0.0-M15 (1 constraints: 42164713)
org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15 (1 constraints: 8f0d3b45)
+org.apache.geronimo.specs:geronimo-annotation_1.0_spec:1.1.1 (1 constraints: f90fda99)
+org.apache.geronimo.specs:geronimo-jaspic_1.0_spec:1.0 (1 constraints: 990f187a)
+org.apache.geronimo.specs:geronimo-jta_1.1_spec:1.1.1 (1 constraints: f90fda99)
org.apache.hadoop:hadoop-annotations:2.7.4 (16 constraints: 3c2741a6)
-org.apache.hadoop:hadoop-auth:2.7.4 (1 constraints: 910d4e2f)
+org.apache.hadoop:hadoop-auth:2.7.4 (2 constraints: 221aed3f)
org.apache.hadoop:hadoop-client:2.7.4 (2 constraints: 2c12103c)
-org.apache.hadoop:hadoop-common:2.7.4 (3 constraints: 4a2296f7)
+org.apache.hadoop:hadoop-common:2.7.4 (6 constraints: 44567288)
org.apache.hadoop:hadoop-hdfs:2.7.4 (4 constraints: bb345226)
org.apache.hadoop:hadoop-mapreduce-client-app:2.7.4 (3 constraints: ae2f1637)
org.apache.hadoop:hadoop-mapreduce-client-common:2.7.4 (4 constraints: 1c4fae67)
-org.apache.hadoop:hadoop-mapreduce-client-core:2.7.4 (4 constraints: 6936ae12)
+org.apache.hadoop:hadoop-mapreduce-client-core:2.7.4 (6 constraints: 954ff132)
org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.7.4 (2 constraints: 3d1d2914)
org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.7.4 (2 constraints: 2828024a)
org.apache.hadoop:hadoop-yarn-api:2.7.4 (10 constraints: 11b87d56)
@@ -130,92 +151,121 @@ org.apache.hadoop:hadoop-yarn-common:2.7.4 (9 constraints: bcb2d777)
org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice:2.7.4 (1 constraints: f6157ecf)
org.apache.hadoop:hadoop-yarn-server-common:2.7.4 (7 constraints: 589281c5)
org.apache.hadoop:hadoop-yarn-server-nodemanager:2.7.4 (2 constraints: 69267b8d)
-org.apache.hadoop:hadoop-yarn-server-resourcemanager:2.7.4 (2 constraints: b0206faf)
+org.apache.hadoop:hadoop-yarn-server-resourcemanager:2.7.4 (2 constraints: b32074af)
org.apache.hadoop:hadoop-yarn-server-web-proxy:2.7.4 (2 constraints: cd28b579)
-org.apache.hive:hive-common:1.2.1 (1 constraints: 740bb5e4)
-org.apache.hive:hive-metastore:1.2.1 (2 constraints: 0b1094b7)
-org.apache.hive:hive-serde:1.2.1 (1 constraints: 350d6320)
-org.apache.hive:hive-shims:1.2.1 (4 constraints: 962f99d3)
-org.apache.hive.shims:hive-shims-0.20S:1.2.1 (1 constraints: 850b5fe5)
-org.apache.hive.shims:hive-shims-0.23:1.2.1 (1 constraints: 850b5fe5)
-org.apache.hive.shims:hive-shims-common:1.2.1 (4 constraints: 233b0a15)
-org.apache.hive.shims:hive-shims-scheduler:1.2.1 (1 constraints: 850b5fe5)
-org.apache.htrace:htrace-core:3.1.0-incubating (2 constraints: cd22cffa)
-org.apache.httpcomponents:httpclient:4.5.6 (4 constraints: 5e316add)
-org.apache.httpcomponents:httpcore:4.4.10 (3 constraints: d327f763)
-org.apache.ivy:ivy:2.4.0 (3 constraints: 09260ef2)
-org.apache.orc:orc-core:1.5.7 (2 constraints: d311ea27)
-org.apache.orc:orc-mapreduce:1.5.7 (1 constraints: c60cce27)
-org.apache.orc:orc-shims:1.5.7 (1 constraints: 430aecbc)
+org.apache.hbase:hbase-annotations:1.1.1 (3 constraints: a226bc12)
+org.apache.hbase:hbase-client:1.1.1 (1 constraints: 340d6020)
+org.apache.hbase:hbase-common:1.1.1 (1 constraints: 8d0c100d)
+org.apache.hbase:hbase-protocol:1.1.1 (2 constraints: 231984f7)
+org.apache.hive:hive-common:2.3.6 (5 constraints: ac4092a5)
+org.apache.hive:hive-exec:2.3.6 (2 constraints: 2512dc3a)
+org.apache.hive:hive-llap-client:2.3.6 (2 constraints: b019e847)
+org.apache.hive:hive-llap-common:2.3.6 (2 constraints: dc1a45a5)
+org.apache.hive:hive-llap-tez:2.3.6 (1 constraints: 0d0bf1d6)
+org.apache.hive:hive-metastore:2.3.6 (2 constraints: 2512dc3a)
+org.apache.hive:hive-serde:2.3.6 (3 constraints: 2128b5ff)
+org.apache.hive:hive-service-rpc:2.3.6 (1 constraints: 7b0bc2e4)
+org.apache.hive:hive-shims:2.3.6 (5 constraints: d13c96b4)
+org.apache.hive:hive-storage-api:2.6.0 (2 constraints: ac18bef7)
+org.apache.hive:hive-vector-code-gen:2.3.6 (1 constraints: 0d0bf1d6)
+org.apache.hive.shims:hive-shims-0.23:2.3.6 (1 constraints: 8c0b6ce5)
+org.apache.hive.shims:hive-shims-common:2.3.6 (3 constraints: 222cfaad)
+org.apache.hive.shims:hive-shims-scheduler:2.3.6 (1 constraints: 8c0b6ce5)
+org.apache.htrace:htrace-core:3.1.0-incubating (4 constraints: 934436e9)
+org.apache.httpcomponents:httpclient:4.5.6 (5 constraints: f53f9dad)
+org.apache.httpcomponents:httpcore:4.4.10 (4 constraints: 6b36e652)
+org.apache.ivy:ivy:2.4.0 (2 constraints: 1f18fdd1)
+org.apache.logging.log4j:log4j-1.2-api:2.6.2 (2 constraints: fb160260)
+org.apache.logging.log4j:log4j-api:2.6.2 (4 constraints: 2e3c9f23)
+org.apache.logging.log4j:log4j-core:2.6.2 (2 constraints: fd1c2464)
+org.apache.logging.log4j:log4j-slf4j-impl:2.6.2 (3 constraints: 8d27cb64)
+org.apache.logging.log4j:log4j-web:2.6.2 (1 constraints: f00b21f3)
+org.apache.orc:orc-core:1.5.8 (3 constraints: c11d09ae)
+org.apache.orc:orc-mapreduce:1.5.8 (1 constraints: c70ccf27)
+org.apache.orc:orc-shims:1.5.8 (1 constraints: 440aedbc)
org.apache.parquet:parquet-avro:1.10.1 (1 constraints: 35052a3b)
org.apache.parquet:parquet-column:1.10.1 (3 constraints: 9529f9ca)
org.apache.parquet:parquet-common:1.10.1 (2 constraints: 4c1e7385)
org.apache.parquet:parquet-encoding:1.10.1 (1 constraints: ca0ef964)
org.apache.parquet:parquet-format:2.4.0 (3 constraints: e72a97ca)
org.apache.parquet:parquet-hadoop:1.10.1 (2 constraints: df1ad0b3)
+org.apache.parquet:parquet-hadoop-bundle:1.8.1 (1 constraints: 7a0bc7e4)
org.apache.parquet:parquet-jackson:1.10.1 (1 constraints: b70ee763)
org.apache.pig:pig:0.14.0 (1 constraints: 37052f3b)
-org.apache.spark:spark-avro_2.12:3.0.0-preview (1 constraints: 3408936b)
-org.apache.spark:spark-catalyst_2.12:3.0.0-preview (1 constraints: eb0f139b)
-org.apache.spark:spark-core_2.12:3.0.0-preview (3 constraints: 30323128)
-org.apache.spark:spark-hive_2.12:3.0.0-preview (1 constraints: 3408936b)
-org.apache.spark:spark-kvstore_2.12:3.0.0-preview (1 constraints: 4410e4ac)
-org.apache.spark:spark-launcher_2.12:3.0.0-preview (1 constraints: 4410e4ac)
-org.apache.spark:spark-network-common_2.12:3.0.0-preview (2 constraints: 02258f19)
-org.apache.spark:spark-network-shuffle_2.12:3.0.0-preview (1 constraints: 4410e4ac)
-org.apache.spark:spark-sketch_2.12:3.0.0-preview (2 constraints: ea219afb)
-org.apache.spark:spark-sql_2.12:3.0.0-preview (1 constraints: 471030ad)
-org.apache.spark:spark-tags_2.12:3.0.0-preview (8 constraints: 4b88bf32)
-org.apache.spark:spark-unsafe_2.12:3.0.0-preview (2 constraints: 4322791f)
-org.apache.thrift:libfb303:0.9.3 (3 constraints: 682504c1)
-org.apache.thrift:libthrift:0.12.0 (5 constraints: 9941518e)
-org.apache.xbean:xbean-asm7-shaded:4.14 (2 constraints: 8019e63c)
+org.apache.spark:spark-avro_2.12:3.0.0-preview2 (1 constraints: 6608f973)
+org.apache.spark:spark-catalyst_2.12:3.0.0-preview2 (1 constraints: 1d1030ab)
+org.apache.spark:spark-core_2.12:3.0.0-preview2 (3 constraints: c63225ac)
+org.apache.spark:spark-hive_2.12:3.0.0-preview2 (1 constraints: 6608f973)
+org.apache.spark:spark-kvstore_2.12:3.0.0-preview2 (1 constraints: 76105abd)
+org.apache.spark:spark-launcher_2.12:3.0.0-preview2 (1 constraints: 76105abd)
+org.apache.spark:spark-network-common_2.12:3.0.0-preview2 (2 constraints: 6625235b)
+org.apache.spark:spark-network-shuffle_2.12:3.0.0-preview2 (1 constraints: 76105abd)
+org.apache.spark:spark-sketch_2.12:3.0.0-preview2 (2 constraints: 4e228939)
+org.apache.spark:spark-sql_2.12:3.0.0-preview2 (1 constraints: 7910a9bd)
+org.apache.spark:spark-tags_2.12:3.0.0-preview2 (8 constraints: db8979b5)
+org.apache.spark:spark-unsafe_2.12:3.0.0-preview2 (2 constraints: a722e45d)
+org.apache.thrift:libfb303:0.9.3 (3 constraints: 2828a707)
+org.apache.thrift:libthrift:0.12.0 (7 constraints: ef5a2934)
+org.apache.twill:twill-api:0.6.0-incubating (2 constraints: e422b4e1)
+org.apache.twill:twill-common:0.6.0-incubating (4 constraints: 3d467991)
+org.apache.twill:twill-core:0.6.0-incubating (1 constraints: d70f9d7c)
+org.apache.twill:twill-discovery-api:0.6.0-incubating (3 constraints: 25345d4c)
+org.apache.twill:twill-discovery-core:0.6.0-incubating (2 constraints: 332039f9)
+org.apache.twill:twill-zookeeper:0.6.0-incubating (3 constraints: 94341288)
+org.apache.velocity:velocity:1.5 (1 constraints: c70e875e)
+org.apache.xbean:xbean-asm7-shaded:4.15 (2 constraints: 82190f3d)
org.apache.yetus:audience-annotations:0.5.0 (1 constraints: 850d2528)
-org.apache.zookeeper:zookeeper:3.4.14 (11 constraints: 48a76cef)
+org.apache.zookeeper:zookeeper:3.4.14 (14 constraints: d8cfaf37)
org.checkerframework:checker-qual:2.8.1 (2 constraints: 1a1a3944)
-org.codehaus.jackson:jackson-core-asl:1.9.13 (13 constraints: c6a96183)
+org.codehaus.groovy:groovy-all:2.4.4 (1 constraints: 0c0bf2d6)
+org.codehaus.jackson:jackson-core-asl:1.9.13 (12 constraints: d69bba49)
org.codehaus.jackson:jackson-jaxrs:1.9.13 (2 constraints: 821bca9d)
-org.codehaus.jackson:jackson-mapper-asl:1.9.13 (13 constraints: 28a66532)
+org.codehaus.jackson:jackson-mapper-asl:1.9.13 (13 constraints: ffa443b4)
org.codehaus.jackson:jackson-xc:1.9.13 (2 constraints: 821bca9d)
org.codehaus.janino:commons-compiler:3.0.15 (3 constraints: 65285c17)
org.codehaus.janino:janino:3.0.15 (2 constraints: 6d1cd220)
org.codehaus.jettison:jettison:1.1 (4 constraints: a84e24a9)
org.codehaus.mojo:animal-sniffer-annotations:1.17 (1 constraints: ed09d8aa)
-org.datanucleus:datanucleus-api-jdo:3.2.6 (2 constraints: 5a1d241c)
-org.datanucleus:datanucleus-core:3.2.10 (5 constraints: 4b4465e0)
-org.datanucleus:datanucleus-rdbms:3.2.9 (2 constraints: 601db41c)
+org.datanucleus:datanucleus-api-jdo:4.2.4 (1 constraints: 3b0d7520)
+org.datanucleus:datanucleus-core:4.1.17 (3 constraints: fd258d22)
+org.datanucleus:datanucleus-rdbms:4.1.19 (1 constraints: 700ddf2d)
+org.datanucleus:javax.jdo:3.2.0-m3 (1 constraints: 030ea249)
org.eclipse.jdt:core:3.1.1 (1 constraints: b40a38d8)
+org.eclipse.jetty.aggregate:jetty-all:7.6.0.v20120127 (1 constraints: 260ea577)
+org.eclipse.jetty.orbit:javax.servlet:3.0.0.v201112011016 (1 constraints: dd0e53b1)
org.fusesource.leveldbjni:leveldbjni-all:1.8 (9 constraints: 93a6dbe7)
-org.glassfish.hk2:hk2-api:2.5.0 (1 constraints: 6c0c5e0c)
-org.glassfish.hk2:hk2-locator:2.5.0 (1 constraints: 4210dcad)
-org.glassfish.hk2:hk2-utils:2.5.0 (2 constraints: 1d17de76)
+org.glassfish.hk2:hk2-api:2.6.1 (1 constraints: 6e0c620c)
+org.glassfish.hk2:hk2-locator:2.6.1 (1 constraints: 4410e0ad)
+org.glassfish.hk2:hk2-utils:2.6.1 (2 constraints: 21173277)
org.glassfish.hk2:osgi-resource-locator:1.0.3 (2 constraints: 7d23b265)
-org.glassfish.hk2.external:aopalliance-repackaged:2.5.0 (2 constraints: 1d17de76)
-org.glassfish.hk2.external:jakarta.inject:2.5.0 (8 constraints: ed811f01)
-org.glassfish.jersey.containers:jersey-container-servlet:2.29 (1 constraints: f10cc129)
-org.glassfish.jersey.containers:jersey-container-servlet-core:2.29 (2 constraints: b324b295)
-org.glassfish.jersey.core:jersey-client:2.29 (2 constraints: c81dc63c)
-org.glassfish.jersey.core:jersey-common:2.29 (7 constraints: 6382122d)
-org.glassfish.jersey.core:jersey-server:2.29 (3 constraints: 4b3e492d)
-org.glassfish.jersey.inject:jersey-hk2:2.29 (1 constraints: f10cc129)
-org.glassfish.jersey.media:jersey-media-jaxb:2.29 (1 constraints: d810c8b2)
-org.iq80.snappy:snappy:0.2 (1 constraints: 890d5927)
+org.glassfish.hk2.external:aopalliance-repackaged:2.6.1 (2 constraints: 21173277)
+org.glassfish.hk2.external:jakarta.inject:2.6.1 (8 constraints: fd81450c)
+org.glassfish.jersey.containers:jersey-container-servlet:2.29.1 (1 constraints: 500d3044)
+org.glassfish.jersey.containers:jersey-container-servlet-core:2.29.1 (2 constraints: 7125a112)
+org.glassfish.jersey.core:jersey-client:2.29.1 (2 constraints: 861e81a5)
+org.glassfish.jersey.core:jersey-common:2.29.1 (7 constraints: fc846bf1)
+org.glassfish.jersey.core:jersey-server:2.29.1 (3 constraints: 683f0d5c)
+org.glassfish.jersey.inject:jersey-hk2:2.29.1 (1 constraints: 500d3044)
+org.glassfish.jersey.media:jersey-media-jaxb:2.29.1 (1 constraints: 371105d5)
+org.hamcrest:hamcrest-core:1.3 (2 constraints: 7910aeb0)
org.javassist:javassist:3.22.0-CR2 (1 constraints: 900dbf4d)
-org.jodd:jodd-core:3.5.2 (2 constraints: 0d1b0d94)
-org.json:json:20090211 (1 constraints: 890c4218)
+org.jodd:jodd-core:3.5.2 (1 constraints: 1f0d1037)
+org.jruby.jcodings:jcodings:1.0.8 (2 constraints: 9d15c301)
+org.jruby.joni:joni:2.1.2 (1 constraints: 8f0c160d)
org.json4s:json4s-ast_2.12:3.6.6 (1 constraints: 110baae9)
org.json4s:json4s-core_2.12:3.6.6 (1 constraints: 510c6316)
org.json4s:json4s-jackson_2.12:3.6.6 (1 constraints: 210de336)
org.json4s:json4s-scalap_2.12:3.6.6 (1 constraints: 110baae9)
-org.lz4:lz4-java:1.6.0 (1 constraints: 190dd336)
+org.lz4:lz4-java:1.7.0 (1 constraints: 1a0dd636)
org.mortbay.jetty:jetty:6.1.26 (5 constraints: cf45d2d7)
org.mortbay.jetty:jetty-sslengine:6.1.26 (1 constraints: c30d113d)
-org.mortbay.jetty:jetty-util:6.1.26 (7 constraints: 7e689dae)
+org.mortbay.jetty:jetty-util:6.1.26 (8 constraints: 507594c4)
org.mortbay.jetty:jsp-2.1:6.1.14 (1 constraints: 9408a38d)
org.mortbay.jetty:jsp-api-2.1:6.1.14 (2 constraints: 7e130c9d)
org.mortbay.jetty:servlet-api:2.5-20081211 (1 constraints: 390cbd19)
org.mortbay.jetty:servlet-api-2.5:6.1.14 (2 constraints: e51482f7)
org.objenesis:objenesis:2.5.1 (2 constraints: 19198bcb)
+org.ow2.asm:asm-all:5.0.2 (1 constraints: 0d0ceaf6)
org.roaringbitmap:RoaringBitmap:0.7.45 (1 constraints: 520d2744)
org.roaringbitmap:shims:0.7.45 (1 constraints: 260eb249)
org.scala-lang:scala-library:2.12.10 (12 constraints: f7aba36d)
@@ -224,35 +274,22 @@ org.scala-lang.modules:scala-parser-combinators_2.12:1.1.2 (1 constraints: d20e7
org.scala-lang.modules:scala-xml_2.12:1.2.0 (1 constraints: 150dc736)
org.slf4j:jcl-over-slf4j:1.7.16 (1 constraints: 510d2844)
org.slf4j:jul-to-slf4j:1.7.16 (1 constraints: 510d2844)
-org.slf4j:slf4j-api:1.7.25 (48 constraints: a9b2a1d1)
+org.slf4j:slf4j-api:1.7.28 (65 constraints: 37aed665)
org.sonatype.sisu.inject:cglib:2.2.1-v20090111 (1 constraints: aa0cfd36)
-org.spark-project.hive:hive-exec:1.2.1.spark2 (1 constraints: 9a0fb19c)
-org.spark-project.hive:hive-metastore:1.2.1.spark2 (1 constraints: 9a0fb19c)
org.spark-project.spark:unused:1.0.0 (12 constraints: a6ab49db)
org.tukaani:xz:1.5 (1 constraints: f008458a)
org.xerial.snappy:snappy-java:1.1.7.3 (3 constraints: 12262c3f)
oro:oro:2.0.8 (3 constraints: 3c22c237)
-stax:stax-api:1.0.1 (2 constraints: ea186edd)
-tomcat:jasper-compiler:5.5.12 (1 constraints: 9508ab8d)
-tomcat:jasper-runtime:5.5.12 (1 constraints: 9508ab8d)
+stax:stax-api:1.0.1 (1 constraints: 040bded6)
+tomcat:jasper-compiler:5.5.23 (2 constraints: 93169c60)
+tomcat:jasper-runtime:5.5.23 (2 constraints: 93169c60)
xerces:xercesImpl:2.9.1 (1 constraints: ac0ccc0f)
xml-apis:xml-apis:1.3.04 (1 constraints: b008af8c)
xmlenc:xmlenc:0.52 (3 constraints: 05228b2f)
[Test dependencies]
-junit:junit:4.12 (1 constraints: db04ff30)
-net.hydromatic:eigenbase-properties:1.1.5 (1 constraints: 5f0daf2c)
-org.apache.calcite:calcite-avatica:1.2.0-incubating (2 constraints: 01216c5c)
-org.apache.calcite:calcite-core:1.2.0-incubating (1 constraints: 560fda68)
-org.apache.calcite:calcite-linq4j:1.2.0-incubating (1 constraints: ac1147d8)
-org.apache.curator:apache-curator:2.6.0 (1 constraints: 0a0bf4d6)
org.apache.hadoop:hadoop-mapreduce-client-hs:2.7.4 (1 constraints: b70fad84)
org.apache.hadoop:hadoop-minicluster:2.7.4 (1 constraints: 0e050d36)
org.apache.hadoop:hadoop-yarn-server-tests:2.7.4 (1 constraints: b70fad84)
-org.apache.hive:hive-ant:1.2.1 (1 constraints: 060be4d6)
-org.apache.hive:hive-exec:1.2.1 (1 constraints: 0605f735)
-org.apache.velocity:velocity:1.5 (1 constraints: 480ae2b4)
-org.codehaus.groovy:groovy-all:2.1.6 (1 constraints: 0b0bebd6)
-org.hamcrest:hamcrest-core:1.3 (2 constraints: 7910aeb0)
org.mockito:mockito-core:1.10.19 (1 constraints: 6e059840)
org.slf4j:slf4j-simple:1.7.5 (1 constraints: 0f050a36)
diff --git a/versions.props b/versions.props
index eb1f6d7..5aec6ae 100644
--- a/versions.props
+++ b/versions.props
@@ -5,8 +5,8 @@ org.apache.hadoop:* = 2.7.3
org.apache.hive:hive-standalone-metastore = 1.2.1
org.apache.orc:orc-core = 1.5.6
org.apache.parquet:parquet-avro = 1.10.1
-org.apache.spark:spark-hive_2.12 = 3.0.0-preview
-org.apache.spark:spark-avro_2.12 = 3.0.0-preview
+org.apache.spark:spark-hive_2.12 = 3.0.0-preview2
+org.apache.spark:spark-avro_2.12 = 3.0.0-preview2
org.apache.pig:pig = 0.14.0
org.apache.commons:commons-lang3 = 3.9
com.fasterxml.jackson.*:* = 2.10.0