You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2020/05/28 00:18:43 UTC
[hudi] branch release-0.5.3 created (now 54bbe32)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a change to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git.
at 54bbe32 [HUDI-938] Removing incubating/incubator from project (#1658)
This branch includes the following new commits:
new f6c87b5 Moving to 0.5.3-SNAPSHOT for bug-fix release over 0.5.2
new dd7952e [HUDI-652] Decouple HoodieReadClient and AbstractHoodieClient to break the inheritance chain (#1372)
new 1d16b15 [HUDI-681]Remove embeddedTimelineService from HoodieReadClient (#1388)
new cebce61 [HUDI-629]: Replace Guava's Hashing with an equivalent in NumericUtils.java (#1350)
new 2112d0a [HUDI - 738] Add validation to DeltaStreamer to fail fast when filterDupes is enabled on UPSERT mode. (#1505)
new e3f7659 [HUDI-799] Use appropriate FS when loading configs (#1517)
new 18453a4 [HUDI-713] Fix conversion of Spark array of struct type to Avro schema (#1406)
new 9a2b3e3 [HUDI-656][Performance] Return a dummy Spark relation after writing the DataFrame (#1394)
new ea48ca9 [HUDI-850] Avoid unnecessary listings in incremental cleaning mode (#1576)
new 96b2359 [HUDI-724] Parallelize getSmallFiles for partitions (#1421)
new 0050e00 [HUDI-607] Fix to allow creation/syncing of Hive tables partitioned by Date type columns (#1330)
new d535381 Add constructor to HoodieROTablePathFilter (#1413)
new bd70601 [HUDI-539] Make ROPathFilter conf member serializable (#1415)
new d476ac5 Add changes for presto mor queries (#1578)
new cc4aa45 [HUDI-782] Add support of Aliyun object storage service. (#1506)
new 86698b5 [HUDI-716] Exception: Not an Avro data file when running HoodieCleanClient.runClean (#1432)
new 2e5f601 [HUDI-400] Check upgrade from old plan to new plan for compaction (#1422)
new 45efa69 [MINOR] Update DOAP with 0.5.2 Release (#1448)
new 2f4ee0a [HUDI-742] Fix Java Math Exception (#1466)
new e80dd04 [HUDI-717] Fixed usage of HiveDriver for DDL statements. (#1416)
new f9c3b07 [HUDI-727]: Copy default values of fields if not present when rewriting incoming record with new schema (#1427)
new 500b390 [HUDI-795] Handle auto-deleted empty aux folder (#1515)
new 0dcc204 [MINOR]: Fix cli docs for DeltaStreamer (#1547)
new 288bc4d [HUDI-852] adding check for table name for Append Save mode (#1580)
new be205ed [MINOR] fixed building IndexFileFilter with a wrong condition in HoodieGlobalBloomIndex class (#1537)
new 77d7500 [HUDI-616] Fixed parquet files getting created on local FS (#1434)
new 5fbb5a7 [HUDI-902] Avoid exception when getSchemaProvider (#1584)
new 8324f6e [HUDI-789]Adjust logic of upsert in HDFSParquetImporter (#1511)
new da043af [HUDI-889] Writer supports useJdbc configuration when hive synchronization is enabled (#1627)
new 4095de4 [HUDI-820] cleaner repair command should only inspect clean metadata files (#1542)
new 7c766c3 HUDI-528 Handle empty commit in incremental pulling (#1612)
new 0fb02e7 [HUDI-895] Remove unnecessary listing .hoodie folder when using timeline server (#1636)
new 555ff1a [HUDI-858] Allow multiple operations to be executed within a single commit (#1633)
new 4192cc6 [HUDI-863] get decimal properties from derived spark DataType (#1596)
new b69bb18 [HUDI-846][HUDI-848] Enable Incremental cleaning and embedded timeline-server by default (#1634)
new 07345ba Fixing test failure in TestHoodieClientOnCopyOnWriteStorage
new 0a77a3b Fixing test failures for TestRepairsCommand
new 498c86b [MINOR] Remove incubating from README
new 3a776f6 [HUDI-926] Removing DISCLAIMER from the repo (#1657)
new 54bbe32 [HUDI-938] Removing incubating/incubator from project (#1658)
The 40 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[hudi] 35/40: [HUDI-846][HUDI-848] Enable Incremental cleaning and
embedded timeline-server by default (#1634)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b69bb18fdd3d5ab825d1f59024b10204890968ff
Author: Balaji Varadarajan <va...@uber.com>
AuthorDate: Wed May 20 05:29:43 2020 -0700
[HUDI-846][HUDI-848] Enable Incremental cleaning and embedded timeline-server by default (#1634)
---
.../apache/hudi/config/HoodieCompactionConfig.java | 2 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 2 +-
.../hudi/table/compact/TestHoodieCompactor.java | 10 ++++-
hudi-hive/pom.xml | 6 ---
.../org/apache/hudi/hive/util/HiveTestService.java | 1 +
hudi-spark/pom.xml | 44 +++++++++++++++++++++-
hudi-utilities/pom.xml | 7 +---
pom.xml | 9 +----
8 files changed, 57 insertions(+), 24 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
index 074ea78..5fa2b16 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
@@ -95,7 +95,7 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig {
private static final String DEFAULT_CLEANER_POLICY = HoodieCleaningPolicy.KEEP_LATEST_COMMITS.name();
private static final String DEFAULT_AUTO_CLEAN = "true";
private static final String DEFAULT_INLINE_COMPACT = "false";
- private static final String DEFAULT_INCREMENTAL_CLEANER = "false";
+ private static final String DEFAULT_INCREMENTAL_CLEANER = "true";
private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = "1";
private static final String DEFAULT_CLEANER_FILE_VERSIONS_RETAINED = "3";
private static final String DEFAULT_CLEANER_COMMITS_RETAINED = "10";
diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 24984db..47c109a 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -79,7 +79,7 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
private static final String DEFAULT_FINALIZE_WRITE_PARALLELISM = DEFAULT_PARALLELISM;
private static final String EMBEDDED_TIMELINE_SERVER_ENABLED = "hoodie.embed.timeline.server";
- private static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_ENABLED = "false";
+ private static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_ENABLED = "true";
private static final String FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP = "hoodie.fail.on.timeline.archiving";
private static final String DEFAULT_FAIL_ON_TIMELINE_ARCHIVING_ENABLED = "true";
diff --git a/hudi-client/src/test/java/org/apache/hudi/table/compact/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/table/compact/TestHoodieCompactor.java
index 482d721..8fa55ec 100644
--- a/hudi-client/src/test/java/org/apache/hudi/table/compact/TestHoodieCompactor.java
+++ b/hudi-client/src/test/java/org/apache/hudi/table/compact/TestHoodieCompactor.java
@@ -27,8 +27,12 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieInstant.State;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieMemoryConfig;
@@ -148,9 +152,13 @@ public class TestHoodieCompactor extends HoodieClientTestHarness {
HoodieIndex index = new HoodieBloomIndex<>(config);
updatedRecords = index.tagLocation(updatedRecordsRDD, jsc, table).collect();
- // Write them to corresponding avro logfiles
+ // Write them to corresponding avro logfiles. Also, set the state transition properly.
HoodieTestUtils.writeRecordsToLogFiles(fs, metaClient.getBasePath(),
HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS, updatedRecords);
+ metaClient.getActiveTimeline().transitionRequestedToInflight(new HoodieInstant(State.REQUESTED,
+ HoodieTimeline.DELTA_COMMIT_ACTION, newCommitTime), Option.empty());
+ writeClient.commit(newCommitTime, jsc.emptyRDD(), Option.empty());
+ metaClient.reloadActiveTimeline();
// Verify that all data file has one log file
metaClient = HoodieTableMetaClient.reload(metaClient);
diff --git a/hudi-hive/pom.xml b/hudi-hive/pom.xml
index 01e6ff1..5ad8708 100644
--- a/hudi-hive/pom.xml
+++ b/hudi-hive/pom.xml
@@ -153,12 +153,6 @@
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- <scope>test</scope>
- </dependency>
-
<!-- Hadoop - Test -->
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
index 0cef82b..ee5b09a 100644
--- a/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
+++ b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
@@ -105,6 +105,7 @@ public class HiveTestService {
executorService = Executors.newSingleThreadExecutor();
tServer = startMetaStore(bindIP, metastorePort, serverConf);
+ serverConf.set("hive.in.test", "true");
hiveServer = startHiveServer(serverConf);
String serverHostname;
diff --git a/hudi-spark/pom.xml b/hudi-spark/pom.xml
index 501cd8f..6b90a69 100644
--- a/hudi-spark/pom.xml
+++ b/hudi-spark/pom.xml
@@ -220,6 +220,12 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -248,7 +254,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
</dependency>
<!-- Hive -->
@@ -272,16 +288,42 @@
<groupId>${hive.groupid}</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-common</artifactId>
<version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Hoodie - Test -->
diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml
index b8b71ac..6211311 100644
--- a/hudi-utilities/pom.xml
+++ b/hudi-utilities/pom.xml
@@ -82,13 +82,8 @@
<!-- Needs to be at the top to ensure we get the correct dependency versions for jetty-server -->
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>jetty-all</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
+ <classifier>uber</classifier>
<scope>test</scope>
</dependency>
diff --git a/pom.xml b/pom.xml
index b49c653..8603024 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
<scalatest.version>3.0.1</scalatest.version>
<surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire.properties</surefire-log4j.file>
<thrift.version>0.12.0</thrift.version>
- <jetty.version>7.6.0.v20120127</jetty.version>
+ <jetty.version>9.4.15.v20190215</jetty.version>
<hbase.version>1.2.3</hbase.version>
<codehaus-jackson.version>1.9.13</codehaus-jackson.version>
<h2.version>1.4.199</h2.version>
@@ -456,13 +456,6 @@
<artifactId>jersey-container-servlet-core</artifactId>
<version>${glassfish.version}</version>
</dependency>
- <!-- Needed for running HiveServer for Tests -->
- <dependency>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- <scope>test</scope>
- <version>${jetty.version}</version>
- </dependency>
<!-- Avro -->
<dependency>
[hudi] 13/40: [HUDI-539] Make ROPathFilter conf member serializable
(#1415)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit bd7060186f7165f01aa8508c3fd8c3c81b012a2f
Author: vinoth chandar <vi...@users.noreply.github.com>
AuthorDate: Tue Mar 17 12:52:48 2020 -0700
[HUDI-539] Make ROPathFilter conf member serializable (#1415)
---
.../main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
index d879a2f..eb02a57 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
@@ -18,6 +18,7 @@
package org.apache.hudi.hadoop;
+import org.apache.hudi.common.SerializableConfiguration;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -66,7 +67,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
/**
* Hadoop configurations for the FileSystem.
*/
- private Configuration conf;
+ private SerializableConfiguration conf;
private transient FileSystem fs;
@@ -77,7 +78,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
public HoodieROTablePathFilter(Configuration conf) {
this.hoodiePathCache = new HashMap<>();
this.nonHoodiePathCache = new HashSet<>();
- this.conf = conf;
+ this.conf = new SerializableConfiguration(conf);
}
/**
@@ -102,7 +103,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
Path folder = null;
try {
if (fs == null) {
- fs = path.getFileSystem(conf);
+ fs = path.getFileSystem(conf.get());
}
// Assumes path is a file
[hudi] 37/40: Fixing test failures for TestRepairsCommand
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 0a77a3bbaba64d9b5930429d465b5f4071d494f8
Author: Sivabalan Narayanan <si...@uber.com>
AuthorDate: Mon May 25 09:54:02 2020 -0400
Fixing test failures for TestRepairsCommand
---
.../apache/hudi/cli/commands/RepairsCommand.java | 6 +++---
.../hudi/cli/commands/TestRepairsCommand.java | 2 +-
.../src/test/resources/log4j-surefire.properties | 25 ++++++++++++++++++++++
.../src/test/resources/table-config.properties | 21 ++++++++++++++++++
4 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 7feebed..d1d9e5e 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -108,6 +108,7 @@ public class RepairsCommand implements CommandMarker {
HoodiePartitionMetadata partitionMetadata =
new HoodiePartitionMetadata(HoodieCLI.fs, latestCommit, basePath, partitionPath);
partitionMetadata.trySave(0);
+ row[2] = "Repaired";
}
}
rows[ind++] = row;
@@ -118,8 +119,7 @@ public class RepairsCommand implements CommandMarker {
@CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!")
public String overwriteHoodieProperties(
- @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with")
- final String overwriteFilePath) throws IOException {
+ @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with") final String overwriteFilePath) throws IOException {
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
Properties newProps = new Properties();
@@ -135,7 +135,7 @@ public class RepairsCommand implements CommandMarker {
String[][] rows = new String[allPropKeys.size()][];
int ind = 0;
for (String propKey : allPropKeys) {
- String[] row = new String[]{
+ String[] row = new String[] {
propKey,
oldProps.getOrDefault(propKey, "null"),
newProps.getOrDefault(propKey, "null").toString()
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
index 1192915..9e78ac7 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
@@ -188,7 +188,7 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest {
for (int i = 100; i < 104; i++) {
String timestamp = String.valueOf(i);
// Write corrupted requested Compaction
- HoodieTestCommitMetadataGenerator.createCompactionRequestedFile(tablePath, timestamp, conf);
+ HoodieTestCommitMetadataGenerator.createEmptyCleanRequestedFile(tablePath, timestamp, conf);
}
// reload meta client
diff --git a/hudi-cli/src/test/resources/log4j-surefire.properties b/hudi-cli/src/test/resources/log4j-surefire.properties
new file mode 100644
index 0000000..11d2f28
--- /dev/null
+++ b/hudi-cli/src/test/resources/log4j-surefire.properties
@@ -0,0 +1,25 @@
+###
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+log4j.rootLogger=WARN, A1
+log4j.category.org.apache=INFO
+log4j.category.org.apache.parquet.hadoop=WARN
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
diff --git a/hudi-cli/src/test/resources/table-config.properties b/hudi-cli/src/test/resources/table-config.properties
new file mode 100644
index 0000000..78170f9
--- /dev/null
+++ b/hudi-cli/src/test/resources/table-config.properties
@@ -0,0 +1,21 @@
+###
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+hoodie.table.name=test_table
+hoodie.table.type=COPY_ON_WRITE
+hoodie.archivelog.folder=archive
+hoodie.timeline.layout.version=1
\ No newline at end of file
[hudi] 23/40: [MINOR]: Fix cli docs for DeltaStreamer (#1547)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 0dcc20458d36fdb6d052a62ca52a77fa0c31745f
Author: dengziming <de...@gmail.com>
AuthorDate: Thu Apr 23 02:37:17 2020 +0800
[MINOR]: Fix cli docs for DeltaStreamer (#1547)
---
.../org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
index 3098bf3..5250a33 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
@@ -169,7 +169,7 @@ public class HoodieDeltaStreamer implements Serializable {
"file://" + System.getProperty("user.dir") + "/src/test/resources/delta-streamer-config/dfs-source.properties";
@Parameter(names = {"--hoodie-conf"}, description = "Any configuration that can be set in the properties file "
- + "(using the CLI parameter \"--propsFilePath\") can also be passed command line using this parameter")
+ + "(using the CLI parameter \"--props\") can also be passed command line using this parameter")
public List<String> configs = new ArrayList<>();
@Parameter(names = {"--source-class"},
[hudi] 32/40: [HUDI-895] Remove unnecessary listing .hoodie folder
when using timeline server (#1636)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 0fb02e7ad1111f1b84d48c8983e9a4cd7eb8b91e
Author: Balaji Varadarajan <va...@uber.com>
AuthorDate: Sun May 17 18:18:53 2020 -0700
[HUDI-895] Remove unnecessary listing .hoodie folder when using timeline server (#1636)
---
.../org/apache/hudi/client/HoodieWriteClient.java | 6 +--
.../java/org/apache/hudi/table/HoodieTable.java | 6 +--
.../common/table/view/FileSystemViewManager.java | 59 +++++++++++++---------
3 files changed, 40 insertions(+), 31 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
index 90bc9b3..ba39b9b 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
@@ -872,10 +872,8 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
HoodieTimeline.compareTimestamps(latestPending.getTimestamp(), instantTime, HoodieTimeline.LESSER),
"Latest pending compaction instant time must be earlier than this instant time. Latest Compaction :"
+ latestPending + ", Ingesting at " + instantTime));
- HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
- HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
- String commitActionType = table.getMetaClient().getCommitActionType();
- activeTimeline.createNewInstant(new HoodieInstant(State.REQUESTED, commitActionType, instantTime));
+ metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(State.REQUESTED, metaClient.getCommitActionType(),
+ instantTime));
}
/**
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
index ad0196c..2f106c0 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -150,21 +150,21 @@ public abstract class HoodieTable<T extends HoodieRecordPayload> implements Seri
* Get the base file only view of the file system for this table.
*/
public BaseFileOnlyView getBaseFileOnlyView() {
- return getViewManager().getFileSystemView(metaClient.getBasePath());
+ return getViewManager().getFileSystemView(metaClient);
}
/**
* Get the full view of the file system for this table.
*/
public SliceView getSliceView() {
- return getViewManager().getFileSystemView(metaClient.getBasePath());
+ return getViewManager().getFileSystemView(metaClient);
}
/**
* Get complete view of the file system for this table with ability to force sync.
*/
public SyncableFileSystemView getHoodieView() {
- return getViewManager().getFileSystemView(metaClient.getBasePath());
+ return getViewManager().getFileSystemView(metaClient);
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index 8d50ef5..ffeafaa 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -59,10 +59,10 @@ public class FileSystemViewManager {
// Map from Base-Path to View
private final ConcurrentHashMap<String, SyncableFileSystemView> globalViewMap;
// Factory Map to create file-system views
- private final Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator;
+ private final Function2<HoodieTableMetaClient, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator;
public FileSystemViewManager(SerializableConfiguration conf, FileSystemViewStorageConfig viewStorageConfig,
- Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) {
+ Function2<HoodieTableMetaClient, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) {
this.conf = new SerializableConfiguration(conf);
this.viewStorageConfig = viewStorageConfig;
this.globalViewMap = new ConcurrentHashMap<>();
@@ -88,7 +88,21 @@ public class FileSystemViewManager {
* @return
*/
public SyncableFileSystemView getFileSystemView(String basePath) {
- return globalViewMap.computeIfAbsent(basePath, (path) -> viewCreator.apply(path, viewStorageConfig));
+ return globalViewMap.computeIfAbsent(basePath, (path) -> {
+ HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), path);
+ return viewCreator.apply(metaClient, viewStorageConfig);
+ });
+ }
+
+ /**
+ * Main API to get the file-system view for the base-path.
+ *
+ * @param metaClient HoodieTableMetaClient
+ * @return
+ */
+ public SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient) {
+ return globalViewMap.computeIfAbsent(metaClient.getBasePath(),
+ (path) -> viewCreator.apply(metaClient, viewStorageConfig));
}
/**
@@ -106,12 +120,11 @@ public class FileSystemViewManager {
*
* @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
- * @param basePath Base Path of table
+ * @param metaClient HoodieTableMetaClient
* @return
*/
private static RocksDbBasedFileSystemView createRocksDBBasedFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, String basePath) {
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), basePath, true);
+ FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new RocksDbBasedFileSystemView(metaClient, timeline, viewConf);
}
@@ -121,13 +134,12 @@ public class FileSystemViewManager {
*
* @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
- * @param basePath Base Path of table
+ * @param metaClient HoodieTableMetaClient
* @return
*/
private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, String basePath) {
- LOG.info("Creating SpillableMap based view for basePath " + basePath);
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), basePath, true);
+ FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
+ LOG.info("Creating SpillableMap based view for basePath " + metaClient.getBasePath());
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf);
}
@@ -137,13 +149,12 @@ public class FileSystemViewManager {
*
* @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
- * @param basePath Base Path of table
+ * @param metaClient HoodieTableMetaClient
* @return
*/
private static HoodieTableFileSystemView createInMemoryFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, String basePath) {
- LOG.info("Creating InMemory based view for basePath " + basePath);
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), basePath, true);
+ FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
+ LOG.info("Creating InMemory based view for basePath " + metaClient.getBasePath());
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new HoodieTableFileSystemView(metaClient, timeline, viewConf.isIncrementalTimelineSyncEnabled());
}
@@ -178,34 +189,34 @@ public class FileSystemViewManager {
case EMBEDDED_KV_STORE:
LOG.info("Creating embedded rocks-db based Table View");
return new FileSystemViewManager(conf, config,
- (basePath, viewConf) -> createRocksDBBasedFileSystemView(conf, viewConf, basePath));
+ (metaClient, viewConf) -> createRocksDBBasedFileSystemView(conf, viewConf, metaClient));
case SPILLABLE_DISK:
LOG.info("Creating Spillable Disk based Table View");
return new FileSystemViewManager(conf, config,
- (basePath, viewConf) -> createSpillableMapBasedFileSystemView(conf, viewConf, basePath));
+ (metaClient, viewConf) -> createSpillableMapBasedFileSystemView(conf, viewConf, metaClient));
case MEMORY:
LOG.info("Creating in-memory based Table View");
return new FileSystemViewManager(conf, config,
- (basePath, viewConfig) -> createInMemoryFileSystemView(conf, viewConfig, basePath));
+ (metaClient, viewConfig) -> createInMemoryFileSystemView(conf, viewConfig, metaClient));
case REMOTE_ONLY:
LOG.info("Creating remote only table view");
- return new FileSystemViewManager(conf, config, (basePath, viewConfig) -> createRemoteFileSystemView(conf,
- viewConfig, new HoodieTableMetaClient(conf.newCopy(), basePath)));
+ return new FileSystemViewManager(conf, config, (metaClient, viewConfig) -> createRemoteFileSystemView(conf,
+ viewConfig, metaClient));
case REMOTE_FIRST:
LOG.info("Creating remote first table view");
- return new FileSystemViewManager(conf, config, (basePath, viewConfig) -> {
+ return new FileSystemViewManager(conf, config, (metaClient, viewConfig) -> {
RemoteHoodieTableFileSystemView remoteFileSystemView =
- createRemoteFileSystemView(conf, viewConfig, new HoodieTableMetaClient(conf.newCopy(), basePath));
+ createRemoteFileSystemView(conf, viewConfig, metaClient);
SyncableFileSystemView secondaryView;
switch (viewConfig.getSecondaryStorageType()) {
case MEMORY:
- secondaryView = createInMemoryFileSystemView(conf, viewConfig, basePath);
+ secondaryView = createInMemoryFileSystemView(conf, viewConfig, metaClient);
break;
case EMBEDDED_KV_STORE:
- secondaryView = createRocksDBBasedFileSystemView(conf, viewConfig, basePath);
+ secondaryView = createRocksDBBasedFileSystemView(conf, viewConfig, metaClient);
break;
case SPILLABLE_DISK:
- secondaryView = createSpillableMapBasedFileSystemView(conf, viewConfig, basePath);
+ secondaryView = createSpillableMapBasedFileSystemView(conf, viewConfig, metaClient);
break;
default:
throw new IllegalArgumentException("Secondary Storage type can only be in-memory or spillable. Was :"
[hudi] 06/40: [HUDI-799] Use appropriate FS when loading configs
(#1517)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit e3f7659b3470a16f2d46ab75df576a7f00b432cd
Author: Alexander Filipchik <af...@gmail.com>
AuthorDate: Thu Apr 16 13:49:39 2020 -0700
[HUDI-799] Use appropriate FS when loading configs (#1517)
Co-authored-by: Alex Filipchik <al...@csscompany.com>
---
.../org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
index 60947be..3098bf3 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
@@ -362,7 +362,9 @@ public class HoodieDeltaStreamer implements Serializable {
ValidationUtils.checkArgument(!cfg.filterDupes || cfg.operation != Operation.UPSERT,
"'--filter-dupes' needs to be disabled when '--op' is 'UPSERT' to ensure updates are not missed.");
- this.props = UtilHelpers.readConfig(fs, new Path(cfg.propsFilePath), cfg.configs).getConfig();
+ this.props = UtilHelpers.readConfig(
+ FSUtils.getFs(cfg.propsFilePath, jssc.hadoopConfiguration()),
+ new Path(cfg.propsFilePath), cfg.configs).getConfig();
LOG.info("Creating delta streamer with configs : " + props.toString());
this.schemaProvider = UtilHelpers.createSchemaProvider(cfg.schemaProviderClassName, props, jssc);
[hudi] 34/40: [HUDI-863] get decimal properties from derived spark
DataType (#1596)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 4192cc65d02640f53db78922a1783103e7a46cfb
Author: rolandjohann <rm...@gmail.com>
AuthorDate: Mon May 18 13:28:27 2020 +0200
[HUDI-863] get decimal properties from derived spark DataType (#1596)
---
.../org/apache/hudi/AvroConversionHelper.scala | 22 ++++++++++------------
.../org/apache/hudi/AvroConversionUtils.scala | 4 +---
2 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala
index 43225bc..69e6376 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala
@@ -268,8 +268,7 @@ object AvroConversionHelper {
createConverter(sourceAvroSchema, targetSqlType, List.empty[String])
}
- def createConverterToAvro(avroSchema: Schema,
- dataType: DataType,
+ def createConverterToAvro(dataType: DataType,
structName: String,
recordNamespace: String): Any => Any = {
dataType match {
@@ -284,13 +283,15 @@ object AvroConversionHelper {
if (item == null) null else item.asInstanceOf[Byte].intValue
case ShortType => (item: Any) =>
if (item == null) null else item.asInstanceOf[Short].intValue
- case dec: DecimalType => (item: Any) =>
- Option(item).map { _ =>
- val bigDecimalValue = item.asInstanceOf[java.math.BigDecimal]
- val decimalConversions = new DecimalConversion()
- decimalConversions.toFixed(bigDecimalValue, avroSchema.getField(structName).schema().getTypes.get(0),
- LogicalTypes.decimal(dec.precision, dec.scale))
- }.orNull
+ case dec: DecimalType =>
+ val schema = SchemaConverters.toAvroType(dec, nullable = false, structName, recordNamespace)
+ (item: Any) => {
+ Option(item).map { _ =>
+ val bigDecimalValue = item.asInstanceOf[java.math.BigDecimal]
+ val decimalConversions = new DecimalConversion()
+ decimalConversions.toFixed(bigDecimalValue, schema, LogicalTypes.decimal(dec.precision, dec.scale))
+ }.orNull
+ }
case TimestampType => (item: Any) =>
// Convert time to microseconds since spark-avro by default converts TimestampType to
// Avro Logical TimestampMicros
@@ -299,7 +300,6 @@ object AvroConversionHelper {
Option(item).map(_.asInstanceOf[Date].toLocalDate.toEpochDay.toInt).orNull
case ArrayType(elementType, _) =>
val elementConverter = createConverterToAvro(
- avroSchema,
elementType,
structName,
recordNamespace)
@@ -320,7 +320,6 @@ object AvroConversionHelper {
}
case MapType(StringType, valueType, _) =>
val valueConverter = createConverterToAvro(
- avroSchema,
valueType,
structName,
recordNamespace)
@@ -340,7 +339,6 @@ object AvroConversionHelper {
val childNameSpace = if (recordNamespace != "") s"$recordNamespace.$structName" else structName
val fieldConverters = structType.fields.map(field =>
createConverterToAvro(
- avroSchema,
field.dataType,
field.name,
childNameSpace))
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala
index 04de1c7..bdb8955 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala
@@ -38,14 +38,12 @@ object AvroConversionUtils {
: RDD[GenericRecord] = {
// Use the Avro schema to derive the StructType which has the correct nullability information
val dataType = SchemaConverters.toSqlType(avroSchema).dataType.asInstanceOf[StructType]
- val avroSchemaAsJsonString = avroSchema.toString
val encoder = RowEncoder.apply(dataType).resolveAndBind()
df.queryExecution.toRdd.map(encoder.fromRow)
.mapPartitions { records =>
if (records.isEmpty) Iterator.empty
else {
- val avroSchema = new Schema.Parser().parse(avroSchemaAsJsonString)
- val convertor = AvroConversionHelper.createConverterToAvro(avroSchema, dataType, structName, recordNamespace)
+ val convertor = AvroConversionHelper.createConverterToAvro(dataType, structName, recordNamespace)
records.map { x => convertor(x).asInstanceOf[GenericRecord] }
}
}
[hudi] 10/40: [HUDI-724] Parallelize getSmallFiles for partitions
(#1421)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 96b2359fda4d5b55e51e8f347228fe2f7cf6b24f
Author: ffcchi <fe...@gmail.com>
AuthorDate: Mon Mar 30 01:14:38 2020 -0600
[HUDI-724] Parallelize getSmallFiles for partitions (#1421)
Co-authored-by: Feichi Feng <fe...@amazon.com>
---
.../org/apache/hudi/client/HoodieWriteClient.java | 4 +--
.../apache/hudi/table/HoodieCopyOnWriteTable.java | 37 ++++++++++++++++------
.../apache/hudi/table/HoodieMergeOnReadTable.java | 12 +++----
.../java/org/apache/hudi/table/HoodieTable.java | 4 +--
.../apache/hudi/table/TestCopyOnWriteTable.java | 2 +-
5 files changed, 36 insertions(+), 23 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
index e201487..90bc9b3 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
@@ -480,9 +480,9 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
private Partitioner getPartitioner(HoodieTable table, boolean isUpsert, WorkloadProfile profile) {
if (isUpsert) {
- return table.getUpsertPartitioner(profile);
+ return table.getUpsertPartitioner(profile, jsc);
} else {
- return table.getInsertPartitioner(profile);
+ return table.getInsertPartitioner(profile, jsc);
}
}
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
index 82b08b7..4c91c77 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
@@ -81,6 +81,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
/**
@@ -142,16 +143,16 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
}
@Override
- public Partitioner getUpsertPartitioner(WorkloadProfile profile) {
+ public Partitioner getUpsertPartitioner(WorkloadProfile profile, JavaSparkContext jsc) {
if (profile == null) {
throw new HoodieUpsertException("Need workload profile to construct the upsert partitioner.");
}
- return new UpsertPartitioner(profile);
+ return new UpsertPartitioner(profile, jsc);
}
@Override
- public Partitioner getInsertPartitioner(WorkloadProfile profile) {
- return getUpsertPartitioner(profile);
+ public Partitioner getInsertPartitioner(WorkloadProfile profile, JavaSparkContext jsc) {
+ return getUpsertPartitioner(profile, jsc);
}
@Override
@@ -569,14 +570,14 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
*/
protected HoodieRollingStatMetadata rollingStatMetadata;
- UpsertPartitioner(WorkloadProfile profile) {
+ UpsertPartitioner(WorkloadProfile profile, JavaSparkContext jsc) {
updateLocationToBucket = new HashMap<>();
partitionPathToInsertBuckets = new HashMap<>();
bucketInfoMap = new HashMap<>();
globalStat = profile.getGlobalStat();
rollingStatMetadata = getRollingStats();
assignUpdates(profile);
- assignInserts(profile);
+ assignInserts(profile, jsc);
LOG.info("Total Buckets :" + totalBuckets + ", buckets info => " + bucketInfoMap + ", \n"
+ "Partition to insert buckets => " + partitionPathToInsertBuckets + ", \n"
@@ -602,18 +603,24 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
return bucket;
}
- private void assignInserts(WorkloadProfile profile) {
+ private void assignInserts(WorkloadProfile profile, JavaSparkContext jsc) {
// for new inserts, compute buckets depending on how many records we have for each partition
Set<String> partitionPaths = profile.getPartitionPaths();
long averageRecordSize =
averageBytesPerRecord(metaClient.getActiveTimeline().getCommitTimeline().filterCompletedInstants(),
config.getCopyOnWriteRecordSizeEstimate());
LOG.info("AvgRecordSize => " + averageRecordSize);
+
+ Map<String, List<SmallFile>> partitionSmallFilesMap =
+ getSmallFilesForPartitions(new ArrayList<String>(partitionPaths), jsc);
+
for (String partitionPath : partitionPaths) {
WorkloadStat pStat = profile.getWorkloadStat(partitionPath);
if (pStat.getNumInserts() > 0) {
- List<SmallFile> smallFiles = getSmallFiles(partitionPath);
+ List<SmallFile> smallFiles = partitionSmallFilesMap.get(partitionPath);
+ this.smallFiles.addAll(smallFiles);
+
LOG.info("For partitionPath : " + partitionPath + " Small Files => " + smallFiles);
long totalUnassignedInserts = pStat.getNumInserts();
@@ -675,6 +682,18 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
}
}
+ private Map<String, List<SmallFile>> getSmallFilesForPartitions(List<String> partitionPaths, JavaSparkContext jsc) {
+
+ Map<String, List<SmallFile>> partitionSmallFilesMap = new HashMap<>();
+ if (partitionPaths != null && partitionPaths.size() > 0) {
+ JavaRDD<String> partitionPathRdds = jsc.parallelize(partitionPaths, partitionPaths.size());
+ partitionSmallFilesMap = partitionPathRdds.mapToPair((PairFunction<String, String, List<SmallFile>>)
+ partitionPath -> new Tuple2<>(partitionPath, getSmallFiles(partitionPath))).collectAsMap();
+ }
+
+ return partitionSmallFilesMap;
+ }
+
/**
* Returns a list of small files in the given partition path.
*/
@@ -697,8 +716,6 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
sf.location = new HoodieRecordLocation(FSUtils.getCommitTime(filename), FSUtils.getFileId(filename));
sf.sizeBytes = file.getFileSize();
smallFileLocations.add(sf);
- // Update the global small files list
- smallFiles.add(sf);
}
}
}
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java
index 50d41b3..938a5fd 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java
@@ -89,11 +89,11 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends Hoodi
}
@Override
- public Partitioner getUpsertPartitioner(WorkloadProfile profile) {
+ public Partitioner getUpsertPartitioner(WorkloadProfile profile, JavaSparkContext jsc) {
if (profile == null) {
throw new HoodieUpsertException("Need workload profile to construct the upsert partitioner.");
}
- mergeOnReadUpsertPartitioner = new MergeOnReadUpsertPartitioner(profile);
+ mergeOnReadUpsertPartitioner = new MergeOnReadUpsertPartitioner(profile, jsc);
return mergeOnReadUpsertPartitioner;
}
@@ -323,8 +323,8 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends Hoodi
*/
class MergeOnReadUpsertPartitioner extends HoodieCopyOnWriteTable.UpsertPartitioner {
- MergeOnReadUpsertPartitioner(WorkloadProfile profile) {
- super(profile);
+ MergeOnReadUpsertPartitioner(WorkloadProfile profile, JavaSparkContext jsc) {
+ super(profile, jsc);
}
@Override
@@ -374,16 +374,12 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends Hoodi
sf.location = new HoodieRecordLocation(FSUtils.getCommitTime(filename), FSUtils.getFileId(filename));
sf.sizeBytes = getTotalFileSize(smallFileSlice);
smallFileLocations.add(sf);
- // Update the global small files list
- smallFiles.add(sf);
} else {
HoodieLogFile logFile = smallFileSlice.getLogFiles().findFirst().get();
sf.location = new HoodieRecordLocation(FSUtils.getBaseCommitTimeFromLogPath(logFile.getPath()),
FSUtils.getFileIdFromLogPath(logFile.getPath()));
sf.sizeBytes = getTotalFileSize(smallFileSlice);
smallFileLocations.add(sf);
- // Update the global small files list
- smallFiles.add(sf);
}
}
}
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
index 2e73ef0..ad0196c 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -115,12 +115,12 @@ public abstract class HoodieTable<T extends HoodieRecordPayload> implements Seri
/**
* Provides a partitioner to perform the upsert operation, based on the workload profile.
*/
- public abstract Partitioner getUpsertPartitioner(WorkloadProfile profile);
+ public abstract Partitioner getUpsertPartitioner(WorkloadProfile profile, JavaSparkContext jsc);
/**
* Provides a partitioner to perform the insert operation, based on the workload profile.
*/
- public abstract Partitioner getInsertPartitioner(WorkloadProfile profile);
+ public abstract Partitioner getInsertPartitioner(WorkloadProfile profile, JavaSparkContext jsc);
/**
* Return whether this HoodieTable implementation can benefit from workload profiling.
diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java
index 95248a4..ec64080 100644
--- a/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java
+++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java
@@ -415,7 +415,7 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
records.addAll(updateRecords);
WorkloadProfile profile = new WorkloadProfile(jsc.parallelize(records));
HoodieCopyOnWriteTable.UpsertPartitioner partitioner =
- (HoodieCopyOnWriteTable.UpsertPartitioner) table.getUpsertPartitioner(profile);
+ (HoodieCopyOnWriteTable.UpsertPartitioner) table.getUpsertPartitioner(profile, jsc);
assertEquals("Update record should have gone to the 1 update partition", 0, partitioner.getPartition(
new Tuple2<>(updateRecords.get(0).getKey(), Option.ofNullable(updateRecords.get(0).getCurrentLocation()))));
return partitioner;
[hudi] 11/40: [HUDI-607] Fix to allow creation/syncing of Hive
tables partitioned by Date type columns (#1330)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 0050e00afdc728793f9928cbb594e657c7219c41
Author: Udit Mehrotra <um...@illinois.edu>
AuthorDate: Sun Mar 1 10:42:58 2020 -0800
[HUDI-607] Fix to allow creation/syncing of Hive tables partitioned by Date type columns (#1330)
---
.../main/java/org/apache/hudi/DataSourceUtils.java | 40 +++++++++++++-
hudi-spark/src/test/java/DataSourceUtilsTest.java | 61 ++++++++++++++++++++++
2 files changed, 100 insertions(+), 1 deletion(-)
diff --git a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
index 475a925..99a795d 100644
--- a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
+++ b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
@@ -18,6 +18,8 @@
package org.apache.hudi;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
import org.apache.hudi.client.HoodieReadClient;
import org.apache.hudi.client.HoodieWriteClient;
import org.apache.hudi.client.WriteStatus;
@@ -43,6 +45,7 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import java.io.IOException;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -78,7 +81,8 @@ public class DataSourceUtils {
// return, if last part of name
if (i == parts.length - 1) {
- return val;
+ Schema fieldSchema = valueNode.getSchema().getField(part).schema();
+ return convertValueForSpecificDataTypes(fieldSchema, val);
} else {
// VC: Need a test here
if (!(val instanceof GenericRecord)) {
@@ -98,6 +102,40 @@ public class DataSourceUtils {
}
/**
+ * This method converts values for fields with certain Avro/Parquet data types that require special handling.
+ *
+ * Logical Date Type is converted to actual Date value instead of Epoch Integer which is how it is
+ * represented/stored in parquet.
+ *
+ * @param fieldSchema avro field schema
+ * @param fieldValue avro field value
+ * @return field value either converted (for certain data types) or as it is.
+ */
+ private static Object convertValueForSpecificDataTypes(Schema fieldSchema, Object fieldValue) {
+ if (fieldSchema == null) {
+ return fieldValue;
+ }
+
+ if (isLogicalTypeDate(fieldSchema)) {
+ return LocalDate.ofEpochDay(Long.parseLong(fieldValue.toString()));
+ }
+ return fieldValue;
+ }
+
+ /**
+ * Given an Avro field schema checks whether the field is of Logical Date Type or not.
+ *
+ * @param fieldSchema avro field schema
+ * @return boolean indicating whether fieldSchema is of Avro's Date Logical Type
+ */
+ private static boolean isLogicalTypeDate(Schema fieldSchema) {
+ if (fieldSchema.getType() == Schema.Type.UNION) {
+ return fieldSchema.getTypes().stream().anyMatch(schema -> schema.getLogicalType() == LogicalTypes.date());
+ }
+ return fieldSchema.getLogicalType() == LogicalTypes.date();
+ }
+
+ /**
* Create a key generator class via reflection, passing in any configs needed.
* <p>
* If the class name of key generator is configured through the properties file, i.e., {@code props}, use the
diff --git a/hudi-spark/src/test/java/DataSourceUtilsTest.java b/hudi-spark/src/test/java/DataSourceUtilsTest.java
new file mode 100644
index 0000000..4fe7547
--- /dev/null
+++ b/hudi-spark/src/test/java/DataSourceUtilsTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.DataSourceUtils;
+import org.junit.Test;
+
+import java.time.LocalDate;
+
+import static org.junit.Assert.assertEquals;
+
+public class DataSourceUtilsTest {
+
+ @Test
+ public void testAvroRecordsFieldConversion() {
+ // There are fields event_date1, event_date2, event_date3 with logical type as Date. event_date1 & event_date3 are
+ // of UNION schema type, which is a union of null and date type in different orders. event_date2 is non-union
+ // date type
+ String avroSchemaString = "{\"type\": \"record\"," + "\"name\": \"events\"," + "\"fields\": [ "
+ + "{\"name\": \"event_date1\", \"type\" : [{\"type\" : \"int\", \"logicalType\" : \"date\"}, \"null\"]},"
+ + "{\"name\": \"event_date2\", \"type\" : {\"type\": \"int\", \"logicalType\" : \"date\"}},"
+ + "{\"name\": \"event_date3\", \"type\" : [\"null\", {\"type\" : \"int\", \"logicalType\" : \"date\"}]},"
+ + "{\"name\": \"event_name\", \"type\": \"string\"},"
+ + "{\"name\": \"event_organizer\", \"type\": \"string\"}"
+ + "]}";
+
+ Schema avroSchema = new Schema.Parser().parse(avroSchemaString);
+ GenericRecord record = new GenericData.Record(avroSchema);
+ record.put("event_date1", 18000);
+ record.put("event_date2", 18001);
+ record.put("event_date3", 18002);
+ record.put("event_name", "Hudi Meetup");
+ record.put("event_organizer", "Hudi PMC");
+
+ assertEquals(LocalDate.ofEpochDay(18000).toString(), DataSourceUtils.getNestedFieldValAsString(record, "event_date1",
+ true));
+ assertEquals(LocalDate.ofEpochDay(18001).toString(), DataSourceUtils.getNestedFieldValAsString(record, "event_date2",
+ true));
+ assertEquals(LocalDate.ofEpochDay(18002).toString(), DataSourceUtils.getNestedFieldValAsString(record, "event_date3",
+ true));
+ assertEquals("Hudi Meetup", DataSourceUtils.getNestedFieldValAsString(record, "event_name", true));
+ assertEquals("Hudi PMC", DataSourceUtils.getNestedFieldValAsString(record, "event_organizer", true));
+ }
+}
[hudi] 03/40: [HUDI-681]Remove embeddedTimelineService from
HoodieReadClient (#1388)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 1d16b151b9079c4ad9bfb08bfa6114c3ad585e0f
Author: hongdd <jn...@163.com>
AuthorDate: Mon Mar 9 18:31:04 2020 +0800
[HUDI-681]Remove embeddedTimelineService from HoodieReadClient (#1388)
* [HUDI-681]Remove embeddedTimelineService from HoodieReadClient
---
.../org/apache/hudi/client/HoodieReadClient.java | 21 ++-------------------
.../main/java/org/apache/hudi/DataSourceUtils.java | 10 ++++------
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 5 +----
.../hudi/utilities/deltastreamer/DeltaSync.java | 2 +-
4 files changed, 8 insertions(+), 30 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
index 33d661b..d1e92b5 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
@@ -19,7 +19,6 @@
package org.apache.hudi.client;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
-import org.apache.hudi.client.embedded.EmbeddedTimelineService;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
@@ -72,18 +71,10 @@ public class HoodieReadClient<T extends HoodieRecordPayload> implements Serializ
/**
* @param basePath path to Hoodie table
*/
- public HoodieReadClient(JavaSparkContext jsc, String basePath, Option<EmbeddedTimelineService> timelineService) {
+ public HoodieReadClient(JavaSparkContext jsc, String basePath) {
this(jsc, HoodieWriteConfig.newBuilder().withPath(basePath)
// by default we use HoodieBloomIndex
- .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build(),
- timelineService);
- }
-
- /**
- * @param basePath path to Hoodie table
- */
- public HoodieReadClient(JavaSparkContext jsc, String basePath) {
- this(jsc, basePath, Option.empty());
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build());
}
/**
@@ -100,14 +91,6 @@ public class HoodieReadClient<T extends HoodieRecordPayload> implements Serializ
* @param clientConfig instance of HoodieWriteConfig
*/
public HoodieReadClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig) {
- this(jsc, clientConfig, Option.empty());
- }
-
- /**
- * @param clientConfig instance of HoodieWriteConfig
- */
- public HoodieReadClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig,
- Option<EmbeddedTimelineService> timelineService) {
this.jsc = jsc;
final String basePath = clientConfig.getBasePath();
// Create a Hoodie table which encapsulated the commits and files visible
diff --git a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
index 35c5955..475a925 100644
--- a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
+++ b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
@@ -21,11 +21,9 @@ package org.apache.hudi;
import org.apache.hudi.client.HoodieReadClient;
import org.apache.hudi.client.HoodieWriteClient;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.embedded.EmbeddedTimelineService;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
-import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.TypedProperties;
import org.apache.hudi.config.HoodieCompactionConfig;
@@ -184,9 +182,9 @@ public class DataSourceUtils {
@SuppressWarnings("unchecked")
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords,
- HoodieWriteConfig writeConfig, Option<EmbeddedTimelineService> timelineService) {
+ HoodieWriteConfig writeConfig) {
try {
- HoodieReadClient client = new HoodieReadClient<>(jssc, writeConfig, timelineService);
+ HoodieReadClient client = new HoodieReadClient<>(jssc, writeConfig);
return client.tagLocation(incomingHoodieRecords)
.filter(r -> !((HoodieRecord<HoodieRecordPayload>) r).isCurrentLocationKnown());
} catch (TableNotFoundException e) {
@@ -198,10 +196,10 @@ public class DataSourceUtils {
@SuppressWarnings("unchecked")
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords,
- Map<String, String> parameters, Option<EmbeddedTimelineService> timelineService) {
+ Map<String, String> parameters) {
HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().withPath(parameters.get("path")).withProps(parameters).build();
- return dropDuplicates(jssc, incomingHoodieRecords, writeConfig, timelineService);
+ return dropDuplicates(jssc, incomingHoodieRecords, writeConfig);
}
public static HiveSyncConfig buildHiveSyncConfig(TypedProperties props, String basePath) {
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 80a01d3..326595f 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -131,10 +131,7 @@ private[hudi] object HoodieSparkSqlWriter {
val hoodieRecords =
if (parameters(INSERT_DROP_DUPS_OPT_KEY).toBoolean) {
- DataSourceUtils.dropDuplicates(
- jsc,
- hoodieAllIncomingRecords,
- mapAsJavaMap(parameters), client.getTimelineServer)
+ DataSourceUtils.dropDuplicates(jsc, hoodieAllIncomingRecords, mapAsJavaMap(parameters))
} else {
hoodieAllIncomingRecords
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
index 97d3d42..4b69d22 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
@@ -357,7 +357,7 @@ public class DeltaSync implements Serializable {
if (cfg.filterDupes) {
// turn upserts to insert
cfg.operation = cfg.operation == Operation.UPSERT ? Operation.INSERT : cfg.operation;
- records = DataSourceUtils.dropDuplicates(jssc, records, writeClient.getConfig(), writeClient.getTimelineServer());
+ records = DataSourceUtils.dropDuplicates(jssc, records, writeClient.getConfig());
}
boolean isEmpty = records.isEmpty();
[hudi] 05/40: [HUDI - 738] Add validation to DeltaStreamer to fail
fast when filterDupes is enabled on UPSERT mode. (#1505)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 2112d0ada59cb12b3be6eb7d407ba48b494c505f
Author: Bhavani Sudha Saktheeswaran <bh...@uber.com>
AuthorDate: Fri Apr 10 08:58:55 2020 -0700
[HUDI - 738] Add validation to DeltaStreamer to fail fast when filterDupes is enabled on UPSERT mode. (#1505)
Summary:
This fix ensures for UPSERT operation, '--filter-dupes' is disabled and fails fast if not. Otherwise it would drop all updates silently and only take in new records.
---
.../org/apache/hudi/utilities/deltastreamer/DeltaSync.java | 5 -----
.../hudi/utilities/deltastreamer/HoodieDeltaStreamer.java | 9 ++++-----
.../org/apache/hudi/utilities/TestHoodieDeltaStreamer.java | 14 ++++++++++++--
3 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
index 3073dfa..4645ade 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
@@ -180,9 +180,6 @@ public class DeltaSync implements Serializable {
UtilHelpers.createSource(cfg.sourceClassName, props, jssc, sparkSession, schemaProvider));
this.hiveConf = hiveConf;
- if (cfg.filterDupes) {
- cfg.operation = cfg.operation == Operation.UPSERT ? Operation.INSERT : cfg.operation;
- }
// If schemaRegistry already resolved, setup write-client
setupWriteClient();
@@ -355,8 +352,6 @@ public class DeltaSync implements Serializable {
Option<String> scheduledCompactionInstant = Option.empty();
// filter dupes if needed
if (cfg.filterDupes) {
- // turn upserts to insert
- cfg.operation = cfg.operation == Operation.UPSERT ? Operation.INSERT : cfg.operation;
records = DataSourceUtils.dropDuplicates(jssc, records, writeClient.getConfig());
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
index 01ab1cc..60947be 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
@@ -359,16 +359,15 @@ public class HoodieDeltaStreamer implements Serializable {
tableType = HoodieTableType.valueOf(cfg.tableType);
}
+ ValidationUtils.checkArgument(!cfg.filterDupes || cfg.operation != Operation.UPSERT,
+ "'--filter-dupes' needs to be disabled when '--op' is 'UPSERT' to ensure updates are not missed.");
+
this.props = UtilHelpers.readConfig(fs, new Path(cfg.propsFilePath), cfg.configs).getConfig();
LOG.info("Creating delta streamer with configs : " + props.toString());
this.schemaProvider = UtilHelpers.createSchemaProvider(cfg.schemaProviderClassName, props, jssc);
- if (cfg.filterDupes) {
- cfg.operation = cfg.operation == Operation.UPSERT ? Operation.INSERT : cfg.operation;
- }
-
deltaSync = new DeltaSync(cfg, sparkSession, schemaProvider, tableType, props, jssc, fs, hiveConf,
- this::onInitializingWriteClient);
+ this::onInitializingWriteClient);
}
public DeltaSync getDeltaSync() {
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
index 9d324dc..1435faa 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
@@ -593,7 +593,7 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
// Generate the same 1000 records + 1000 new ones for upsert
cfg.filterDupes = true;
cfg.sourceLimit = 2000;
- cfg.operation = Operation.UPSERT;
+ cfg.operation = Operation.INSERT;
new HoodieDeltaStreamer(cfg, jsc).sync();
TestHelpers.assertRecordCount(2000, tableBasePath + "/*/*.parquet", sqlContext);
TestHelpers.assertCommitMetadata("00001", tableBasePath, dfs, 2);
@@ -606,7 +606,7 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
HoodieTableMetaClient mClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), tableBasePath, true);
HoodieInstant lastFinished = mClient.getCommitsTimeline().filterCompletedInstants().lastInstant().get();
HoodieDeltaStreamer.Config cfg2 = TestHelpers.makeDropAllConfig(tableBasePath, Operation.UPSERT);
- cfg2.filterDupes = true;
+ cfg2.filterDupes = false;
cfg2.sourceLimit = 2000;
cfg2.operation = Operation.UPSERT;
cfg2.configs.add(String.format("%s=false", HoodieCompactionConfig.AUTO_CLEAN_PROP));
@@ -622,6 +622,16 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
.fromBytes(mClient.getActiveTimeline().getInstantDetails(newLastFinished).get(), HoodieCommitMetadata.class);
System.out.println("New Commit Metadata=" + commitMetadata);
assertTrue(commitMetadata.getPartitionToWriteStats().isEmpty());
+
+ // Try UPSERT with filterDupes true. Expect exception
+ cfg2.filterDupes = true;
+ cfg2.operation = Operation.UPSERT;
+ try {
+ new HoodieDeltaStreamer(cfg2, jsc).sync();
+ } catch (IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains("'--filter-dupes' needs to be disabled when '--op' is 'UPSERT' to ensure updates are not missed."));
+ }
+
}
@Test
[hudi] 26/40: [HUDI-616] Fixed parquet files getting created on
local FS (#1434)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 77d7500d3bbcedda81fa182908984e658d6689e5
Author: Pratyaksh Sharma <pr...@gmail.com>
AuthorDate: Sun Mar 22 19:49:47 2020 +0530
[HUDI-616] Fixed parquet files getting created on local FS (#1434)
---
.../test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
index 1435faa..606b6bb 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
@@ -99,7 +99,7 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
private static final String PROPS_FILENAME_TEST_SOURCE = "test-source.properties";
private static final String PROPS_FILENAME_TEST_INVALID = "test-invalid.properties";
private static final String PROPS_FILENAME_TEST_PARQUET = "test-parquet-dfs-source.properties";
- private static final String PARQUET_SOURCE_ROOT = dfsBasePath + "/parquetFiles";
+ private static String PARQUET_SOURCE_ROOT;
private static final int PARQUET_NUM_RECORDS = 5;
private static final Logger LOG = LogManager.getLogger(TestHoodieDeltaStreamer.class);
@@ -108,6 +108,7 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
@BeforeClass
public static void initClass() throws Exception {
UtilitiesTestBase.initClass(true);
+ PARQUET_SOURCE_ROOT = dfsBasePath + "/parquetFiles";
// prepare the configs.
UtilitiesTestBase.Helpers.copyToDFS("delta-streamer-config/base.properties", dfs, dfsBasePath + "/base.properties");
[hudi] 27/40: [HUDI-902] Avoid exception when getSchemaProvider
(#1584)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5fbb5a76f9ded25e9b6bd15e2db40e1a7c7a0f4d
Author: Raymond Xu <27...@users.noreply.github.com>
AuthorDate: Fri May 15 21:33:02 2020 -0700
[HUDI-902] Avoid exception when getSchemaProvider (#1584)
* When no new input data, don't throw exception for null SchemaProvider
* Return the newly added NullSchemaProvider instead
---
.../apache/hudi/utilities/sources/InputBatch.java | 24 +++++++++++--
.../hudi/utilities/sources/TestInputBatch.java | 40 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 2 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java
index dcf56f3..1acb25b 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java
@@ -19,9 +19,13 @@
package org.apache.hudi.utilities.sources;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.TypedProperties;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.utilities.schema.SchemaProvider;
+import org.apache.avro.Schema;
+import org.apache.spark.api.java.JavaSparkContext;
+
public class InputBatch<T> {
private final Option<T> batch;
@@ -49,9 +53,25 @@ public class InputBatch<T> {
}
public SchemaProvider getSchemaProvider() {
- if (schemaProvider == null) {
+ if (batch.isPresent() && schemaProvider == null) {
throw new HoodieException("Please provide a valid schema provider class!");
}
- return schemaProvider;
+ return Option.ofNullable(schemaProvider).orElse(new NullSchemaProvider());
+ }
+
+ public static class NullSchemaProvider extends SchemaProvider {
+
+ public NullSchemaProvider() {
+ this(null, null);
+ }
+
+ public NullSchemaProvider(TypedProperties props, JavaSparkContext jssc) {
+ super(props, jssc);
+ }
+
+ @Override
+ public Schema getSourceSchema() {
+ return Schema.create(Schema.Type.NULL);
+ }
}
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java
new file mode 100644
index 0000000..314ae48
--- /dev/null
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java
@@ -0,0 +1,40 @@
+package org.apache.hudi.utilities.sources;
+
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.utilities.schema.RowBasedSchemaProvider;
+import org.apache.hudi.utilities.schema.SchemaProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertSame;
+
+public class TestInputBatch {
+
+ @Rule
+ public ExpectedException exceptionRule = ExpectedException.none();
+
+ @Test
+ public void getSchemaProviderShouldThrowException() {
+ final InputBatch<String> inputBatch = new InputBatch<>(Option.of("foo"), null, null);
+ exceptionRule.expect(HoodieException.class);
+ exceptionRule.expectMessage("Please provide a valid schema provider class!");
+ inputBatch.getSchemaProvider();
+ }
+
+ @Test
+ public void getSchemaProviderShouldReturnNullSchemaProvider() {
+ final InputBatch<String> inputBatch = new InputBatch<>(Option.empty(), null, null);
+ SchemaProvider schemaProvider = inputBatch.getSchemaProvider();
+ assertTrue(schemaProvider instanceof InputBatch.NullSchemaProvider);
+ }
+
+ @Test
+ public void getSchemaProviderShouldReturnGivenSchemaProvider() {
+ SchemaProvider schemaProvider = new RowBasedSchemaProvider(null);
+ final InputBatch<String> inputBatch = new InputBatch<>(Option.of("foo"), null, schemaProvider);
+ assertSame(schemaProvider, inputBatch.getSchemaProvider());
+ }
+}
[hudi] 20/40: [HUDI-717] Fixed usage of HiveDriver for DDL
statements. (#1416)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit e80dd0414b1842fd1128fb7d6262b43f8426e048
Author: Prashant Wason <pw...@uber.com>
AuthorDate: Fri Apr 3 16:23:05 2020 -0700
[HUDI-717] Fixed usage of HiveDriver for DDL statements. (#1416)
When using HiveDriver mode in HudiHiveClient, Hive 2.x DDL operations like ALTER PARTITION may fail. This is because Hive 2.x doesn't like `db`.`table_name` for operations. In this fix, we set the name of the database in the SessionState create for the Driver.
---
.../org/apache/hudi/hive/HoodieHiveClient.java | 4 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 91 +++++++++++++++++++++-
.../test/java/org/apache/hudi/hive/TestUtil.java | 12 +--
3 files changed, 99 insertions(+), 8 deletions(-)
diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
index acbff7f..54de3c0 100644
--- a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
+++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
@@ -198,7 +198,8 @@ public class HoodieHiveClient {
for (String partition : partitions) {
String partitionClause = getPartitionClause(partition);
Path partitionPath = FSUtils.getPartitionPath(syncConfig.basePath, partition);
- String fullPartitionPath = partitionPath.toUri().getScheme().equals(StorageSchemes.HDFS.getScheme())
+ String partitionScheme = partitionPath.toUri().getScheme();
+ String fullPartitionPath = StorageSchemes.HDFS.getScheme().equals(partitionScheme)
? FSUtils.getDFSFullPartitionPath(fs, partitionPath) : partitionPath.toString();
String changePartition =
alterTable + " PARTITION (" + partitionClause + ") SET LOCATION '" + fullPartitionPath + "'";
@@ -505,6 +506,7 @@ public class HoodieHiveClient {
try {
final long startTime = System.currentTimeMillis();
ss = SessionState.start(configuration);
+ ss.setCurrentDatabase(syncConfig.databaseName);
hiveDriver = new org.apache.hadoop.hive.ql.Driver(configuration);
final long endTime = System.currentTimeMillis();
LOG.info(String.format("Time taken to start SessionState and create Driver: %s ms", (endTime - startTime)));
diff --git a/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index 7265f33..0f6bab3 100644
--- a/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++ b/hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -168,6 +168,47 @@ public class TestHiveSyncTool {
hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size());
assertEquals("The last commit that was sycned should be updated in the TBLPROPERTIES", commitTime,
hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get());
+
+ // Adding of new partitions
+ List<String> newPartition = Arrays.asList("2050/01/01");
+ hiveClient.addPartitionsToTable(TestUtil.hiveSyncConfig.tableName, Arrays.asList());
+ assertEquals("No new partition should be added", 5,
+ hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size());
+ hiveClient.addPartitionsToTable(TestUtil.hiveSyncConfig.tableName, newPartition);
+ assertEquals("New partition should be added", 6,
+ hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size());
+
+ // Update partitions
+ hiveClient.updatePartitionsToTable(TestUtil.hiveSyncConfig.tableName, Arrays.asList());
+ assertEquals("Partition count should remain the same", 6,
+ hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size());
+ hiveClient.updatePartitionsToTable(TestUtil.hiveSyncConfig.tableName, newPartition);
+ assertEquals("Partition count should remain the same", 6,
+ hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size());
+
+ // Alter partitions
+ // Manually change a hive partition location to check if the sync will detect
+ // it and generage a partition update event for it.
+ hiveClient.updateHiveSQL("ALTER TABLE `" + TestUtil.hiveSyncConfig.tableName
+ + "` PARTITION (`datestr`='2050-01-01') SET LOCATION '/some/new/location'");
+
+ hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
+ List<Partition> hivePartitions = hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName);
+ List<String> writtenPartitionsSince = hiveClient.getPartitionsWrittenToSince(Option.empty());
+ writtenPartitionsSince.add(newPartition.get(0));
+ List<PartitionEvent> partitionEvents = hiveClient.getPartitionEvents(hivePartitions, writtenPartitionsSince);
+ assertEquals("There should be only one paritition event", 1, partitionEvents.size());
+ assertEquals("The one partition event must of type UPDATE", PartitionEventType.UPDATE,
+ partitionEvents.iterator().next().eventType);
+
+ tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
+ tool.syncHoodieTable();
+ // Sync should update the changed partition to correct path
+ List<Partition> tablePartitions = hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName);
+ assertEquals("The one partition we wrote should be added to hive", 6, tablePartitions.size());
+ assertEquals("The last commit that was sycned should be 100", commitTime,
+ hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get());
+
}
@Test
@@ -250,7 +291,7 @@ public class TestHiveSyncTool {
TestUtil.hiveSyncConfig.useJdbc = this.useJdbc;
String commitTime = "100";
String deltaCommitTime = "101";
- TestUtil.createMORTable(commitTime, deltaCommitTime, 5);
+ TestUtil.createMORTable(commitTime, deltaCommitTime, 5, true);
String roTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_READ_OPTIMIZED_TABLE;
HoodieHiveClient hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
@@ -294,7 +335,7 @@ public class TestHiveSyncTool {
String commitTime = "100";
String deltaCommitTime = "101";
String snapshotTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE;
- TestUtil.createMORTable(commitTime, deltaCommitTime, 5);
+ TestUtil.createMORTable(commitTime, deltaCommitTime, 5, true);
HoodieHiveClient hiveClientRT =
new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
@@ -363,4 +404,50 @@ public class TestHiveSyncTool {
assertEquals("The last commit that was sycned should be updated in the TBLPROPERTIES", commitTime,
hiveClient.getLastCommitTimeSynced(hiveSyncConfig.tableName).get());
}
+
+ @Test
+ public void testReadSchemaForMOR() throws Exception {
+ TestUtil.hiveSyncConfig.useJdbc = this.useJdbc;
+ String commitTime = "100";
+ String snapshotTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE;
+ TestUtil.createMORTable(commitTime, "", 5, false);
+ HoodieHiveClient hiveClientRT =
+ new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
+
+ assertFalse("Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE
+ + " should not exist initially", hiveClientRT.doesTableExist(snapshotTableName));
+
+ // Lets do the sync
+ HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
+ tool.syncHoodieTable();
+
+ assertTrue("Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE
+ + " should exist after sync completes", hiveClientRT.doesTableExist(snapshotTableName));
+
+ // Schema being read from compacted base files
+ assertEquals("Hive Schema should match the table schema + partition field", hiveClientRT.getTableSchema(snapshotTableName).size(),
+ SchemaTestUtil.getSimpleSchema().getFields().size() + 1);
+ assertEquals("Table partitions should match the number of partitions we wrote", 5,
+ hiveClientRT.scanTablePartitions(snapshotTableName).size());
+
+ // Now lets create more partitions and these are the only ones which needs to be synced
+ DateTime dateTime = DateTime.now().plusDays(6);
+ String commitTime2 = "102";
+ String deltaCommitTime2 = "103";
+
+ TestUtil.addMORPartitions(1, true, false, dateTime, commitTime2, deltaCommitTime2);
+ // Lets do the sync
+ tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
+ tool.syncHoodieTable();
+ hiveClientRT = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem);
+
+ // Schema being read from the log files
+ assertEquals("Hive Schema should match the evolved table schema + partition field",
+ hiveClientRT.getTableSchema(snapshotTableName).size(), SchemaTestUtil.getEvolvedSchema().getFields().size() + 1);
+ // Sync should add the one partition
+ assertEquals("The 1 partition we wrote should be added to hive", 6, hiveClientRT.scanTablePartitions(snapshotTableName).size());
+ assertEquals("The last commit that was sycned should be 103", deltaCommitTime2,
+ hiveClientRT.getLastCommitTimeSynced(snapshotTableName).get());
+ }
+
}
diff --git a/hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java b/hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java
index 571e949..ead0db9 100644
--- a/hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java
+++ b/hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java
@@ -107,7 +107,6 @@ public class TestUtil {
hiveSyncConfig = new HiveSyncConfig();
hiveSyncConfig.jdbcUrl = "jdbc:hive2://127.0.0.1:9999/";
- hiveSyncConfig.databaseName = "hdrone_test";
hiveSyncConfig.hiveUser = "";
hiveSyncConfig.hivePass = "";
hiveSyncConfig.databaseName = "testdb";
@@ -167,7 +166,8 @@ public class TestUtil {
createCommitFile(commitMetadata, commitTime);
}
- static void createMORTable(String commitTime, String deltaCommitTime, int numberOfPartitions)
+ static void createMORTable(String commitTime, String deltaCommitTime, int numberOfPartitions,
+ boolean createDeltaCommit)
throws IOException, InitializationError, URISyntaxException, InterruptedException {
Path path = new Path(hiveSyncConfig.basePath);
FileIOUtils.deleteDirectory(new File(hiveSyncConfig.basePath));
@@ -185,9 +185,11 @@ public class TestUtil {
commitMetadata.getPartitionToWriteStats()
.forEach((key, value) -> value.forEach(l -> compactionMetadata.addWriteStat(key, l)));
createCompactionCommitFile(compactionMetadata, commitTime);
- // Write a delta commit
- HoodieCommitMetadata deltaMetadata = createLogFiles(commitMetadata.getPartitionToWriteStats(), true);
- createDeltaCommitFile(deltaMetadata, deltaCommitTime);
+ if (createDeltaCommit) {
+ // Write a delta commit
+ HoodieCommitMetadata deltaMetadata = createLogFiles(commitMetadata.getPartitionToWriteStats(), true);
+ createDeltaCommitFile(deltaMetadata, deltaCommitTime);
+ }
}
static void addCOWPartitions(int numberOfPartitions, boolean isParquetSchemaSimple, DateTime startFrom,
[hudi] 22/40: [HUDI-795] Handle auto-deleted empty aux folder
(#1515)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 500b3907a5057b3945115132beb97ef6cae37687
Author: Alexander Filipchik <af...@gmail.com>
AuthorDate: Wed Apr 22 09:47:32 2020 -0700
[HUDI-795] Handle auto-deleted empty aux folder (#1515)
Co-authored-by: Alex Filipchik <al...@csscompany.com>
---
.../apache/hudi/table/HoodieCommitArchiveLog.java | 24 +++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCommitArchiveLog.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCommitArchiveLog.java
index fcc9673..aa2465d 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCommitArchiveLog.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCommitArchiveLog.java
@@ -55,6 +55,7 @@ import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaSparkContext;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -219,14 +220,31 @@ public class HoodieCommitArchiveLog {
* @throws IOException in case of error
*/
private boolean deleteAllInstantsOlderorEqualsInAuxMetaFolder(HoodieInstant thresholdInstant) throws IOException {
- List<HoodieInstant> instants = metaClient.scanHoodieInstantsFromFileSystem(
- new Path(metaClient.getMetaAuxiliaryPath()), HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE, false);
+ List<HoodieInstant> instants = null;
+ boolean success = true;
+ try {
+ instants =
+ metaClient.scanHoodieInstantsFromFileSystem(
+ new Path(metaClient.getMetaAuxiliaryPath()),
+ HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE,
+ false);
+ } catch (FileNotFoundException e) {
+ /*
+ * On some FSs deletion of all files in the directory can auto remove the directory itself.
+ * GCS is one example, as it doesn't have real directories and subdirectories. When client
+ * removes all the files from a "folder" on GCS is has to create a special "/" to keep the folder
+ * around. If this doesn't happen (timeout, misconfigured client, ...) folder will be deleted and
+ * in this case we should not break when aux folder is not found.
+ * GCS information: (https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork)
+ */
+ LOG.warn("Aux path not found. Skipping: " + metaClient.getMetaAuxiliaryPath());
+ return success;
+ }
List<HoodieInstant> instantsToBeDeleted =
instants.stream().filter(instant1 -> HoodieTimeline.compareTimestamps(instant1.getTimestamp(),
thresholdInstant.getTimestamp(), HoodieTimeline.LESSER_OR_EQUAL)).collect(Collectors.toList());
- boolean success = true;
for (HoodieInstant deleteInstant : instantsToBeDeleted) {
LOG.info("Deleting instant " + deleteInstant + " in auxiliary meta path " + metaClient.getMetaAuxiliaryPath());
Path metaFile = new Path(metaClient.getMetaAuxiliaryPath(), deleteInstant.getFileName());
[hudi] 38/40: [MINOR] Remove incubating from README
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 498c86b2b8cdd640e9bf00e4a224f7c17ebef618
Author: vinoth chandar <vi...@users.noreply.github.com>
AuthorDate: Sat May 23 14:51:58 2020 -0700
[MINOR] Remove incubating from README
---
README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index 6bb5659..77812a8 100644
--- a/README.md
+++ b/README.md
@@ -15,8 +15,8 @@
limitations under the License.
-->
-# Apache Hudi (Incubating)
-Apache Hudi (Incubating) (pronounced Hoodie) stands for `Hadoop Upserts Deletes and Incrementals`.
+# Apache Hudi
+Apache Hudi (pronounced Hoodie) stands for `Hadoop Upserts Deletes and Incrementals`.
Hudi manages the storage of large analytical datasets on DFS (Cloud stores, HDFS or any Hadoop FileSystem compatible storage).
<https://hudi.apache.org/>
[hudi] 01/40: Moving to 0.5.3-SNAPSHOT for bug-fix release over
0.5.2
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit f6c87b56c1f79697428eafde8d167dbcfe93b282
Author: Bhavani Sudha Saktheeswaran <bh...@uber.com>
AuthorDate: Thu May 7 09:49:23 2020 -0700
Moving to 0.5.3-SNAPSHOT for bug-fix release over 0.5.2
---
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
hudi-cli/pom.xml | 2 +-
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 2 +-
hudi-hadoop-mr/pom.xml | 2 +-
hudi-hive/pom.xml | 2 +-
hudi-integ-test/pom.xml | 2 +-
hudi-spark/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 2 +-
hudi-utilities/pom.xml | 2 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 2 +-
packaging/hudi-hive-bundle/pom.xml | 2 +-
packaging/hudi-presto-bundle/pom.xml | 2 +-
packaging/hudi-spark-bundle/pom.xml | 2 +-
packaging/hudi-timeline-server-bundle/pom.xml | 2 +-
packaging/hudi-utilities-bundle/pom.xml | 2 +-
pom.xml | 2 +-
27 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/docker/hoodie/hadoop/base/pom.xml b/docker/hoodie/hadoop/base/pom.xml
index 4d70b51..2f271c7 100644
--- a/docker/hoodie/hadoop/base/pom.xml
+++ b/docker/hoodie/hadoop/base/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/datanode/pom.xml b/docker/hoodie/hadoop/datanode/pom.xml
index 0aaaf55..c27fa29 100644
--- a/docker/hoodie/hadoop/datanode/pom.xml
+++ b/docker/hoodie/hadoop/datanode/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/historyserver/pom.xml b/docker/hoodie/hadoop/historyserver/pom.xml
index a4c0c5a..302d453 100644
--- a/docker/hoodie/hadoop/historyserver/pom.xml
+++ b/docker/hoodie/hadoop/historyserver/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/hive_base/pom.xml b/docker/hoodie/hadoop/hive_base/pom.xml
index 7099f96..087c60c 100644
--- a/docker/hoodie/hadoop/hive_base/pom.xml
+++ b/docker/hoodie/hadoop/hive_base/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/namenode/pom.xml b/docker/hoodie/hadoop/namenode/pom.xml
index 841e851..5e93c41 100644
--- a/docker/hoodie/hadoop/namenode/pom.xml
+++ b/docker/hoodie/hadoop/namenode/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml
index f6be901..4c77771 100644
--- a/docker/hoodie/hadoop/pom.xml
+++ b/docker/hoodie/hadoop/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/docker/hoodie/hadoop/prestobase/pom.xml b/docker/hoodie/hadoop/prestobase/pom.xml
index 20fdbaa..c8ac0c9 100644
--- a/docker/hoodie/hadoop/prestobase/pom.xml
+++ b/docker/hoodie/hadoop/prestobase/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/spark_base/pom.xml b/docker/hoodie/hadoop/spark_base/pom.xml
index 718ea93..ae80714 100644
--- a/docker/hoodie/hadoop/spark_base/pom.xml
+++ b/docker/hoodie/hadoop/spark_base/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/sparkadhoc/pom.xml b/docker/hoodie/hadoop/sparkadhoc/pom.xml
index 57d45d0..0ad98af 100644
--- a/docker/hoodie/hadoop/sparkadhoc/pom.xml
+++ b/docker/hoodie/hadoop/sparkadhoc/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/sparkmaster/pom.xml b/docker/hoodie/hadoop/sparkmaster/pom.xml
index 3154384..1df5edd 100644
--- a/docker/hoodie/hadoop/sparkmaster/pom.xml
+++ b/docker/hoodie/hadoop/sparkmaster/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/sparkworker/pom.xml b/docker/hoodie/hadoop/sparkworker/pom.xml
index beaeccd..e80a5c9 100644
--- a/docker/hoodie/hadoop/sparkworker/pom.xml
+++ b/docker/hoodie/hadoop/sparkworker/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/hudi-cli/pom.xml b/hudi-cli/pom.xml
index d8e45c8..a5e358c 100644
--- a/hudi-cli/pom.xml
+++ b/hudi-cli/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-client/pom.xml b/hudi-client/pom.xml
index add5c01..a91a19f 100644
--- a/hudi-client/pom.xml
+++ b/hudi-client/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml
index 73ae5a5..ed7c12f 100644
--- a/hudi-common/pom.xml
+++ b/hudi-common/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-hadoop-mr/pom.xml b/hudi-hadoop-mr/pom.xml
index 8a8327c..b229370 100644
--- a/hudi-hadoop-mr/pom.xml
+++ b/hudi-hadoop-mr/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-hive/pom.xml b/hudi-hive/pom.xml
index d1b1f8d..7ff1b3a 100644
--- a/hudi-hive/pom.xml
+++ b/hudi-hive/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml
index 16c416a..fe6f66d 100644
--- a/hudi-integ-test/pom.xml
+++ b/hudi-integ-test/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>hudi-integ-test</artifactId>
diff --git a/hudi-spark/pom.xml b/hudi-spark/pom.xml
index 3d87b22..501cd8f 100644
--- a/hudi-spark/pom.xml
+++ b/hudi-spark/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-timeline-service/pom.xml b/hudi-timeline-service/pom.xml
index 597d1dc..c51ec54 100644
--- a/hudi-timeline-service/pom.xml
+++ b/hudi-timeline-service/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml
index 1ce38ff..b8b71ac 100644
--- a/hudi-utilities/pom.xml
+++ b/hudi-utilities/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-hadoop-mr-bundle/pom.xml b/packaging/hudi-hadoop-mr-bundle/pom.xml
index d8eba14..ae5a99f 100644
--- a/packaging/hudi-hadoop-mr-bundle/pom.xml
+++ b/packaging/hudi-hadoop-mr-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-hive-bundle/pom.xml b/packaging/hudi-hive-bundle/pom.xml
index 1c4ac76..7979b63 100644
--- a/packaging/hudi-hive-bundle/pom.xml
+++ b/packaging/hudi-hive-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-presto-bundle/pom.xml b/packaging/hudi-presto-bundle/pom.xml
index b503021..1235182 100644
--- a/packaging/hudi-presto-bundle/pom.xml
+++ b/packaging/hudi-presto-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-spark-bundle/pom.xml b/packaging/hudi-spark-bundle/pom.xml
index f26f217..5cbf2b7 100644
--- a/packaging/hudi-spark-bundle/pom.xml
+++ b/packaging/hudi-spark-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-timeline-server-bundle/pom.xml b/packaging/hudi-timeline-server-bundle/pom.xml
index 4a855c1..f7f9249 100644
--- a/packaging/hudi-timeline-server-bundle/pom.xml
+++ b/packaging/hudi-timeline-server-bundle/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-utilities-bundle/pom.xml b/packaging/hudi-utilities-bundle/pom.xml
index 81b9c1c..a84ce2a 100644
--- a/packaging/hudi-utilities-bundle/pom.xml
+++ b/packaging/hudi-utilities-bundle/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index d6cfe40..b49c653 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.hudi</groupId>
<artifactId>hudi</artifactId>
<packaging>pom</packaging>
- <version>0.5.2-incubating</version>
+ <version>0.5.3-SNAPSHOT</version>
<description>Apache Hudi (incubating) brings stream style processing on big data</description>
<url>https://github.com/apache/incubator-hudi</url>
<name>Hudi</name>
[hudi] 14/40: Add changes for presto mor queries (#1578)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d476ac541b00b6a00dd690c4a7bfa3fc6037e7ab
Author: bschell <bd...@gmail.com>
AuthorDate: Mon May 4 11:27:14 2020 -0700
Add changes for presto mor queries (#1578)
Adds the neccessary changes to hudi for support of presto querying hudi
merge-on-read table's realtime view.
Co-authored-by: Brandon Scheller <bs...@amazon.com>
---
.../hadoop/UseRecordReaderFromInputFormat.java | 38 ++++++++++++++++++++++
.../realtime/HoodieParquetRealtimeInputFormat.java | 2 ++
.../org/apache/hudi/hadoop/TestAnnotation.java | 23 +++++++++++--
packaging/hudi-presto-bundle/pom.xml | 18 ++++++++++
4 files changed, 79 insertions(+), 2 deletions(-)
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseRecordReaderFromInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseRecordReaderFromInputFormat.java
new file mode 100644
index 0000000..fe87323
--- /dev/null
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseRecordReaderFromInputFormat.java
@@ -0,0 +1,38 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hudi.hadoop;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+* When annotated on a InputFormat, informs the query engines, that they should use the RecordReader provided by the input
+* format to execute the queries.
+*/
+@Inherited
+@Documented
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface UseRecordReaderFromInputFormat {
+
+}
\ No newline at end of file
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
index 4fe041a..59bf441 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hudi.hadoop.UseRecordReaderFromInputFormat;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -62,6 +63,7 @@ import java.util.stream.Stream;
/**
* Input Format, that provides a real-time view of data in a Hoodie table.
*/
+@UseRecordReaderFromInputFormat
@UseFileSplitsFromInputFormat
public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat implements Configurable {
diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java
index 3bdaa64..fefa25c 100644
--- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java
+++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java
@@ -19,7 +19,7 @@
package org.apache.hudi.hadoop;
import org.junit.Test;
-
+import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat;
import java.lang.annotation.Annotation;
import static org.junit.Assert.assertTrue;
@@ -27,7 +27,7 @@ import static org.junit.Assert.assertTrue;
public class TestAnnotation {
@Test
- public void testAnnotation() {
+ public void testHoodieParquetInputFormatAnnotation() {
assertTrue(HoodieParquetInputFormat.class.isAnnotationPresent(UseFileSplitsFromInputFormat.class));
Annotation[] annotations = HoodieParquetInputFormat.class.getAnnotations();
boolean found = false;
@@ -38,4 +38,23 @@ public class TestAnnotation {
}
assertTrue(found);
}
+
+ @Test
+ public void testHoodieParquetRealtimeInputFormatAnnotations() {
+ assertTrue(HoodieParquetRealtimeInputFormat.class.isAnnotationPresent(UseFileSplitsFromInputFormat.class));
+ assertTrue(HoodieParquetRealtimeInputFormat.class.isAnnotationPresent(UseRecordReaderFromInputFormat.class));
+ Annotation[] annotations = HoodieParquetRealtimeInputFormat.class.getAnnotations();
+ boolean foundFileSplitsAnnotation = false;
+ boolean foundRecordReaderAnnotation = false;
+ for (Annotation annotation : annotations) {
+ if ("UseFileSplitsFromInputFormat".equals(annotation.annotationType().getSimpleName())) {
+ foundFileSplitsAnnotation = true;
+ }
+ if ("UseRecordReaderFromInputFormat".equals(annotation.annotationType().getSimpleName())) {
+ foundRecordReaderAnnotation = true;
+ }
+ }
+ assertTrue(foundFileSplitsAnnotation);
+ assertTrue(foundRecordReaderAnnotation);
+ }
}
diff --git a/packaging/hudi-presto-bundle/pom.xml b/packaging/hudi-presto-bundle/pom.xml
index 1235182..4900e39 100644
--- a/packaging/hudi-presto-bundle/pom.xml
+++ b/packaging/hudi-presto-bundle/pom.xml
@@ -68,6 +68,7 @@
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.parquet:parquet-avro</include>
+ <include>org.apache.avro:avro</include>
<include>com.esotericsoftware:kryo-shaded</include>
<include>org.objenesis:objenesis</include>
<include>com.esotericsoftware:minlog</include>
@@ -76,6 +77,10 @@
<relocations>
<relocation>
+ <pattern>org.apache.avro.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.avro.</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>com.esotericsoftware.kryo.</pattern>
<shadedPattern>org.apache.hudi.com.esotericsoftware.kryo.</shadedPattern>
</relocation>
@@ -128,5 +133,18 @@
<artifactId>hudi-hadoop-mr-bundle</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- Parquet -->
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
[hudi] 29/40: [HUDI-889] Writer supports useJdbc configuration when
hive synchronization is enabled (#1627)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit da043af11e731b3e53d998c3db3a66f13fd59026
Author: cxzl25 <cx...@users.noreply.github.com>
AuthorDate: Thu May 14 00:20:13 2020 +0800
[HUDI-889] Writer supports useJdbc configuration when hive synchronization is enabled (#1627)
---
hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java | 2 ++
hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala | 2 ++
hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala | 4 +++-
3 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
index 99a795d..aa32e9b 100644
--- a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
+++ b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
@@ -261,6 +261,8 @@ public class DataSourceUtils {
hiveSyncConfig.partitionValueExtractorClass =
props.getString(DataSourceWriteOptions.HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY(),
SlashEncodedDayPartitionValueExtractor.class.getName());
+ hiveSyncConfig.useJdbc = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_USE_JDBC_OPT_KEY(),
+ DataSourceWriteOptions.DEFAULT_HIVE_USE_JDBC_OPT_VAL()));
return hiveSyncConfig;
}
}
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala
index 10978e4..939a17c 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala
@@ -263,6 +263,7 @@ object DataSourceWriteOptions {
val HIVE_PARTITION_FIELDS_OPT_KEY = "hoodie.datasource.hive_sync.partition_fields"
val HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY = "hoodie.datasource.hive_sync.partition_extractor_class"
val HIVE_USE_PRE_APACHE_INPUT_FORMAT_OPT_KEY = "hoodie.datasource.hive_sync.use_pre_apache_input_format"
+ val HIVE_USE_JDBC_OPT_KEY = "hoodie.datasource.hive_sync.use_jdbc"
// DEFAULT FOR HIVE SPECIFIC CONFIGS
val DEFAULT_HIVE_SYNC_ENABLED_OPT_VAL = "false"
@@ -275,4 +276,5 @@ object DataSourceWriteOptions {
val DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL = classOf[SlashEncodedDayPartitionValueExtractor].getCanonicalName
val DEFAULT_HIVE_ASSUME_DATE_PARTITION_OPT_VAL = "false"
val DEFAULT_USE_PRE_APACHE_INPUT_FORMAT_OPT_VAL = "false"
+ val DEFAULT_HIVE_USE_JDBC_OPT_VAL = "true"
}
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index e5886eb..5ad1f60 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -216,7 +216,8 @@ private[hudi] object HoodieSparkSqlWriter {
HIVE_URL_OPT_KEY -> DEFAULT_HIVE_URL_OPT_VAL,
HIVE_PARTITION_FIELDS_OPT_KEY -> DEFAULT_HIVE_PARTITION_FIELDS_OPT_VAL,
HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY -> DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL,
- HIVE_STYLE_PARTITIONING_OPT_KEY -> DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL
+ HIVE_STYLE_PARTITIONING_OPT_KEY -> DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL,
+ HIVE_USE_JDBC_OPT_KEY -> DEFAULT_HIVE_USE_JDBC_OPT_VAL
) ++ translateStorageTypeToTableType(parameters)
}
@@ -247,6 +248,7 @@ private[hudi] object HoodieSparkSqlWriter {
hiveSyncConfig.partitionFields =
ListBuffer(parameters(HIVE_PARTITION_FIELDS_OPT_KEY).split(",").map(_.trim).filter(!_.isEmpty).toList: _*)
hiveSyncConfig.partitionValueExtractorClass = parameters(HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY)
+ hiveSyncConfig.useJdbc = parameters(HIVE_USE_JDBC_OPT_KEY).toBoolean
hiveSyncConfig
}
[hudi] 21/40: [HUDI-727]: Copy default values of fields if not
present when rewriting incoming record with new schema (#1427)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit f9c3b078966fad1843554853b6179c04cae1d7f8
Author: Pratyaksh Sharma <pr...@gmail.com>
AuthorDate: Mon Apr 13 06:25:26 2020 +0530
[HUDI-727]: Copy default values of fields if not present when rewriting incoming record with new schema (#1427)
---
.../apache/hudi/common/util/HoodieAvroUtils.java | 43 +++++++++++------
.../hudi/common/util/TestHoodieAvroUtils.java | 54 +++++++++++++++++++++-
.../realtime/AbstractRealtimeRecordReader.java | 4 +-
3 files changed, 84 insertions(+), 17 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
index 02e48e3..32ea71e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
@@ -32,7 +32,6 @@ import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
-import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.node.NullNode;
import java.io.ByteArrayInputStream;
@@ -44,6 +43,7 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -104,15 +104,15 @@ public class HoodieAvroUtils {
List<Schema.Field> parentFields = new ArrayList<>();
Schema.Field commitTimeField =
- new Schema.Field(HoodieRecord.COMMIT_TIME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
+ new Schema.Field(HoodieRecord.COMMIT_TIME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", (Object) null);
Schema.Field commitSeqnoField =
- new Schema.Field(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
+ new Schema.Field(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", (Object) null);
Schema.Field recordKeyField =
- new Schema.Field(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
+ new Schema.Field(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", (Object) null);
Schema.Field partitionPathField =
- new Schema.Field(HoodieRecord.PARTITION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
+ new Schema.Field(HoodieRecord.PARTITION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", (Object) null);
Schema.Field fileNameField =
- new Schema.Field(HoodieRecord.FILENAME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
+ new Schema.Field(HoodieRecord.FILENAME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", (Object) null);
parentFields.add(commitTimeField);
parentFields.add(commitSeqnoField);
@@ -121,8 +121,8 @@ public class HoodieAvroUtils {
parentFields.add(fileNameField);
for (Schema.Field field : schema.getFields()) {
if (!isMetadataField(field.name())) {
- Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue());
- for (Map.Entry<String, JsonNode> prop : field.getJsonProps().entrySet()) {
+ Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal());
+ for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) {
newField.addProp(prop.getKey(), prop.getValue());
}
parentFields.add(newField);
@@ -191,7 +191,7 @@ public class HoodieAvroUtils {
* schema.
*/
public static GenericRecord rewriteRecord(GenericRecord record, Schema newSchema) {
- return rewrite(record, record.getSchema(), newSchema);
+ return rewrite(record, getCombinedFieldsToWrite(record.getSchema(), newSchema), newSchema);
}
/**
@@ -199,13 +199,17 @@ public class HoodieAvroUtils {
* schema.
*/
public static GenericRecord rewriteRecordWithOnlyNewSchemaFields(GenericRecord record, Schema newSchema) {
- return rewrite(record, newSchema, newSchema);
+ return rewrite(record, new LinkedHashSet<>(newSchema.getFields()), newSchema);
}
- private static GenericRecord rewrite(GenericRecord record, Schema schemaWithFields, Schema newSchema) {
+ private static GenericRecord rewrite(GenericRecord record, LinkedHashSet<Field> fieldsToWrite, Schema newSchema) {
GenericRecord newRecord = new GenericData.Record(newSchema);
- for (Schema.Field f : schemaWithFields.getFields()) {
- newRecord.put(f.name(), record.get(f.name()));
+ for (Schema.Field f : fieldsToWrite) {
+ if (record.get(f.name()) == null) {
+ newRecord.put(f.name(), f.defaultVal());
+ } else {
+ newRecord.put(f.name(), record.get(f.name()));
+ }
}
if (!GenericData.get().validate(newSchema, newRecord)) {
throw new SchemaCompatabilityException(
@@ -214,6 +218,19 @@ public class HoodieAvroUtils {
return newRecord;
}
+ /**
+ * Generates a super set of fields from both old and new schema.
+ */
+ private static LinkedHashSet<Field> getCombinedFieldsToWrite(Schema oldSchema, Schema newSchema) {
+ LinkedHashSet<Field> allFields = new LinkedHashSet<>(oldSchema.getFields());
+ for (Schema.Field f : newSchema.getFields()) {
+ if (!allFields.contains(f) && !isMetadataField(f.name())) {
+ allFields.add(f);
+ }
+ }
+ return allFields;
+ }
+
public static byte[] compress(String text) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java
index 2f925b8..b2c13f8 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java
@@ -19,7 +19,8 @@
package org.apache.hudi.common.util;
import org.apache.avro.Schema;
-import org.codehaus.jackson.JsonNode;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
import org.junit.Assert;
import org.junit.Test;
@@ -30,11 +31,25 @@ import java.util.Map;
*/
public class TestHoodieAvroUtils {
+ private static String EVOLVED_SCHEMA = "{\"type\": \"record\",\"name\": \"testrec1\",\"fields\": [ "
+ + "{\"name\": \"timestamp\",\"type\": \"double\"},{\"name\": \"_row_key\", \"type\": \"string\"},"
+ + "{\"name\": \"non_pii_col\", \"type\": \"string\"},"
+ + "{\"name\": \"pii_col\", \"type\": \"string\", \"column_category\": \"user_profile\"},"
+ + "{\"name\": \"new_col1\", \"type\": \"string\", \"default\": \"dummy_val\"},"
+ + "{\"name\": \"new_col2\", \"type\": [\"int\", \"null\"]}]}";
+
private static String EXAMPLE_SCHEMA = "{\"type\": \"record\",\"name\": \"testrec\",\"fields\": [ "
+ "{\"name\": \"timestamp\",\"type\": \"double\"},{\"name\": \"_row_key\", \"type\": \"string\"},"
+ "{\"name\": \"non_pii_col\", \"type\": \"string\"},"
+ "{\"name\": \"pii_col\", \"type\": \"string\", \"column_category\": \"user_profile\"}]}";
+
+ private static String SCHEMA_WITH_METADATA_FIELD = "{\"type\": \"record\",\"name\": \"testrec2\",\"fields\": [ "
+ + "{\"name\": \"timestamp\",\"type\": \"double\"},{\"name\": \"_row_key\", \"type\": \"string\"},"
+ + "{\"name\": \"non_pii_col\", \"type\": \"string\"},"
+ + "{\"name\": \"pii_col\", \"type\": \"string\", \"column_category\": \"user_profile\"},"
+ + "{\"name\": \"_hoodie_commit_time\", \"type\": [\"null\", \"string\"]}]}";
+
@Test
public void testPropsPresent() {
Schema schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(EXAMPLE_SCHEMA));
@@ -45,7 +60,7 @@ public class TestHoodieAvroUtils {
}
Assert.assertNotNull("field name is null", field.name());
- Map<String, JsonNode> props = field.getJsonProps();
+ Map<String, Object> props = field.getObjectProps();
Assert.assertNotNull("The property is null", props);
if (field.name().equals("pii_col")) {
@@ -57,4 +72,39 @@ public class TestHoodieAvroUtils {
}
Assert.assertTrue("column pii_col doesn't show up", piiPresent);
}
+
+ @Test
+ public void testDefaultValue() {
+ GenericRecord rec = new GenericData.Record(new Schema.Parser().parse(EVOLVED_SCHEMA));
+ rec.put("_row_key", "key1");
+ rec.put("non_pii_col", "val1");
+ rec.put("pii_col", "val2");
+ rec.put("timestamp", 3.5);
+ GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, new Schema.Parser().parse(EVOLVED_SCHEMA));
+ Assert.assertEquals(rec1.get("new_col1"), "dummy_val");
+ Assert.assertNull(rec1.get("new_col2"));
+ }
+
+ @Test
+ public void testDefaultValueWithSchemaEvolution() {
+ GenericRecord rec = new GenericData.Record(new Schema.Parser().parse(EXAMPLE_SCHEMA));
+ rec.put("_row_key", "key1");
+ rec.put("non_pii_col", "val1");
+ rec.put("pii_col", "val2");
+ rec.put("timestamp", 3.5);
+ GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, new Schema.Parser().parse(EVOLVED_SCHEMA));
+ Assert.assertEquals(rec1.get("new_col1"), "dummy_val");
+ Assert.assertNull(rec1.get("new_col2"));
+ }
+
+ @Test
+ public void testMetadataField() {
+ GenericRecord rec = new GenericData.Record(new Schema.Parser().parse(EXAMPLE_SCHEMA));
+ rec.put("_row_key", "key1");
+ rec.put("non_pii_col", "val1");
+ rec.put("pii_col", "val2");
+ rec.put("timestamp", 3.5);
+ GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, new Schema.Parser().parse(SCHEMA_WITH_METADATA_FIELD));
+ Assert.assertNull(rec1.get("_hoodie_commit_time"));
+ }
}
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
index df635d6..782a6fc 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java
@@ -212,7 +212,7 @@ public abstract class AbstractRealtimeRecordReader {
throw new HoodieException("Field " + fn + " not found in log schema. Query cannot proceed! "
+ "Derived Schema Fields: " + new ArrayList<>(schemaFieldsMap.keySet()));
} else {
- projectedFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue()));
+ projectedFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
}
}
@@ -367,7 +367,7 @@ public abstract class AbstractRealtimeRecordReader {
Field field = schemaFieldsMap.get(columnName.toLowerCase());
if (field != null) {
- hiveSchemaFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue()));
+ hiveSchemaFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
} else {
// Hive has some extra virtual columns like BLOCK__OFFSET__INSIDE__FILE which do not exist in table schema.
// They will get skipped as they won't be found in the original schema.
[hudi] 39/40: [HUDI-926] Removing DISCLAIMER from the repo (#1657)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 3a776f6eb2793b31ca4a76fbbe59e205b83ab75b
Author: leesf <le...@apache.org>
AuthorDate: Sun May 24 18:27:08 2020 +0800
[HUDI-926] Removing DISCLAIMER from the repo (#1657)
---
DISCLAIMER | 10 ----------
1 file changed, 10 deletions(-)
diff --git a/DISCLAIMER b/DISCLAIMER
deleted file mode 100644
index a4db88b..0000000
--- a/DISCLAIMER
+++ /dev/null
@@ -1,10 +0,0 @@
-Apache Hudi (incubating) is an effort undergoing incubation
-at The Apache Software Foundation (ASF), sponsored by the Apache Incubator.
-
-Incubation is required of all newly accepted projects until a further review
-indicates that the infrastructure, communications, and decision making process
-have stabilized in a manner consistent with other successful ASF projects.
-
-While incubation status is not necessarily a reflection of the
-completeness or stability of the code, it does indicate that the
-project has yet to be fully endorsed by the ASF.
[hudi] 16/40: [HUDI-716] Exception: Not an Avro data file when
running HoodieCleanClient.runClean (#1432)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 86698b516001fc30427f3f15a56ec5289d17a9c1
Author: lamber-ken <la...@163.com>
AuthorDate: Mon Mar 30 13:19:17 2020 -0500
[HUDI-716] Exception: Not an Avro data file when running HoodieCleanClient.runClean (#1432)
---
.../apache/hudi/cli/commands/RepairsCommand.java | 21 +++++++++++++++++++
.../org/apache/hudi/client/HoodieCleanClient.java | 6 +++++-
.../java/org/apache/hudi/table/TestCleaner.java | 19 +++++++++++++++++
.../java/org/apache/hudi/common/util/FSUtils.java | 9 ++++++++
.../apache/hudi/common/model/HoodieTestUtils.java | 24 ++++++++++++++++++++++
5 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 83af13c..7a65336 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -25,9 +25,12 @@ import org.apache.hudi.cli.utils.SparkUtil;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
import org.apache.spark.launcher.SparkLauncher;
import org.springframework.shell.core.CommandMarker;
import org.springframework.shell.core.annotation.CliCommand;
@@ -51,6 +54,8 @@ import static org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME
@Component
public class RepairsCommand implements CommandMarker {
+ private static final Logger LOG = Logger.getLogger(RepairsCommand.class);
+
@CliCommand(value = "repair deduplicate",
help = "De-duplicate a partition path contains duplicates & produce repaired files to replace with")
public String deduplicate(
@@ -137,4 +142,20 @@ public class RepairsCommand implements CommandMarker {
}
return HoodiePrintHelper.print(new String[] {"Property", "Old Value", "New Value"}, rows);
}
+
+ @CliCommand(value = "repair corrupted clean files", help = "repair corrupted clean files")
+ public void removeCorruptedPendingCleanAction() {
+
+ HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
+ HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
+
+ activeTimeline.filterInflightsAndRequested().getInstants().forEach(instant -> {
+ try {
+ CleanerUtils.getCleanerPlan(client, instant);
+ } catch (IOException e) {
+ LOG.warn("try to remove corrupted instant file: " + instant);
+ FSUtils.deleteInstantFile(client.getFs(), client.getMetaPath(), instant);
+ }
+ });
+ }
}
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
index 15ba2a7..600e801 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
@@ -85,7 +85,11 @@ public class HoodieCleanClient<T extends HoodieRecordPayload> extends AbstractHo
// If there are inflight(failed) or previously requested clean operation, first perform them
table.getCleanTimeline().filterInflightsAndRequested().getInstants().forEach(hoodieInstant -> {
LOG.info("There were previously unfinished cleaner operations. Finishing Instant=" + hoodieInstant);
- runClean(table, hoodieInstant);
+ try {
+ runClean(table, hoodieInstant);
+ } catch (Exception e) {
+ LOG.warn("Failed to perform previous clean operation, instant: " + hoodieInstant, e);
+ }
});
Option<HoodieCleanerPlan> cleanerPlanOpt = scheduleClean(startCleanTime);
diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
index dee545a..2fc478d 100644
--- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
+++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
@@ -990,6 +990,25 @@ public class TestCleaner extends TestHoodieClientBase {
}
/**
+ * Test clean previous corrupted cleanFiles.
+ */
+ @Test
+ public void testCleanPreviousCorruptedCleanFiles() {
+ HoodieWriteConfig config =
+ HoodieWriteConfig.newBuilder()
+ .withPath(basePath).withAssumeDatePartitioning(true)
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ .withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS).retainFileVersions(1).build())
+ .build();
+
+ HoodieTestUtils.createCorruptedPendingCleanFiles(metaClient, getNextInstant());
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+
+ List<HoodieCleanStat> cleanStats = runCleaner(config);
+ assertEquals("Must not clean any files", 0, cleanStats.size());
+ }
+
+ /**
* Common test method for validating pending compactions.
*
* @param config Hoodie Write Config
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
index add03d5..5ce7178 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
@@ -485,6 +485,15 @@ public class FSUtils {
});
}
+ public static void deleteInstantFile(FileSystem fs, String metaPath, HoodieInstant instant) {
+ try {
+ LOG.warn("try to delete instant file: " + instant);
+ fs.delete(new Path(metaPath, instant.getFileName()), false);
+ } catch (IOException e) {
+ throw new HoodieIOException("Could not delete instant file" + instant.getFileName(), e);
+ }
+ }
+
public static void deleteOlderRestoreMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants) {
// TODO - this should be archived when archival is made general for all meta-data
// skip MIN_ROLLBACK_TO_KEEP and delete rest
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
index cae3d89..2129168 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
@@ -179,6 +179,30 @@ public class HoodieTestUtils {
}
}
+ public static void createCorruptedPendingCleanFiles(HoodieTableMetaClient metaClient, String commitTime) {
+ Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(commitTime),
+ HoodieTimeline.makeInflightCleanerFileName(commitTime)).forEach(f -> {
+ FSDataOutputStream os = null;
+ try {
+ Path commitFile = new Path(
+ metaClient.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + f);
+ os = metaClient.getFs().create(commitFile, true);
+ // Write empty clean metadata
+ os.write(new byte[0]);
+ } catch (IOException ioe) {
+ throw new HoodieIOException(ioe.getMessage(), ioe);
+ } finally {
+ if (null != os) {
+ try {
+ os.close();
+ } catch (IOException e) {
+ throw new HoodieIOException(e.getMessage(), e);
+ }
+ }
+ }
+ });
+ }
+
public static String createNewDataFile(String basePath, String partitionPath, String commitTime)
throws IOException {
String fileID = UUID.randomUUID().toString();
[hudi] 25/40: [MINOR] fixed building IndexFileFilter with a wrong
condition in HoodieGlobalBloomIndex class (#1537)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit be205eda53ab3210aea455a377aaf7610b8e35e1
Author: Carm <15...@qq.com>
AuthorDate: Sun May 10 09:45:07 2020 +0800
[MINOR] fixed building IndexFileFilter with a wrong condition in HoodieGlobalBloomIndex class (#1537)
---
.../main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
index 3c6cc72..49286f0 100644
--- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
+++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
@@ -86,7 +86,7 @@ public class HoodieGlobalBloomIndex<T extends HoodieRecordPayload> extends Hoodi
JavaPairRDD<String, String> partitionRecordKeyPairRDD) {
IndexFileFilter indexFileFilter =
- config.getBloomIndexPruneByRanges() ? new IntervalTreeBasedGlobalIndexFileFilter(partitionToFileIndexInfo)
+ config.useBloomIndexTreebasedFilter() ? new IntervalTreeBasedGlobalIndexFileFilter(partitionToFileIndexInfo)
: new ListBasedGlobalIndexFileFilter(partitionToFileIndexInfo);
return partitionRecordKeyPairRDD.map(partitionRecordKeyPair -> {
[hudi] 40/40: [HUDI-938] Removing incubating/incubator from project
(#1658)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 54bbe32d741733e405b241fd95d1dc2dfefea144
Author: leesf <le...@apache.org>
AuthorDate: Sun May 24 18:28:13 2020 +0800
[HUDI-938] Removing incubating/incubator from project (#1658)
---
NOTICE | 2 +-
README.md | 4 ++--
doap_HUDI.rdf | 7 +++----
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 2 +-
pom.xml | 14 +++++---------
scripts/release/cut_release_branch.sh | 12 ++++++------
scripts/release/preparation_before_release.sh | 4 ++--
scripts/release/validate_staged_release.sh | 22 +++++++++++-----------
8 files changed, 31 insertions(+), 36 deletions(-)
diff --git a/NOTICE b/NOTICE
index 59e56b4..2f1aee6 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,4 +1,4 @@
-Apache Hudi (incubating)
+Apache Hudi
Copyright 2019-2020 The Apache Software Foundation
This product includes software developed at
diff --git a/README.md b/README.md
index 77812a8..00ad886 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ Hudi manages the storage of large analytical datasets on DFS (Cloud stores, HDFS
<https://hudi.apache.org/>
-[](https://travis-ci.org/apache/incubator-hudi)
+[](https://travis-ci.org/apache/hudi)
[](https://www.apache.org/licenses/LICENSE-2.0.html)
[](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.hudi%22)
[](https://join.slack.com/t/apache-hudi/shared_invite/enQtODYyNDAxNzc5MTg2LTE5OTBlYmVhYjM0N2ZhOTJjOWM4YzBmMWU2MjZjMGE4NDc5ZDFiOGQ2N2VkYTVkNzU3ZDQ4OTI1NmFmYWQ0NzE)
@@ -53,7 +53,7 @@ Prerequisites for building Apache Hudi:
```
# Checkout code and build
-git clone https://github.com/apache/incubator-hudi.git && cd incubator-hudi
+git clone https://github.com/apache/hudi.git && cd hudi
mvn clean package -DskipTests -DskipITs
```
diff --git a/doap_HUDI.rdf b/doap_HUDI.rdf
index af45a41..b92db82 100644
--- a/doap_HUDI.rdf
+++ b/doap_HUDI.rdf
@@ -26,7 +26,6 @@
<license rdf:resource="http://usefulinc.com/doap/licenses/asl20" />
<name>Apache Hudi</name>
<homepage rdf:resource="https://hudi.apache.org" />
- <asfext:pmc rdf:resource="https://incubator.apache.org" />
<shortdesc>Ingests and Manages storage of large analytical datasets</shortdesc>
<description>Hudi (pronounced “Hoodie”) brings stream processing to big data, providing upserts, deletes and incremental data streams.</description>
<bug-database rdf:resource="https://issues.apache.org/jira/browse/HUDI" />
@@ -54,13 +53,13 @@
</release>
<repository>
<GitRepository>
- <location rdf:resource="https://github.com/apache/incubator-hudi.git"/>
- <browse rdf:resource="https://github.com/apache/incubator-hudi"/>
+ <location rdf:resource="https://github.com/apache/hudi.git"/>
+ <browse rdf:resource="https://github.com/apache/hudi"/>
</GitRepository>
</repository>
<maintainer>
<foaf:Person>
- <foaf:name>Apache Hudi-incubating PPMC</foaf:name>
+ <foaf:name>Apache Hudi PMC</foaf:name>
<foaf:mbox rdf:resource="mailto:dev@hudi.apache.org"/>
</foaf:Person>
</maintainer>
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
index f61028e..95795d7 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
@@ -26,7 +26,7 @@ import org.junit.Test;
import java.util.List;
/**
- * Goes through steps described in https://hudi.incubator.apache.org/docker_demo.html
+ * Goes through steps described in https://hudi.apache.org/docker_demo.html
*
* To run this as a standalone test in the IDE or command line. First bring up the demo setup using
* `docker/setup_demo.sh` and then run the test class as you would do normally.
diff --git a/pom.xml b/pom.xml
index 8603024..a09e3cc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,8 +30,8 @@
<artifactId>hudi</artifactId>
<packaging>pom</packaging>
<version>0.5.3-SNAPSHOT</version>
- <description>Apache Hudi (incubating) brings stream style processing on big data</description>
- <url>https://github.com/apache/incubator-hudi</url>
+ <description>Apache Hudi brings stream style processing on big data</description>
+ <url>https://github.com/apache/hudi</url>
<name>Hudi</name>
<modules>
@@ -119,9 +119,9 @@
</properties>
<scm>
- <connection>scm:git:git@github.com:apache/incubator-hudi.git</connection>
- <developerConnection>scm:git:git@github.com:apache/incubator-hudi.git</developerConnection>
- <url>git@github.com:apache/incubator-hudi.git</url>
+ <connection>scm:git:git@github.com:apache/hudi.git</connection>
+ <developerConnection>scm:git:git@github.com:apache/hudi.git</developerConnection>
+ <url>git@github.com:apache/hudi.git</url>
<tag>HEAD</tag>
</scm>
@@ -1092,8 +1092,4 @@
</profile>
</profiles>
- <issueManagement>
- <system>GitHub</system>
- <url>https://github.com/apache/incubator-hudi/issues</url>
- </issueManagement>
</project>
diff --git a/scripts/release/cut_release_branch.sh b/scripts/release/cut_release_branch.sh
index f8f0d9f..49b5839 100755
--- a/scripts/release/cut_release_branch.sh
+++ b/scripts/release/cut_release_branch.sh
@@ -16,7 +16,7 @@
# limitations under the License.
#
-# This script will update apache hudi (incubating) master branch with next release version
+# This script will update apache hudi master branch with next release version
# and cut release branch for current development version.
# Parse parameters passing into the script
@@ -34,7 +34,7 @@ function clean_up(){
}
if [[ $# -eq 1 && $1 = "-h" ]]; then
- echo "This script will update apache hudi(incubating) master branch with next release version and cut release branch for current development version."
+ echo "This script will update apache hudi master branch with next release version and cut release branch for current development version."
echo "There are 3 params required:"
echo "--release=\${CURRENT_RELEASE_VERSION}"
echo "--next_release=\${NEXT_RELEASE_VERSION}"
@@ -63,8 +63,8 @@ fi
MASTER_BRANCH=master
RELEASE_BRANCH=release-${RELEASE}
-GITHUB_REPO_URL=git@github.com:apache/incubator-hudi.git
-HUDI_ROOT_DIR=incubator-hudi
+GITHUB_REPO_URL=git@github.com:apache/hudi.git
+HUDI_ROOT_DIR=hudi
LOCAL_CLONE_DIR=hudi_release_${RELEASE}
echo "=====================Environment Variables====================="
@@ -119,9 +119,9 @@ else
exit
fi
-# Checkout and update release branch - Add incubating and remove snapshot
+# Checkout and update release branch
git checkout ${RELEASE_BRANCH}
-mvn versions:set -DnewVersion=${RELEASE}-incubating-rc${RC_NUM}
+mvn versions:set -DnewVersion=${RELEASE}-rc${RC_NUM}
echo "==================Current working branch======================="
echo ${RELEASE_BRANCH}
diff --git a/scripts/release/preparation_before_release.sh b/scripts/release/preparation_before_release.sh
index e827d5b..65e9aa8 100755
--- a/scripts/release/preparation_before_release.sh
+++ b/scripts/release/preparation_before_release.sh
@@ -22,8 +22,8 @@ set -e
LOCAL_SVN_DIR=local_svn_dir
ROOT_SVN_URL=https://dist.apache.org/repos/dist/
-DEV_REPO=dev/incubator
-RELEASE_REPO=release/incubator
+DEV_REPO=dev
+RELEASE_REPO=release
HUDI_REPO=hudi
cd ~
diff --git a/scripts/release/validate_staged_release.sh b/scripts/release/validate_staged_release.sh
index b90e5cf..aa0f9d7 100755
--- a/scripts/release/validate_staged_release.sh
+++ b/scripts/release/validate_staged_release.sh
@@ -31,7 +31,7 @@ fi
REDIRECT=' > /dev/null 2>&1'
if [[ $# -lt 2 ]]; then
- echo "This script will validate source release candidate published in dist for apache hudi(incubating)"
+ echo "This script will validate source release candidate published in dist for apache hudi"
echo "There are two params required:"
echo "--release=\${CURRENT_RELEASE_VERSION}"
echo "--rc_num=\${RC_NUM}"
@@ -65,11 +65,11 @@ rm -rf $WORK_DIR
mkdir $WORK_DIR
pushd $WORK_DIR
-# Checkout dist incubator repo
+# Checkout dist repo
LOCAL_SVN_DIR=local_svn_dir
ROOT_SVN_URL=https://dist.apache.org/repos/dist/
-DEV_REPO=dev/incubator
-#RELEASE_REPO=release/incubator
+DEV_REPO=dev
+#RELEASE_REPO=release
HUDI_REPO=hudi
rm -rf $LOCAL_SVN_DIR
@@ -77,23 +77,23 @@ mkdir $LOCAL_SVN_DIR
cd $LOCAL_SVN_DIR
(bash -c "svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO} $REDIRECT") || (echo -e "\t\t Unable to checkout ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO} . Please run with --verbose to get details\n" && exit -1)
-cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
-$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512
+cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-rc${RC_NUM}
+$SHASUM hudi-${RELEASE_VERSION}-rc${RC_NUM}.src.tgz > got.sha512
echo "Checking Checksum of Source Release"
-diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512
+diff -u hudi-${RELEASE_VERSION}-rc${RC_NUM}.src.tgz.sha512 got.sha512
echo -e "\t\tChecksum Check of Source Release - [OK]\n"
# Download KEYS file
-curl https://dist.apache.org/repos/dist/release/incubator/hudi/KEYS > ../KEYS
+curl https://dist.apache.org/repos/dist/release/hudi/KEYS > ../KEYS
# GPG Check
echo "Checking Signature"
-(bash -c "gpg --import ../KEYS $REDIRECT" && bash -c "gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT" && echo -e "\t\tSignature Check - [OK]\n") || (echo -e "\t\tSignature Check - [FAILED] - Run with --verbose to get details\n" && exit -1)
+(bash -c "gpg --import ../KEYS $REDIRECT" && bash -c "gpg --verify hudi-${RELEASE_VERSION}-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-rc${RC_NUM}.src.tgz $REDIRECT" && echo -e "\t\tSignature Check - [OK]\n") || (echo -e "\t\tSignature Check - [FAILED] - Run with --verbose to get details\n" && exit -1)
# Untar
-(bash -c "tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT") || (echo -e "\t\t Unable to untar hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz . Please run with --verbose to get details\n" && exit -1)
-cd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
+(bash -c "tar -zxf hudi-${RELEASE_VERSION}-rc${RC_NUM}.src.tgz $REDIRECT") || (echo -e "\t\t Unable to untar hudi-${RELEASE_VERSION}-rc${RC_NUM}.src.tgz . Please run with --verbose to get details\n" && exit -1)
+cd hudi-${RELEASE_VERSION}-rc${RC_NUM}
### BEGIN: Binary Files Check
echo "Checking for binary files in source release"
[hudi] 09/40: [HUDI-850] Avoid unnecessary listings in incremental
cleaning mode (#1576)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit ea48ca9b62a2f216460a0795ef9d519481510cd7
Author: Balaji Varadarajan <va...@uber.com>
AuthorDate: Fri May 1 21:37:21 2020 -0700
[HUDI-850] Avoid unnecessary listings in incremental cleaning mode (#1576)
---
.../java/org/apache/hudi/table/CleanHelper.java | 85 ++++++++++++++++------
.../java/org/apache/hudi/table/TestCleaner.java | 42 +++++------
2 files changed, 85 insertions(+), 42 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/CleanHelper.java b/hudi-client/src/main/java/org/apache/hudi/table/CleanHelper.java
index 3c73c7e..38eddd7 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/CleanHelper.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/CleanHelper.java
@@ -38,13 +38,13 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
-
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -90,34 +90,77 @@ public class CleanHelper<T extends HoodieRecordPayload<T>> implements Serializab
* @throws IOException when underlying file-system throws this exception
*/
public List<String> getPartitionPathsToClean(Option<HoodieInstant> newInstantToRetain) throws IOException {
- if (config.incrementalCleanerModeEnabled() && newInstantToRetain.isPresent()
- && (HoodieCleaningPolicy.KEEP_LATEST_COMMITS == config.getCleanerPolicy())) {
- Option<HoodieInstant> lastClean =
- hoodieTable.getCleanTimeline().filterCompletedInstants().lastInstant();
+ switch (config.getCleanerPolicy()) {
+ case KEEP_LATEST_COMMITS:
+ return getPartitionPathsForCleanByCommits(newInstantToRetain);
+ case KEEP_LATEST_FILE_VERSIONS:
+ return getPartitionPathsForFullCleaning();
+ default:
+ throw new IllegalStateException("Unknown Cleaner Policy");
+ }
+ }
+
+ /**
+ * Return partition paths for cleaning by commits mode.
+ * @param instantToRetain Earliest Instant to retain
+ * @return list of partitions
+ * @throws IOException
+ */
+ private List<String> getPartitionPathsForCleanByCommits(Option<HoodieInstant> instantToRetain) throws IOException {
+ if (!instantToRetain.isPresent()) {
+ LOG.info("No earliest commit to retain. No need to scan partitions !!");
+ return Collections.emptyList();
+ }
+
+ if (config.incrementalCleanerModeEnabled()) {
+ Option<HoodieInstant> lastClean = hoodieTable.getCleanTimeline().filterCompletedInstants().lastInstant();
if (lastClean.isPresent()) {
HoodieCleanMetadata cleanMetadata = AvroUtils
.deserializeHoodieCleanMetadata(hoodieTable.getActiveTimeline().getInstantDetails(lastClean.get()).get());
if ((cleanMetadata.getEarliestCommitToRetain() != null)
&& (cleanMetadata.getEarliestCommitToRetain().length() > 0)) {
- LOG.warn("Incremental Cleaning mode is enabled. Looking up partition-paths that have since changed "
- + "since last cleaned at " + cleanMetadata.getEarliestCommitToRetain()
- + ". New Instant to retain : " + newInstantToRetain);
- return hoodieTable.getCompletedCommitsTimeline().getInstants().filter(instant -> HoodieTimeline.compareTimestamps(instant.getTimestamp(), cleanMetadata.getEarliestCommitToRetain(),
- HoodieTimeline.GREATER_OR_EQUAL) && HoodieTimeline.compareTimestamps(instant.getTimestamp(),
- newInstantToRetain.get().getTimestamp(), HoodieTimeline.LESSER)).flatMap(instant -> {
- try {
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(hoodieTable.getActiveTimeline().getInstantDetails(instant).get(), HoodieCommitMetadata.class);
- return commitMetadata.getPartitionToWriteStats().keySet().stream();
- } catch (IOException e) {
- throw new HoodieIOException(e.getMessage(), e);
- }
- }).distinct().collect(Collectors.toList());
+ return getPartitionPathsForIncrementalCleaning(cleanMetadata, instantToRetain);
}
}
}
- // Otherwise go to brute force mode of scanning all partitions
- return FSUtils.getAllPartitionPaths(hoodieTable.getMetaClient().getFs(),
- hoodieTable.getMetaClient().getBasePath(), config.shouldAssumeDatePartitioning());
+ return getPartitionPathsForFullCleaning();
+ }
+
+ /**
+ * Use Incremental Mode for finding partition paths.
+ * @param cleanMetadata
+ * @param newInstantToRetain
+ * @return
+ */
+ private List<String> getPartitionPathsForIncrementalCleaning(HoodieCleanMetadata cleanMetadata,
+ Option<HoodieInstant> newInstantToRetain) {
+ LOG.warn("Incremental Cleaning mode is enabled. Looking up partition-paths that have since changed "
+ + "since last cleaned at " + cleanMetadata.getEarliestCommitToRetain()
+ + ". New Instant to retain : " + newInstantToRetain);
+ return hoodieTable.getCompletedCommitsTimeline().getInstants().filter(
+ instant -> HoodieTimeline.compareTimestamps(instant.getTimestamp(), cleanMetadata.getEarliestCommitToRetain(),
+ HoodieTimeline.GREATER_OR_EQUAL) && HoodieTimeline.compareTimestamps(instant.getTimestamp(),
+ newInstantToRetain.get().getTimestamp(), HoodieTimeline.LESSER)).flatMap(instant -> {
+ try {
+ HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
+ .fromBytes(hoodieTable.getActiveTimeline().getInstantDetails(instant).get(),
+ HoodieCommitMetadata.class);
+ return commitMetadata.getPartitionToWriteStats().keySet().stream();
+ } catch (IOException e) {
+ throw new HoodieIOException(e.getMessage(), e);
+ }
+ }).distinct().collect(Collectors.toList());
+ }
+
+ /**
+ * Scan and list all paritions for cleaning.
+ * @return all partitions paths for the dataset.
+ * @throws IOException
+ */
+ private List<String> getPartitionPathsForFullCleaning() throws IOException {
+ // Go to brute force mode of scanning all partitions
+ return FSUtils.getAllPartitionPaths(hoodieTable.getMetaClient().getFs(), hoodieTable.getMetaClient().getBasePath(),
+ config.shouldAssumeDatePartitioning());
}
/**
diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
index f6ad230..dee545a 100644
--- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
+++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
@@ -108,7 +108,8 @@ public class TestCleaner extends TestHoodieClientBase {
*/
private String insertFirstBigBatchForClientCleanerTest(HoodieWriteConfig cfg, HoodieWriteClient client,
Function2<List<HoodieRecord>, String, Integer> recordGenFunction,
- Function3<JavaRDD<WriteStatus>, HoodieWriteClient, JavaRDD<HoodieRecord>, String> insertFn) throws Exception {
+ Function3<JavaRDD<WriteStatus>, HoodieWriteClient, JavaRDD<HoodieRecord>, String> insertFn,
+ HoodieCleaningPolicy cleaningPolicy) throws Exception {
/*
* do a big insert (this is basically same as insert part of upsert, just adding it here so we can catch breakages
@@ -131,10 +132,15 @@ public class TestCleaner extends TestHoodieClientBase {
HoodieTable table = HoodieTable.getHoodieTable(metaClient, client.getConfig(), jsc);
assertFalse(table.getCompletedCommitsTimeline().empty());
- String commitTime = table.getCompletedCommitsTimeline().getInstants().findFirst().get().getTimestamp();
- assertFalse(table.getCompletedCleanTimeline().empty());
- assertEquals("The clean instant should be the same as the commit instant", commitTime,
- table.getCompletedCleanTimeline().getInstants().findFirst().get().getTimestamp());
+ if (cleaningPolicy.equals(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)) {
+ // We no longer write empty cleaner plans when there are not enough commits present
+ assertTrue(table.getCompletedCleanTimeline().empty());
+ } else {
+ String instantTime = table.getCompletedCommitsTimeline().getInstants().findFirst().get().getTimestamp();
+ assertFalse(table.getCompletedCleanTimeline().empty());
+ assertEquals("The clean instant should be the same as the commit instant", instantTime,
+ table.getCompletedCleanTimeline().getInstants().findFirst().get().getTimestamp());
+ }
HoodieIndex index = HoodieIndex.createIndex(cfg, jsc);
List<HoodieRecord> taggedRecords = index.tagLocation(jsc.parallelize(records, 1), jsc, table).collect();
@@ -205,7 +211,8 @@ public class TestCleaner extends TestHoodieClientBase {
final Function2<List<HoodieRecord>, String, Integer> recordUpsertGenWrappedFunction =
generateWrapRecordsFn(isPreppedAPI, cfg, dataGen::generateUniqueUpdates);
- insertFirstBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn);
+ insertFirstBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn,
+ HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS);
Map<HoodieFileGroupId, FileSlice> compactionFileIdToLatestFileSlice = new HashMap<>();
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -353,7 +360,7 @@ public class TestCleaner extends TestHoodieClientBase {
int maxCommits = 3; // keep upto 3 commits from the past
HoodieWriteConfig cfg = getConfigBuilder()
.withCompactionConfig(HoodieCompactionConfig.newBuilder()
- .withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS).retainCommits(maxCommits).build())
+ .withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS).retainCommits(maxCommits).build())
.withParallelism(1, 1).withBulkInsertParallelism(1).withFinalizeWriteParallelism(1)
.withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true).build())
.build();
@@ -365,7 +372,8 @@ public class TestCleaner extends TestHoodieClientBase {
final Function2<List<HoodieRecord>, String, Integer> recordUpsertGenWrappedFunction =
generateWrapRecordsFn(isPreppedAPI, cfg, dataGen::generateUniqueUpdates);
- insertFirstBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn);
+ insertFirstBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn,
+ HoodieCleaningPolicy.KEEP_LATEST_COMMITS);
// Keep doing some writes and clean inline. Make sure we have expected number of files remaining.
HoodieTestUtils.monotonicIncreasingCommitTimestamps(8, 1).forEach(newCommitTime -> {
@@ -380,7 +388,9 @@ public class TestCleaner extends TestHoodieClientBase {
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table1 = HoodieTable.getHoodieTable(metaClient, cfg, jsc);
HoodieTimeline activeTimeline = table1.getCompletedCommitsTimeline();
- Option<HoodieInstant> earliestRetainedCommit = activeTimeline.nthFromLastInstant(maxCommits - 1);
+ // NOTE: See CleanPlanner#getFilesToCleanKeepingLatestCommits. We explicitly keep one commit before earliest
+ // commit
+ Option<HoodieInstant> earliestRetainedCommit = activeTimeline.nthFromLastInstant(maxCommits);
Set<HoodieInstant> acceptableCommits = activeTimeline.getInstants().collect(Collectors.toSet());
if (earliestRetainedCommit.isPresent()) {
acceptableCommits
@@ -753,12 +763,7 @@ public class TestCleaner extends TestHoodieClientBase {
metaClient = HoodieTableMetaClient.reload(metaClient);
List<HoodieCleanStat> hoodieCleanStatsOne = runCleaner(config, simulateFailureRetry);
- assertEquals("Must not clean any files", 0,
- getCleanStat(hoodieCleanStatsOne, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles()
- .size());
- assertEquals("Must not clean any files", 0,
- getCleanStat(hoodieCleanStatsOne, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles()
- .size());
+ assertEquals("Must not scan any partitions and clean any files", 0, hoodieCleanStatsOne.size());
assertTrue(HoodieTestUtils.doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000",
file1P0C0));
assertTrue(HoodieTestUtils.doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000",
@@ -788,12 +793,7 @@ public class TestCleaner extends TestHoodieClientBase {
new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, "001"),
Option.of(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
List<HoodieCleanStat> hoodieCleanStatsTwo = runCleaner(config, simulateFailureRetry);
- assertEquals("Must not clean any files", 0,
- getCleanStat(hoodieCleanStatsTwo, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles()
- .size());
- assertEquals("Must not clean any files", 0,
- getCleanStat(hoodieCleanStatsTwo, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles()
- .size());
+ assertEquals("Must not scan any partitions and clean any files", 0, hoodieCleanStatsTwo.size());
assertTrue(HoodieTestUtils.doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001",
file2P0C1));
assertTrue(HoodieTestUtils.doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001",
[hudi] 12/40: Add constructor to HoodieROTablePathFilter (#1413)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d535381e3f98008cd00161304b7a4e59644f5da0
Author: bschell <bd...@gmail.com>
AuthorDate: Mon Mar 16 15:19:16 2020 -0700
Add constructor to HoodieROTablePathFilter (#1413)
Allows HoodieROTablePathFilter to accept a configuration for
initializing the filesystem. This fixes a bug with Presto's use of this
pathfilter.
Co-authored-by: Brandon Scheller <bs...@amazon.com>
---
.../org/apache/hudi/hadoop/HoodieROTablePathFilter.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
index 66ec864..d879a2f 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
@@ -63,12 +63,21 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
*/
private HashSet<String> nonHoodiePathCache;
+ /**
+ * Hadoop configurations for the FileSystem.
+ */
+ private Configuration conf;
private transient FileSystem fs;
public HoodieROTablePathFilter() {
- hoodiePathCache = new HashMap<>();
- nonHoodiePathCache = new HashSet<>();
+ this(new Configuration());
+ }
+
+ public HoodieROTablePathFilter(Configuration conf) {
+ this.hoodiePathCache = new HashMap<>();
+ this.nonHoodiePathCache = new HashSet<>();
+ this.conf = conf;
}
/**
@@ -93,7 +102,7 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
Path folder = null;
try {
if (fs == null) {
- fs = path.getFileSystem(new Configuration());
+ fs = path.getFileSystem(conf);
}
// Assumes path is a file
[hudi] 04/40: [HUDI-629]: Replace Guava's Hashing with an
equivalent in NumericUtils.java (#1350)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit cebce61a3496f8788f5b96d6aedce0a8394fc325
Author: Suneel Marthi <sm...@apache.org>
AuthorDate: Fri Mar 13 20:28:05 2020 -0400
[HUDI-629]: Replace Guava's Hashing with an equivalent in NumericUtils.java (#1350)
* [HUDI-629]: Replace Guava's Hashing with an equivalent in NumericUtils.java
---
hudi-client/pom.xml | 5 --
.../apache/hudi/client/CompactionAdminClient.java | 22 +++----
.../org/apache/hudi/client/HoodieCleanClient.java | 4 +-
.../org/apache/hudi/client/HoodieWriteClient.java | 14 ++---
.../apache/hudi/config/HoodieCompactionConfig.java | 12 ++--
.../org/apache/hudi/config/HoodieWriteConfig.java | 1 -
.../bloom/BucketizedBloomCheckPartitioner.java | 5 +-
.../apache/hudi/table/HoodieCopyOnWriteTable.java | 6 +-
.../apache/hudi/table/HoodieMergeOnReadTable.java | 8 +--
.../compact/HoodieMergeOnReadTableCompactor.java | 10 ++--
.../apache/hudi/table/rollback/RollbackHelper.java | 4 +-
.../apache/hudi/config/TestHoodieWriteConfig.java | 2 +-
.../hudi/common/model/TimelineLayoutVersion.java | 6 +-
.../hudi/common/table/HoodieTableMetaClient.java | 8 +--
.../hudi/common/table/log/HoodieLogFileReader.java | 7 +--
.../table/timeline/HoodieActiveTimeline.java | 48 +++++++--------
.../table/view/AbstractTableFileSystemView.java | 23 ++++---
.../table/view/FileSystemViewStorageConfig.java | 5 +-
.../table/view/HoodieTableFileSystemView.java | 19 +++---
.../view/RemoteHoodieTableFileSystemView.java | 12 ++--
.../table/view/RocksDbBasedFileSystemView.java | 6 +-
.../org/apache/hudi/common/util/AvroUtils.java | 3 +-
.../java/org/apache/hudi/common/util/FSUtils.java | 5 +-
.../hudi/common/util/FailSafeConsistencyGuard.java | 7 +--
.../org/apache/hudi/common/util/NumericUtils.java | 30 ++++++++++
.../org/apache/hudi/common/util/RocksDBDAO.java | 17 +++---
.../apache/hudi/common/util/ValidationUtils.java | 70 ++++++++++++++++++++++
.../common/util/queue/BoundedInMemoryQueue.java | 4 +-
.../hudi/common/versioning/MetadataMigrator.java | 7 +--
.../versioning/clean/CleanV1MigrationHandler.java | 9 +--
.../versioning/clean/CleanV2MigrationHandler.java | 4 +-
.../compaction/CompactionV1MigrationHandler.java | 11 ++--
.../compaction/CompactionV2MigrationHandler.java | 11 ++--
.../hudi/common/minicluster/HdfsTestService.java | 4 +-
.../common/minicluster/ZookeeperTestService.java | 9 +--
.../hudi/common/table/log/TestHoodieLogFormat.java | 2 +-
.../table/view/TestIncrementalFSViewSync.java | 16 ++---
.../apache/hudi/common/util/TestNumericUtils.java | 26 ++++++++
.../realtime/HoodieParquetRealtimeInputFormat.java | 4 +-
hudi-hive/pom.xml | 5 --
.../org/apache/hudi/hive/HoodieHiveClient.java | 4 +-
.../hudi/hive/MultiPartKeysValueExtractor.java | 4 +-
.../org/apache/hudi/hive/SchemaDifference.java | 2 +-
.../org/apache/hudi/hive/util/HiveTestService.java | 8 +--
hudi-integ-test/pom.xml | 7 ---
.../timeline/service/FileSystemViewHandler.java | 4 +-
.../hudi/utilities/HoodieWithTimelineServer.java | 4 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 4 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 10 ++--
.../deltastreamer/HoodieDeltaStreamer.java | 4 +-
.../sources/helpers/IncrSourceHelper.java | 14 +++--
51 files changed, 308 insertions(+), 228 deletions(-)
diff --git a/hudi-client/pom.xml b/hudi-client/pom.xml
index a91a19f..7b665de 100644
--- a/hudi-client/pom.xml
+++ b/hudi-client/pom.xml
@@ -119,11 +119,6 @@
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<scope>test</scope>
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/CompactionAdminClient.java b/hudi-client/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
index 713fed4..7d2d664 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
@@ -36,13 +36,13 @@ import org.apache.hudi.common.util.AvroUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.table.compact.OperationResult;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
@@ -138,7 +138,7 @@ public class CompactionAdminClient extends AbstractHoodieClient {
// TODO: Add a rollback instant but for compaction
HoodieInstant instant = new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, compactionInstant);
boolean deleted = metaClient.getFs().delete(new Path(metaClient.getMetaPath(), instant.getFileName()), false);
- Preconditions.checkArgument(deleted, "Unable to delete compaction instant.");
+ ValidationUtils.checkArgument(deleted, "Unable to delete compaction instant.");
}
return res;
}
@@ -247,7 +247,7 @@ public class CompactionAdminClient extends AbstractHoodieClient {
List<HoodieLogFile> logFilesToBeMoved =
merged.getLogFiles().filter(lf -> lf.getLogVersion() > maxVersion).collect(Collectors.toList());
return logFilesToBeMoved.stream().map(lf -> {
- Preconditions.checkArgument(lf.getLogVersion() - maxVersion > 0, "Expect new log version to be sane");
+ ValidationUtils.checkArgument(lf.getLogVersion() - maxVersion > 0, "Expect new log version to be sane");
HoodieLogFile newLogFile = new HoodieLogFile(new Path(lf.getPath().getParent(),
FSUtils.makeLogFileName(lf.getFileId(), "." + FSUtils.getFileExtensionFromLog(lf.getPath()),
compactionInstant, lf.getLogVersion() - maxVersion, HoodieLogFormat.UNKNOWN_WRITE_TOKEN)));
@@ -266,9 +266,9 @@ public class CompactionAdminClient extends AbstractHoodieClient {
protected static void renameLogFile(HoodieTableMetaClient metaClient, HoodieLogFile oldLogFile,
HoodieLogFile newLogFile) throws IOException {
FileStatus[] statuses = metaClient.getFs().listStatus(oldLogFile.getPath());
- Preconditions.checkArgument(statuses.length == 1, "Only one status must be present");
- Preconditions.checkArgument(statuses[0].isFile(), "Source File must exist");
- Preconditions.checkArgument(oldLogFile.getPath().getParent().equals(newLogFile.getPath().getParent()),
+ ValidationUtils.checkArgument(statuses.length == 1, "Only one status must be present");
+ ValidationUtils.checkArgument(statuses[0].isFile(), "Source File must exist");
+ ValidationUtils.checkArgument(oldLogFile.getPath().getParent().equals(newLogFile.getPath().getParent()),
"Log file must only be moved within the parent directory");
metaClient.getFs().rename(oldLogFile.getPath(), newLogFile.getPath());
}
@@ -300,9 +300,9 @@ public class CompactionAdminClient extends AbstractHoodieClient {
new Path(FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()),
new Path(operation.getDataFileName().get())))
.getPath().toString();
- Preconditions.checkArgument(df.isPresent(),
+ ValidationUtils.checkArgument(df.isPresent(),
"Data File must be present. File Slice was : " + fs + ", operation :" + operation);
- Preconditions.checkArgument(df.get().getPath().equals(expPath),
+ ValidationUtils.checkArgument(df.get().getPath().equals(expPath),
"Base Path in operation is specified as " + expPath + " but got path " + df.get().getPath());
}
Set<HoodieLogFile> logFilesInFileSlice = fs.getLogFiles().collect(Collectors.toSet());
@@ -310,7 +310,7 @@ public class CompactionAdminClient extends AbstractHoodieClient {
try {
FileStatus[] fileStatuses = metaClient.getFs().listStatus(new Path(
FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()), new Path(dp)));
- Preconditions.checkArgument(fileStatuses.length == 1, "Expect only 1 file-status");
+ ValidationUtils.checkArgument(fileStatuses.length == 1, "Expect only 1 file-status");
return new HoodieLogFile(fileStatuses[0]);
} catch (FileNotFoundException fe) {
throw new CompactionValidationException(fe.getMessage());
@@ -320,12 +320,12 @@ public class CompactionAdminClient extends AbstractHoodieClient {
}).collect(Collectors.toSet());
Set<HoodieLogFile> missing = logFilesInCompactionOp.stream().filter(lf -> !logFilesInFileSlice.contains(lf))
.collect(Collectors.toSet());
- Preconditions.checkArgument(missing.isEmpty(),
+ ValidationUtils.checkArgument(missing.isEmpty(),
"All log files specified in compaction operation is not present. Missing :" + missing + ", Exp :"
+ logFilesInCompactionOp + ", Got :" + logFilesInFileSlice);
Set<HoodieLogFile> diff = logFilesInFileSlice.stream().filter(lf -> !logFilesInCompactionOp.contains(lf))
.collect(Collectors.toSet());
- Preconditions.checkArgument(diff.stream().allMatch(lf -> lf.getBaseCommitTime().equals(compactionInstant)),
+ ValidationUtils.checkArgument(diff.stream().allMatch(lf -> lf.getBaseCommitTime().equals(compactionInstant)),
"There are some log-files which are neither specified in compaction plan "
+ "nor present after compaction request instant. Some of these :" + diff);
} else {
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
index 019d264..15ba2a7 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
@@ -31,13 +31,13 @@ import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.AvroUtils;
import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.table.HoodieTable;
import com.codahale.metrics.Timer;
-import com.google.common.base.Preconditions;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaSparkContext;
@@ -147,7 +147,7 @@ public class HoodieCleanClient<T extends HoodieRecordPayload> extends AbstractHo
private HoodieCleanMetadata runClean(HoodieTable<T> table, HoodieInstant cleanInstant,
HoodieCleanerPlan cleanerPlan) {
- Preconditions.checkArgument(
+ ValidationUtils.checkArgument(
cleanInstant.getState().equals(State.REQUESTED) || cleanInstant.getState().equals(State.INFLIGHT));
try {
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
index 40be544..e201487 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
@@ -42,6 +42,7 @@ import org.apache.hudi.common.util.AvroUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieCommitException;
@@ -61,7 +62,6 @@ import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
import com.codahale.metrics.Timer;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -580,7 +580,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
}
// Cannot allow savepoint time on a commit that could have been cleaned
- Preconditions.checkArgument(
+ ValidationUtils.checkArgument(
HoodieTimeline.compareTimestamps(commitTime, lastCommitRetained, HoodieTimeline.GREATER_OR_EQUAL),
"Could not savepoint commit " + commitTime + " as this is beyond the lookup window " + lastCommitRetained);
@@ -696,8 +696,8 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
// Make sure the rollback was successful
Option<HoodieInstant> lastInstant =
activeTimeline.reload().getCommitsAndCompactionTimeline().filterCompletedAndCompactionInstants().lastInstant();
- Preconditions.checkArgument(lastInstant.isPresent());
- Preconditions.checkArgument(lastInstant.get().getTimestamp().equals(savepointTime),
+ ValidationUtils.checkArgument(lastInstant.isPresent());
+ ValidationUtils.checkArgument(lastInstant.get().getTimestamp().equals(savepointTime),
savepointTime + "is not the last commit after rolling back " + commitsToRollback + ", last commit was "
+ lastInstant.get().getTimestamp());
return true;
@@ -868,7 +868,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
HoodieTableMetaClient metaClient = createMetaClient(true);
// if there are pending compactions, their instantTime must not be greater than that of this instant time
metaClient.getActiveTimeline().filterPendingCompactionTimeline().lastInstant().ifPresent(latestPending ->
- Preconditions.checkArgument(
+ ValidationUtils.checkArgument(
HoodieTimeline.compareTimestamps(latestPending.getTimestamp(), instantTime, HoodieTimeline.LESSER),
"Latest pending compaction instant time must be earlier than this instant time. Latest Compaction :"
+ latestPending + ", Ingesting at " + instantTime));
@@ -901,7 +901,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
HoodieTableMetaClient metaClient = createMetaClient(true);
// if there are inflight writes, their instantTime must not be less than that of compaction instant time
metaClient.getCommitsTimeline().filterPendingExcludingCompaction().firstInstant().ifPresent(earliestInflight -> {
- Preconditions.checkArgument(
+ ValidationUtils.checkArgument(
HoodieTimeline.compareTimestamps(earliestInflight.getTimestamp(), instantTime, HoodieTimeline.GREATER),
"Earliest write inflight instant time must be later than compaction time. Earliest :" + earliestInflight
+ ", Compaction scheduled at " + instantTime);
@@ -911,7 +911,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
.getActiveTimeline().getCommitsAndCompactionTimeline().getInstants().filter(instant -> HoodieTimeline
.compareTimestamps(instant.getTimestamp(), instantTime, HoodieTimeline.GREATER_OR_EQUAL))
.collect(Collectors.toList());
- Preconditions.checkArgument(conflictingInstants.isEmpty(),
+ ValidationUtils.checkArgument(conflictingInstants.isEmpty(),
"Following instants have timestamps >= compactionInstant (" + instantTime + ") Instants :"
+ conflictingInstants);
HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
index 180dba1..074ea78 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
@@ -20,11 +20,10 @@ package org.apache.hudi.config;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.table.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.compact.strategy.LogFileSizeBasedCompactionStrategy;
-import com.google.common.base.Preconditions;
-
import javax.annotation.concurrent.Immutable;
import java.io.File;
@@ -121,12 +120,9 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig {
private final Properties props = new Properties();
public Builder fromFile(File propertiesFile) throws IOException {
- FileReader reader = new FileReader(propertiesFile);
- try {
+ try (FileReader reader = new FileReader(propertiesFile)) {
this.props.load(reader);
return this;
- } finally {
- reader.close();
}
}
@@ -292,8 +288,8 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig {
int maxInstantsToKeep = Integer.parseInt(props.getProperty(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP));
int cleanerCommitsRetained =
Integer.parseInt(props.getProperty(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP));
- Preconditions.checkArgument(maxInstantsToKeep > minInstantsToKeep);
- Preconditions.checkArgument(minInstantsToKeep > cleanerCommitsRetained,
+ ValidationUtils.checkArgument(maxInstantsToKeep > minInstantsToKeep);
+ ValidationUtils.checkArgument(minInstantsToKeep > cleanerCommitsRetained,
String.format(
"Increase %s=%d to be greater than %s=%d. Otherwise, there is risk of incremental pull "
+ "missing data from few instants.",
diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 489bdb2..f88d96a 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -96,7 +96,6 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
private static final String MAX_CONSISTENCY_CHECKS_PROP = "hoodie.consistency.check.max_checks";
private static int DEFAULT_MAX_CONSISTENCY_CHECKS = 7;
-
private ConsistencyGuardConfig consistencyGuardConfig;
// Hoodie Write Client transparently rewrites File System View config when embedded mode is enabled
diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java
index 17b7506..798120c 100644
--- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java
+++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java
@@ -18,14 +18,13 @@
package org.apache.hudi.index.bloom;
+import org.apache.hudi.common.util.NumericUtils;
import org.apache.hudi.common.util.collection.Pair;
-import com.google.common.hash.Hashing;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.Partitioner;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -144,7 +143,7 @@ public class BucketizedBloomCheckPartitioner extends Partitioner {
@Override
public int getPartition(Object key) {
final Pair<String, String> parts = (Pair<String, String>) key;
- final long hashOfKey = Hashing.md5().hashString(parts.getRight(), StandardCharsets.UTF_8).asLong();
+ final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", parts.getRight());
final List<Integer> candidatePartitions = fileGroupToPartitions.get(parts.getLeft());
final int idx = (int) Math.floorMod(hashOfKey, candidatePartitions.size());
assert idx >= 0;
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
index 4efc0e6..82b08b7 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
@@ -37,6 +37,7 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.common.util.NumericUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor;
@@ -52,7 +53,6 @@ import org.apache.hudi.execution.SparkBoundedInMemoryExecutor;
import org.apache.hudi.io.HoodieCreateHandle;
import org.apache.hudi.io.HoodieMergeHandle;
-import com.google.common.hash.Hashing;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileSystem;
@@ -72,7 +72,6 @@ import org.apache.spark.api.java.function.PairFlatMapFunction;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -732,8 +731,7 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
// pick the target bucket to use based on the weights.
double totalWeight = 0.0;
final long totalInserts = Math.max(1, globalStat.getNumInserts());
- final long hashOfKey =
- Hashing.md5().hashString(keyLocation._1().getRecordKey(), StandardCharsets.UTF_8).asLong();
+ final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", keyLocation._1().getRecordKey());
final double r = 1.0 * Math.floorMod(hashOfKey, totalInserts) / totalInserts;
for (InsertBucket insertBucket : targetBuckets) {
totalWeight += insertBucket.weight;
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java
index c36cdf4..50d41b3 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java
@@ -34,6 +34,7 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieCompactionException;
import org.apache.hudi.exception.HoodieIOException;
@@ -42,7 +43,6 @@ import org.apache.hudi.execution.MergeOnReadLazyInsertIterable;
import org.apache.hudi.io.HoodieAppendHandle;
import org.apache.hudi.table.compact.HoodieMergeOnReadTableCompactor;
-import com.google.common.base.Preconditions;
import org.apache.hudi.table.rollback.RollbackHelper;
import org.apache.hudi.table.rollback.RollbackRequest;
import org.apache.log4j.LogManager;
@@ -422,7 +422,7 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends Hoodi
private List<RollbackRequest> generateAppendRollbackBlocksAction(String partitionPath, HoodieInstant rollbackInstant,
HoodieCommitMetadata commitMetadata) {
- Preconditions.checkArgument(rollbackInstant.getAction().equals(HoodieTimeline.DELTA_COMMIT_ACTION));
+ ValidationUtils.checkArgument(rollbackInstant.getAction().equals(HoodieTimeline.DELTA_COMMIT_ACTION));
// wStat.getPrevCommit() might not give the right commit time in the following
// scenario : If a compaction was scheduled, the new commitTime associated with the requested compaction will be
@@ -439,9 +439,9 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends Hoodi
if (validForRollback) {
// For sanity, log instant time can never be less than base-commit on which we are rolling back
- Preconditions
+ ValidationUtils
.checkArgument(HoodieTimeline.compareTimestamps(fileIdToBaseCommitTimeForLogMap.get(wStat.getFileId()),
- rollbackInstant.getTimestamp(), HoodieTimeline.LESSER_OR_EQUAL));
+ rollbackInstant.getTimestamp(), HoodieTimeline.LESSER_OR_EQUAL));
}
return validForRollback && HoodieTimeline.compareTimestamps(fileIdToBaseCommitTimeForLogMap.get(
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java b/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
index 4795e90..e1d2706 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
@@ -35,14 +35,13 @@ import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.HoodieAvroUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.HoodieCopyOnWriteTable;
import org.apache.hudi.table.HoodieTable;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileSystem;
@@ -56,6 +55,7 @@ import org.apache.spark.util.AccumulatorV2;
import org.apache.spark.util.LongAccumulator;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -125,7 +125,7 @@ public class HoodieMergeOnReadTableCompactor implements HoodieCompactor {
config.getCompactionReverseLogReadEnabled(), config.getMaxDFSStreamBufferSize(),
config.getSpillableMapBasePath());
if (!scanner.iterator().hasNext()) {
- return Lists.<WriteStatus>newArrayList();
+ return new ArrayList<>();
}
Option<HoodieBaseFile> oldDataFileOpt =
@@ -169,7 +169,7 @@ public class HoodieMergeOnReadTableCompactor implements HoodieCompactor {
jsc.sc().register(totalLogFiles);
jsc.sc().register(totalFileSlices);
- Preconditions.checkArgument(hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ,
+ ValidationUtils.checkArgument(hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ,
"Can only compact table of type " + HoodieTableType.MERGE_ON_READ + " and not "
+ hoodieTable.getMetaClient().getTableType().name());
@@ -214,7 +214,7 @@ public class HoodieMergeOnReadTableCompactor implements HoodieCompactor {
// compactions only
HoodieCompactionPlan compactionPlan = config.getCompactionStrategy().generateCompactionPlan(config, operations,
CompactionUtils.getAllPendingCompactionPlans(metaClient).stream().map(Pair::getValue).collect(toList()));
- Preconditions.checkArgument(
+ ValidationUtils.checkArgument(
compactionPlan.getOperations().stream().noneMatch(
op -> fgIdsInPendingCompactions.contains(new HoodieFileGroupId(op.getPartitionPath(), op.getFileId()))),
"Bad Compaction Plan. FileId MUST NOT have multiple pending compactions. "
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java b/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
index e559536..901c51e 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
@@ -28,10 +28,10 @@ import org.apache.hudi.common.table.log.block.HoodieCommandBlock.HoodieCommandBl
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieRollbackException;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PathFilter;
@@ -147,7 +147,7 @@ public class RollbackHelper implements Serializable {
* @return Merged HoodieRollbackStat
*/
private HoodieRollbackStat mergeRollbackStat(HoodieRollbackStat stat1, HoodieRollbackStat stat2) {
- Preconditions.checkArgument(stat1.getPartitionPath().equals(stat2.getPartitionPath()));
+ ValidationUtils.checkArgument(stat1.getPartitionPath().equals(stat2.getPartitionPath()));
final List<String> successDeleteFiles = new ArrayList<>();
final List<String> failedDeleteFiles = new ArrayList<>();
final Map<FileStatus, Long> commandBlocksCount = new HashMap<>();
diff --git a/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
index 80b59a0..0d9c59d 100644
--- a/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
+++ b/hudi-client/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
@@ -37,7 +37,7 @@ public class TestHoodieWriteConfig {
@Test
public void testPropertyLoading() throws IOException {
Builder builder = HoodieWriteConfig.newBuilder().withPath("/tmp");
- Map<String, String> params = new HashMap<>();
+ Map<String, String> params = new HashMap<>(3);
params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP, "1");
params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP, "5");
params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP, "2");
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/TimelineLayoutVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/model/TimelineLayoutVersion.java
index cd45eb3..d05d843 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/TimelineLayoutVersion.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/TimelineLayoutVersion.java
@@ -18,7 +18,7 @@
package org.apache.hudi.common.model;
-import com.google.common.base.Preconditions;
+import org.apache.hudi.common.util.ValidationUtils;
import java.io.Serializable;
import java.util.Objects;
@@ -37,8 +37,8 @@ public class TimelineLayoutVersion implements Serializable, Comparable<TimelineL
private Integer version;
public TimelineLayoutVersion(Integer version) {
- Preconditions.checkArgument(version <= CURR_VERSION);
- Preconditions.checkArgument(version >= VERSION_0);
+ ValidationUtils.checkArgument(version <= CURR_VERSION);
+ ValidationUtils.checkArgument(version >= VERSION_0);
this.version = version;
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index f66348e..5c56d17 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -26,14 +26,14 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.ConsistencyGuardConfig;
-import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.FailSafeConsistencyGuard;
+import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.NoOpConsistencyGuard;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.exception.HoodieException;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -119,7 +119,7 @@ public class HoodieTableMetaClient implements Serializable {
Option<TimelineLayoutVersion> tableConfigVersion = tableConfig.getTimelineLayoutVersion();
if (layoutVersion.isPresent() && tableConfigVersion.isPresent()) {
// Ensure layout version passed in config is not lower than the one seen in hoodie.properties
- Preconditions.checkArgument(layoutVersion.get().compareTo(tableConfigVersion.get()) >= 0,
+ ValidationUtils.checkArgument(layoutVersion.get().compareTo(tableConfigVersion.get()) >= 0,
"Layout Version defined in hoodie properties has higher version (" + tableConfigVersion.get()
+ ") than the one passed in config (" + layoutVersion.get() + ")");
}
@@ -233,7 +233,7 @@ public class HoodieTableMetaClient implements Serializable {
public HoodieWrapperFileSystem getFs() {
if (fs == null) {
FileSystem fileSystem = FSUtils.getFs(metaPath, hadoopConf.newCopy());
- Preconditions.checkArgument(!(fileSystem instanceof HoodieWrapperFileSystem),
+ ValidationUtils.checkArgument(!(fileSystem instanceof HoodieWrapperFileSystem),
"File System not expected to be that of HoodieWrapperFileSystem");
fs = new HoodieWrapperFileSystem(fileSystem,
consistencyGuardConfig.isConsistencyCheckEnabled()
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
index 40a5243..f6e9326 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
@@ -28,11 +28,11 @@ import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.CorruptedLogFileException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieNotSupportedException;
-import com.google.common.base.Preconditions;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.BufferedFSInputStream;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -61,7 +61,6 @@ class HoodieLogFileReader implements HoodieLogFormat.Reader {
private final HoodieLogFile logFile;
private static final byte[] MAGIC_BUFFER = new byte[6];
private final Schema readerSchema;
- private HoodieLogFormat.LogFormatVersion nextBlockVersion;
private boolean readBlockLazily;
private long reverseLogFilePosition;
private long lastReverseLogFilePosition;
@@ -145,13 +144,13 @@ class HoodieLogFileReader implements HoodieLogFormat.Reader {
}
// 2. Read the version for this log format
- this.nextBlockVersion = readVersion();
+ HoodieLogFormat.LogFormatVersion nextBlockVersion = readVersion();
// 3. Read the block type for a log block
if (nextBlockVersion.getVersion() != HoodieLogFormatVersion.DEFAULT_VERSION) {
type = inputStream.readInt();
- Preconditions.checkArgument(type < HoodieLogBlockType.values().length, "Invalid block byte type found " + type);
+ ValidationUtils.checkArgument(type < HoodieLogBlockType.values().length, "Invalid block byte type found " + type);
blockType = HoodieLogBlockType.values()[type];
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
index e5829f8..389314f 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
@@ -23,9 +23,9 @@ import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieIOException;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -140,7 +140,7 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
public void saveAsComplete(HoodieInstant instant, Option<byte[]> data) {
LOG.info("Marking instant complete " + instant);
- Preconditions.checkArgument(instant.isInflight(),
+ ValidationUtils.checkArgument(instant.isInflight(),
"Could not mark an already completed instant as complete again " + instant);
transitionState(instant, HoodieTimeline.getCompletedInstant(instant), data);
LOG.info("Completed " + instant);
@@ -155,18 +155,18 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
}
public void deleteInflight(HoodieInstant instant) {
- Preconditions.checkArgument(instant.isInflight());
+ ValidationUtils.checkArgument(instant.isInflight());
deleteInstantFile(instant);
}
public void deletePending(HoodieInstant instant) {
- Preconditions.checkArgument(!instant.isCompleted());
+ ValidationUtils.checkArgument(!instant.isCompleted());
deleteInstantFile(instant);
}
public void deleteCompactionRequested(HoodieInstant instant) {
- Preconditions.checkArgument(instant.isRequested());
- Preconditions.checkArgument(instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
+ ValidationUtils.checkArgument(instant.isRequested());
+ ValidationUtils.checkArgument(instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
deleteInstantFile(instant);
}
@@ -222,8 +222,8 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
* @return requested instant
*/
public HoodieInstant revertCompactionInflightToRequested(HoodieInstant inflightInstant) {
- Preconditions.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
- Preconditions.checkArgument(inflightInstant.isInflight());
+ ValidationUtils.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
+ ValidationUtils.checkArgument(inflightInstant.isInflight());
HoodieInstant requestedInstant =
new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, inflightInstant.getTimestamp());
if (metaClient.getTimelineLayoutVersion().isNullVersion()) {
@@ -242,8 +242,8 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
* @return inflight instant
*/
public HoodieInstant transitionCompactionRequestedToInflight(HoodieInstant requestedInstant) {
- Preconditions.checkArgument(requestedInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
- Preconditions.checkArgument(requestedInstant.isRequested());
+ ValidationUtils.checkArgument(requestedInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
+ ValidationUtils.checkArgument(requestedInstant.isRequested());
HoodieInstant inflightInstant =
new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, requestedInstant.getTimestamp());
transitionState(requestedInstant, inflightInstant, Option.empty());
@@ -258,8 +258,8 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
* @return commit instant
*/
public HoodieInstant transitionCompactionInflightToComplete(HoodieInstant inflightInstant, Option<byte[]> data) {
- Preconditions.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
- Preconditions.checkArgument(inflightInstant.isInflight());
+ ValidationUtils.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
+ ValidationUtils.checkArgument(inflightInstant.isInflight());
HoodieInstant commitInstant = new HoodieInstant(State.COMPLETED, COMMIT_ACTION, inflightInstant.getTimestamp());
transitionState(inflightInstant, commitInstant, data);
return commitInstant;
@@ -283,8 +283,8 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
* @return commit instant
*/
public HoodieInstant transitionCleanInflightToComplete(HoodieInstant inflightInstant, Option<byte[]> data) {
- Preconditions.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.CLEAN_ACTION));
- Preconditions.checkArgument(inflightInstant.isInflight());
+ ValidationUtils.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.CLEAN_ACTION));
+ ValidationUtils.checkArgument(inflightInstant.isInflight());
HoodieInstant commitInstant = new HoodieInstant(State.COMPLETED, CLEAN_ACTION, inflightInstant.getTimestamp());
// Then write to timeline
transitionState(inflightInstant, commitInstant, data);
@@ -299,15 +299,15 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
* @return commit instant
*/
public HoodieInstant transitionCleanRequestedToInflight(HoodieInstant requestedInstant, Option<byte[]> data) {
- Preconditions.checkArgument(requestedInstant.getAction().equals(HoodieTimeline.CLEAN_ACTION));
- Preconditions.checkArgument(requestedInstant.isRequested());
+ ValidationUtils.checkArgument(requestedInstant.getAction().equals(HoodieTimeline.CLEAN_ACTION));
+ ValidationUtils.checkArgument(requestedInstant.isRequested());
HoodieInstant inflight = new HoodieInstant(State.INFLIGHT, CLEAN_ACTION, requestedInstant.getTimestamp());
transitionState(requestedInstant, inflight, data);
return inflight;
}
private void transitionState(HoodieInstant fromInstant, HoodieInstant toInstant, Option<byte[]> data) {
- Preconditions.checkArgument(fromInstant.getTimestamp().equals(toInstant.getTimestamp()));
+ ValidationUtils.checkArgument(fromInstant.getTimestamp().equals(toInstant.getTimestamp()));
try {
if (metaClient.getTimelineLayoutVersion().isNullVersion()) {
// Re-create the .inflight file by opening a new file and write the commit metadata in
@@ -321,7 +321,7 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
} else {
// Ensures old state exists in timeline
LOG.info("Checking for file exists ?" + new Path(metaClient.getMetaPath(), fromInstant.getFileName()));
- Preconditions.checkArgument(metaClient.getFs().exists(new Path(metaClient.getMetaPath(),
+ ValidationUtils.checkArgument(metaClient.getFs().exists(new Path(metaClient.getMetaPath(),
fromInstant.getFileName())));
// Use Write Once to create Target File
createImmutableFileInPath(new Path(metaClient.getMetaPath(), toInstant.getFileName()), data);
@@ -333,7 +333,7 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
}
private void revertCompleteToInflight(HoodieInstant completed, HoodieInstant inflight) {
- Preconditions.checkArgument(completed.getTimestamp().equals(inflight.getTimestamp()));
+ ValidationUtils.checkArgument(completed.getTimestamp().equals(inflight.getTimestamp()));
Path inFlightCommitFilePath = new Path(metaClient.getMetaPath(), inflight.getFileName());
Path commitFilePath = new Path(metaClient.getMetaPath(), completed.getFileName());
try {
@@ -359,7 +359,7 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
}
boolean success = metaClient.getFs().delete(commitFilePath, false);
- Preconditions.checkArgument(success, "State Reverting failed");
+ ValidationUtils.checkArgument(success, "State Reverting failed");
}
} catch (IOException e) {
throw new HoodieIOException("Could not complete revert " + completed, e);
@@ -368,7 +368,7 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
public void transitionRequestedToInflight(HoodieInstant requested, Option<byte[]> content) {
HoodieInstant inflight = new HoodieInstant(State.INFLIGHT, requested.getAction(), requested.getTimestamp());
- Preconditions.checkArgument(requested.isRequested(), "Instant " + requested + " in wrong state");
+ ValidationUtils.checkArgument(requested.isRequested(), "Instant " + requested + " in wrong state");
transitionState(requested, inflight, content);
}
@@ -377,15 +377,15 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
}
public void saveToCompactionRequested(HoodieInstant instant, Option<byte[]> content, boolean overwrite) {
- Preconditions.checkArgument(instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
+ ValidationUtils.checkArgument(instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION));
// Write workload to auxiliary folder
createFileInAuxiliaryFolder(instant, content);
createFileInMetaPath(instant.getFileName(), content, overwrite);
}
public void saveToCleanRequested(HoodieInstant instant, Option<byte[]> content) {
- Preconditions.checkArgument(instant.getAction().equals(HoodieTimeline.CLEAN_ACTION));
- Preconditions.checkArgument(instant.getState().equals(State.REQUESTED));
+ ValidationUtils.checkArgument(instant.getAction().equals(HoodieTimeline.CLEAN_ACTION));
+ ValidationUtils.checkArgument(instant.getState().equals(State.REQUESTED));
// Plan is stored in meta path
createFileInMetaPath(instant.getFileName(), content, false);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
index ee28cdc..d8f77e4 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
@@ -32,10 +32,10 @@ import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
@@ -116,10 +116,9 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
long fgBuildTimeTakenMs = timer.endTimer();
timer.startTimer();
// Group by partition for efficient updates for both InMemory and DiskBased stuctures.
- fileGroups.stream().collect(Collectors.groupingBy(HoodieFileGroup::getPartitionPath)).entrySet().forEach(entry -> {
- String partition = entry.getKey();
+ fileGroups.stream().collect(Collectors.groupingBy(HoodieFileGroup::getPartitionPath)).forEach((partition, value) -> {
if (!isPartitionAvailableInStore(partition)) {
- storePartitionView(partition, entry.getValue());
+ storePartitionView(partition, value);
}
});
long storePartitionsTs = timer.endTimer();
@@ -209,7 +208,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
*/
private void ensurePartitionLoadedCorrectly(String partition) {
- Preconditions.checkArgument(!isClosed(), "View is already closed");
+ ValidationUtils.checkArgument(!isClosed(), "View is already closed");
// ensure we list files only once even in the face of concurrency
addedPartitions.computeIfAbsent(partition, (partitionPathStr) -> {
@@ -397,11 +396,9 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
public final Stream<HoodieBaseFile> getLatestBaseFilesInRange(List<String> commitsToReturn) {
try {
readLock.lock();
- return fetchAllStoredFileGroups().map(fileGroup -> {
- return Option.fromJavaOptional(
- fileGroup.getAllBaseFiles().filter(baseFile -> commitsToReturn.contains(baseFile.getCommitTime())
- && !isBaseFileDueToPendingCompaction(baseFile)).findFirst());
- }).filter(Option::isPresent).map(Option::get);
+ return fetchAllStoredFileGroups().map(fileGroup -> Option.fromJavaOptional(
+ fileGroup.getAllBaseFiles().filter(baseFile -> commitsToReturn.contains(baseFile.getCommitTime())
+ && !isBaseFileDueToPendingCompaction(baseFile)).findFirst())).filter(Option::isPresent).map(Option::get);
} finally {
readLock.unlock();
}
@@ -443,7 +440,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
String partitionPath = formatPartitionKey(partitionStr);
ensurePartitionLoadedCorrectly(partitionPath);
Option<FileSlice> fs = fetchLatestFileSlice(partitionPath, fileId);
- return fs.map(f -> filterBaseFileAfterPendingCompaction(f));
+ return fs.map(this::filterBaseFileAfterPendingCompaction);
} finally {
readLock.unlock();
}
@@ -480,7 +477,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
ensurePartitionLoadedCorrectly(partitionPath);
Stream<FileSlice> fileSliceStream = fetchLatestFileSlicesBeforeOrOn(partitionPath, maxCommitTime);
if (includeFileSlicesInPendingCompaction) {
- return fileSliceStream.map(fs -> filterBaseFileAfterPendingCompaction(fs));
+ return fileSliceStream.map(this::filterBaseFileAfterPendingCompaction);
} else {
return fileSliceStream.filter(fs -> !isPendingCompactionScheduledForFileId(fs.getFileGroupId()));
}
@@ -815,7 +812,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
/**
* Return Only Commits and Compaction timeline for building file-groups.
*
- * @return
+ * @return {@code HoodieTimeline}
*/
public HoodieTimeline getVisibleCommitsAndCompactionTimeline() {
return visibleCommitsAndCompactionTimeline;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
index 4cf6942..93c5507 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
@@ -18,10 +18,9 @@
package org.apache.hudi.common.table.view;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.DefaultHoodieConfig;
-import com.google.common.base.Preconditions;
-
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
@@ -192,7 +191,7 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig {
// Validations
FileSystemViewStorageType.valueOf(props.getProperty(FILESYSTEM_VIEW_STORAGE_TYPE));
FileSystemViewStorageType.valueOf(props.getProperty(FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE));
- Preconditions.checkArgument(Integer.parseInt(props.getProperty(FILESYSTEM_VIEW_REMOTE_PORT)) > 0);
+ ValidationUtils.checkArgument(Integer.parseInt(props.getProperty(FILESYSTEM_VIEW_REMOTE_PORT)) > 0);
return new FileSystemViewStorageConfig(props);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
index dd71124..1f7165b 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
@@ -25,15 +25,16 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.TableFileSystemView;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileStatus;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -134,15 +135,14 @@ public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystem
@Override
protected void resetPendingCompactionOperations(Stream<Pair<String, CompactionOperation>> operations) {
// Build fileId to Pending Compaction Instants
- this.fgIdToPendingCompaction = createFileIdToPendingCompactionMap(operations.map(entry -> {
- return Pair.of(entry.getValue().getFileGroupId(), Pair.of(entry.getKey(), entry.getValue()));
- }).collect(Collectors.toMap(Pair::getKey, Pair::getValue)));
+ this.fgIdToPendingCompaction = createFileIdToPendingCompactionMap(operations.map(entry ->
+ Pair.of(entry.getValue().getFileGroupId(), Pair.of(entry.getKey(), entry.getValue()))).collect(Collectors.toMap(Pair::getKey, Pair::getValue)));
}
@Override
protected void addPendingCompactionOperations(Stream<Pair<String, CompactionOperation>> operations) {
operations.forEach(opInstantPair -> {
- Preconditions.checkArgument(!fgIdToPendingCompaction.containsKey(opInstantPair.getValue().getFileGroupId()),
+ ValidationUtils.checkArgument(!fgIdToPendingCompaction.containsKey(opInstantPair.getValue().getFileGroupId()),
"Duplicate FileGroupId found in pending compaction operations. FgId :"
+ opInstantPair.getValue().getFileGroupId());
fgIdToPendingCompaction.put(opInstantPair.getValue().getFileGroupId(),
@@ -153,7 +153,7 @@ public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystem
@Override
protected void removePendingCompactionOperations(Stream<Pair<String, CompactionOperation>> operations) {
operations.forEach(opInstantPair -> {
- Preconditions.checkArgument(fgIdToPendingCompaction.containsKey(opInstantPair.getValue().getFileGroupId()),
+ ValidationUtils.checkArgument(fgIdToPendingCompaction.containsKey(opInstantPair.getValue().getFileGroupId()),
"Trying to remove a FileGroupId which is not found in pending compaction operations. FgId :"
+ opInstantPair.getValue().getFileGroupId());
fgIdToPendingCompaction.remove(opInstantPair.getValue().getFileGroupId());
@@ -166,8 +166,7 @@ public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystem
*/
@Override
Stream<HoodieFileGroup> fetchAllStoredFileGroups(String partition) {
- final List<HoodieFileGroup> fileGroups = new ArrayList<>();
- fileGroups.addAll(partitionToFileGroupsMap.get(partition));
+ final List<HoodieFileGroup> fileGroups = new ArrayList<>(partitionToFileGroupsMap.get(partition));
return fileGroups.stream();
}
@@ -200,9 +199,7 @@ public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystem
@Override
public Stream<HoodieFileGroup> fetchAllStoredFileGroups() {
- return partitionToFileGroupsMap.values().stream().flatMap(fg -> {
- return fg.stream();
- });
+ return partitionToFileGroupsMap.values().stream().flatMap(Collection::stream);
}
@Override
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
index 48e1c25..01010a2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
@@ -34,12 +34,12 @@ import org.apache.hudi.common.table.timeline.dto.InstantDTO;
import org.apache.hudi.common.table.timeline.dto.TimelineDTO;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieRemoteException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
import org.apache.http.client.fluent.Request;
import org.apache.http.client.fluent.Response;
import org.apache.http.client.utils.URIBuilder;
@@ -134,14 +134,12 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
private <T> T executeRequest(String requestPath, Map<String, String> queryParameters, TypeReference reference,
RequestMethod method) throws IOException {
- Preconditions.checkArgument(!closed, "View already closed");
+ ValidationUtils.checkArgument(!closed, "View already closed");
URIBuilder builder =
new URIBuilder().setHost(serverHost).setPort(serverPort).setPath(requestPath).setScheme("http");
- queryParameters.entrySet().stream().forEach(entry -> {
- builder.addParameter(entry.getKey(), entry.getValue());
- });
+ queryParameters.forEach(builder::addParameter);
// Adding mandatory parameters - Last instants affecting file-slice
timeline.lastInstant().ifPresent(instant -> builder.addParameter(LAST_INSTANT_TS, instant.getTimestamp()));
@@ -149,7 +147,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
String url = builder.toString();
LOG.info("Sending request : (" + url + ")");
- Response response = null;
+ Response response;
int timeout = 1000 * 300; // 5 min timeout
switch (method) {
case GET:
@@ -197,7 +195,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
Map<String, String> paramsMap = new HashMap<>();
paramsMap.put(BASEPATH_PARAM, basePath);
paramsMap.put(PARTITION_PARAM, partitionPath);
- Preconditions.checkArgument(paramNames.length == paramVals.length);
+ ValidationUtils.checkArgument(paramNames.length == paramVals.length);
for (int i = 0; i < paramNames.length; i++) {
paramsMap.put(paramNames[i], paramVals[i]);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java
index e258702..a89ac58 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java
@@ -29,9 +29,9 @@ import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.RocksDBDAO;
import org.apache.hudi.common.util.RocksDBSchemaHelper;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
@@ -110,7 +110,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
protected void addPendingCompactionOperations(Stream<Pair<String, CompactionOperation>> operations) {
rocksDB.writeBatch(batch ->
operations.forEach(opInstantPair -> {
- Preconditions.checkArgument(!isPendingCompactionScheduledForFileId(opInstantPair.getValue().getFileGroupId()),
+ ValidationUtils.checkArgument(!isPendingCompactionScheduledForFileId(opInstantPair.getValue().getFileGroupId()),
"Duplicate FileGroupId found in pending compaction operations. FgId :"
+ opInstantPair.getValue().getFileGroupId());
rocksDB.putInBatch(batch, schemaHelper.getColFamilyForPendingCompaction(),
@@ -123,7 +123,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
void removePendingCompactionOperations(Stream<Pair<String, CompactionOperation>> operations) {
rocksDB.writeBatch(batch ->
operations.forEach(opInstantPair -> {
- Preconditions.checkArgument(
+ ValidationUtils.checkArgument(
getPendingCompactionOperationWithInstant(opInstantPair.getValue().getFileGroupId()) != null,
"Trying to remove a FileGroupId which is not found in pending compaction operations. FgId :"
+ opInstantPair.getValue().getFileGroupId());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
index 1962894..683a4dd 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
@@ -28,7 +28,6 @@ import org.apache.hudi.avro.model.HoodieSavepointMetadata;
import org.apache.hudi.avro.model.HoodieSavepointPartitionMetadata;
import org.apache.hudi.common.HoodieRollbackStat;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileWriter;
@@ -145,7 +144,7 @@ public class AvroUtils {
throws IOException {
DatumReader<T> reader = new SpecificDatumReader<>(clazz);
FileReader<T> fileReader = DataFileReader.openReader(new SeekableByteArrayInput(bytes), reader);
- Preconditions.checkArgument(fileReader.hasNext(), "Could not deserialize metadata of type " + clazz);
+ ValidationUtils.checkArgument(fileReader.hasNext(), "Could not deserialize metadata of type " + clazz);
return fileReader.next();
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
index 9ffc38c..add03d5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
@@ -28,7 +28,6 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.InvalidHoodiePathException;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -115,11 +114,11 @@ public class FSUtils {
}
public static String translateMarkerToDataPath(String basePath, String markerPath, String instantTs) {
- Preconditions.checkArgument(markerPath.endsWith(HoodieTableMetaClient.MARKER_EXTN));
+ ValidationUtils.checkArgument(markerPath.endsWith(HoodieTableMetaClient.MARKER_EXTN));
String markerRootPath = Path.getPathWithoutSchemeAndAuthority(
new Path(String.format("%s/%s/%s", basePath, HoodieTableMetaClient.TEMPFOLDER_NAME, instantTs))).toString();
int begin = markerPath.indexOf(markerRootPath);
- Preconditions.checkArgument(begin >= 0,
+ ValidationUtils.checkArgument(begin >= 0,
"Not in marker dir. Marker Path=" + markerPath + ", Expected Marker Root=" + markerRootPath);
String rPath = markerPath.substring(begin + markerRootPath.length() + 1);
return String.format("%s/%s%s", basePath, rPath.replace(HoodieTableMetaClient.MARKER_EXTN, ""),
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java
index 70ceed0..e7ce62d 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java
@@ -18,7 +18,6 @@
package org.apache.hudi.common.util;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -47,7 +46,7 @@ public class FailSafeConsistencyGuard implements ConsistencyGuard {
public FailSafeConsistencyGuard(FileSystem fs, ConsistencyGuardConfig consistencyGuardConfig) {
this.fs = fs;
this.consistencyGuardConfig = consistencyGuardConfig;
- Preconditions.checkArgument(consistencyGuardConfig.isConsistencyCheckEnabled());
+ ValidationUtils.checkArgument(consistencyGuardConfig.isConsistencyCheckEnabled());
}
@Override
@@ -115,8 +114,8 @@ public class FailSafeConsistencyGuard implements ConsistencyGuard {
*
* @param filePath File Path
* @param visibility Visibility
- * @return
- * @throws IOException
+ * @return true (if file visible in Path), false (otherwise)
+ * @throws IOException -
*/
private boolean checkFileVisibility(Path filePath, FileVisibility visibility) throws IOException {
try {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/NumericUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/NumericUtils.java
index bb22838..775c1f8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/NumericUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/NumericUtils.java
@@ -18,6 +18,13 @@
package org.apache.hudi.common.util;
+import org.apache.hudi.exception.HoodieException;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Objects;
+
/**
* A utility class for numeric.
*/
@@ -31,4 +38,27 @@ public class NumericUtils {
String pre = "KMGTPE".charAt(exp - 1) + "";
return String.format("%.1f %sB", bytes / Math.pow(1024, exp), pre);
}
+
+ public static long getMessageDigestHash(final String algorithmName, final String string) {
+ MessageDigest md;
+ try {
+ md = MessageDigest.getInstance(algorithmName);
+ } catch (NoSuchAlgorithmException e) {
+ throw new HoodieException(e);
+ }
+ return asLong(Objects.requireNonNull(md).digest(string.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ public static long asLong(byte[] bytes) {
+ ValidationUtils.checkState(bytes.length >= 8, "HashCode#asLong() requires >= 8 bytes.");
+ return padToLong(bytes);
+ }
+
+ public static long padToLong(byte[] bytes) {
+ long retVal = (bytes[0] & 0xFF);
+ for (int i = 1; i < Math.min(bytes.length, 8); i++) {
+ retVal |= (bytes[i] & 0xFFL) << (i * 8);
+ }
+ return retVal;
+ }
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java
index 8884e9c..9296fa7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java
@@ -22,7 +22,6 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
-import com.google.common.base.Preconditions;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.rocksdb.AbstractImmutableNativeReference;
@@ -105,7 +104,7 @@ public class RocksDBDAO {
FileIOUtils.mkdir(new File(rocksDBBasePath));
rocksDB = RocksDB.open(dbOptions, rocksDBBasePath, managedColumnFamilies, managedHandles);
- Preconditions.checkArgument(managedHandles.size() == managedColumnFamilies.size(),
+ ValidationUtils.checkArgument(managedHandles.size() == managedColumnFamilies.size(),
"Unexpected number of handles are returned");
for (int index = 0; index < managedHandles.size(); index++) {
ColumnFamilyHandle handle = managedHandles.get(index);
@@ -113,7 +112,7 @@ public class RocksDBDAO {
String familyNameFromHandle = new String(handle.getName());
String familyNameFromDescriptor = new String(descriptor.getName());
- Preconditions.checkArgument(familyNameFromDescriptor.equals(familyNameFromHandle),
+ ValidationUtils.checkArgument(familyNameFromDescriptor.equals(familyNameFromHandle),
"Family Handles not in order with descriptors");
managedHandlesMap.put(familyNameFromHandle, handle);
managedDescriptorMap.put(familyNameFromDescriptor, descriptor);
@@ -297,7 +296,7 @@ public class RocksDBDAO {
* @param <T> Type of object stored.
*/
public <T extends Serializable> T get(String columnFamilyName, String key) {
- Preconditions.checkArgument(!closed);
+ ValidationUtils.checkArgument(!closed);
try {
byte[] val = getRocksDB().get(managedHandlesMap.get(columnFamilyName), key.getBytes());
return val == null ? null : SerializationUtils.deserialize(val);
@@ -314,7 +313,7 @@ public class RocksDBDAO {
* @param <T> Type of object stored.
*/
public <K extends Serializable, T extends Serializable> T get(String columnFamilyName, K key) {
- Preconditions.checkArgument(!closed);
+ ValidationUtils.checkArgument(!closed);
try {
byte[] val = getRocksDB().get(managedHandlesMap.get(columnFamilyName), SerializationUtils.serialize(key));
return val == null ? null : SerializationUtils.deserialize(val);
@@ -331,7 +330,7 @@ public class RocksDBDAO {
* @param <T> Type of value stored
*/
public <T extends Serializable> Stream<Pair<String, T>> prefixSearch(String columnFamilyName, String prefix) {
- Preconditions.checkArgument(!closed);
+ ValidationUtils.checkArgument(!closed);
final HoodieTimer timer = new HoodieTimer();
timer.startTimer();
long timeTakenMicro = 0;
@@ -360,7 +359,7 @@ public class RocksDBDAO {
* @param <T> Type of value stored
*/
public <T extends Serializable> void prefixDelete(String columnFamilyName, String prefix) {
- Preconditions.checkArgument(!closed);
+ ValidationUtils.checkArgument(!closed);
LOG.info("Prefix DELETE (query=" + prefix + ") on " + columnFamilyName);
final RocksIterator it = getRocksDB().newIterator(managedHandlesMap.get(columnFamilyName));
it.seek(prefix.getBytes());
@@ -396,7 +395,7 @@ public class RocksDBDAO {
* @param columnFamilyName Column family name
*/
public void addColumnFamily(String columnFamilyName) {
- Preconditions.checkArgument(!closed);
+ ValidationUtils.checkArgument(!closed);
managedDescriptorMap.computeIfAbsent(columnFamilyName, colFamilyName -> {
try {
@@ -416,7 +415,7 @@ public class RocksDBDAO {
* @param columnFamilyName Column Family Name
*/
public void dropColumnFamily(String columnFamilyName) {
- Preconditions.checkArgument(!closed);
+ ValidationUtils.checkArgument(!closed);
managedDescriptorMap.computeIfPresent(columnFamilyName, (colFamilyName, descriptor) -> {
ColumnFamilyHandle handle = managedHandlesMap.get(colFamilyName);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java
new file mode 100644
index 0000000..0f7c43e
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util;
+
+/*
+ * Simple utility to test validation conditions (to replace Guava's PreConditions)
+ */
+public class ValidationUtils {
+
+ /**
+ * Ensures the truth of an expression.
+ */
+ public static void checkArgument(final boolean expression) {
+ if (!expression) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression, throwing the custom errorMessage otherwise.
+ */
+ public static void checkArgument(final boolean expression, final String errorMessage) {
+ if (!expression) {
+ throw new IllegalArgumentException(errorMessage);
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @throws IllegalStateException if {@code expression} is false
+ */
+ public static void checkState(final boolean expression) {
+ if (!expression) {
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @param errorMessage - error message
+ * @throws IllegalStateException if {@code expression} is false
+ */
+ public static void checkState(final boolean expression, String errorMessage) {
+ if (!expression) {
+ throw new IllegalStateException(errorMessage);
+ }
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java
index 356f000..2c2f919 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java
@@ -21,9 +21,9 @@ package org.apache.hudi.common.util.queue;
import org.apache.hudi.common.util.DefaultSizeEstimator;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.SizeEstimator;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieException;
-import com.google.common.base.Preconditions;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -262,7 +262,7 @@ public class BoundedInMemoryQueue<I, O> implements Iterable<O> {
@Override
public O next() {
- Preconditions.checkState(hasNext() && this.nextRecord != null);
+ ValidationUtils.checkState(hasNext() && this.nextRecord != null);
final O ret = this.nextRecord;
this.nextRecord = null;
return ret;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java
index 3b2bec8..f92dcea 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/MetadataMigrator.java
@@ -19,10 +19,9 @@
package org.apache.hudi.common.versioning;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
-import com.google.common.base.Preconditions;
-
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -75,8 +74,8 @@ public class MetadataMigrator<T> {
* @return Metadata conforming to the target version
*/
public T migrateToVersion(T metadata, int metadataVersion, int targetVersion) {
- Preconditions.checkArgument(targetVersion >= oldestVersion);
- Preconditions.checkArgument(targetVersion <= latestVersion);
+ ValidationUtils.checkArgument(targetVersion >= oldestVersion);
+ ValidationUtils.checkArgument(targetVersion <= latestVersion);
if (metadataVersion == targetVersion) {
return metadata;
} else if (metadataVersion > targetVersion) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV1MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV1MigrationHandler.java
index 998d4f0..a7434b5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV1MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV1MigrationHandler.java
@@ -22,10 +22,10 @@ import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.versioning.AbstractMigratorBase;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.Path;
import java.util.Map;
@@ -52,7 +52,7 @@ public class CleanV1MigrationHandler extends AbstractMigratorBase<HoodieCleanMet
@Override
public HoodieCleanMetadata downgradeFrom(HoodieCleanMetadata input) {
- Preconditions.checkArgument(input.getVersion() == 2,
+ ValidationUtils.checkArgument(input.getVersion() == 2,
"Input version is " + input.getVersion() + ". Must be 2");
final Path basePath = new Path(metaClient.getBasePath());
@@ -81,16 +81,13 @@ public class CleanV1MigrationHandler extends AbstractMigratorBase<HoodieCleanMet
return Pair.of(partitionPath, cleanPartitionMetadata);
}).collect(Collectors.toMap(Pair::getKey, Pair::getValue));
- HoodieCleanMetadata metadata = HoodieCleanMetadata.newBuilder()
+ return HoodieCleanMetadata.newBuilder()
.setEarliestCommitToRetain(input.getEarliestCommitToRetain())
.setStartCleanTime(input.getStartCleanTime())
.setTimeTakenInMillis(input.getTimeTakenInMillis())
.setTotalFilesDeleted(input.getTotalFilesDeleted())
.setPartitionMetadata(partitionMetadataMap)
.setVersion(getManagedVersion()).build();
-
- return metadata;
-
}
private static String convertToV1Path(Path basePath, String partitionPath, String fileName) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV2MigrationHandler.java
index 84896a0..14b2886 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV2MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/clean/CleanV2MigrationHandler.java
@@ -21,10 +21,10 @@ package org.apache.hudi.common.versioning.clean;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.versioning.AbstractMigratorBase;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.Path;
import java.util.List;
@@ -46,7 +46,7 @@ public class CleanV2MigrationHandler extends AbstractMigratorBase<HoodieCleanMet
@Override
public HoodieCleanMetadata upgradeFrom(HoodieCleanMetadata input) {
- Preconditions.checkArgument(input.getVersion() == 1,
+ ValidationUtils.checkArgument(input.getVersion() == 1,
"Input version is " + input.getVersion() + ". Must be 1");
HoodieCleanMetadata metadata = new HoodieCleanMetadata();
metadata.setEarliestCommitToRetain(input.getEarliestCommitToRetain());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java
index 3abd952..9d23d61 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV1MigrationHandler.java
@@ -22,9 +22,9 @@ import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.versioning.AbstractMigratorBase;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.Path;
import java.util.ArrayList;
@@ -54,19 +54,18 @@ public class CompactionV1MigrationHandler extends AbstractMigratorBase<HoodieCom
@Override
public HoodieCompactionPlan downgradeFrom(HoodieCompactionPlan input) {
- Preconditions.checkArgument(input.getVersion() == 2, "Input version is " + input.getVersion() + ". Must be 2");
+ ValidationUtils.checkArgument(input.getVersion() == 2, "Input version is " + input.getVersion() + ". Must be 2");
HoodieCompactionPlan compactionPlan = new HoodieCompactionPlan();
final Path basePath = new Path(metaClient.getBasePath());
List<HoodieCompactionOperation> v1CompactionOperationList = new ArrayList<>();
if (null != input.getOperations()) {
- v1CompactionOperationList = input.getOperations().stream().map(inp -> {
- return HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime())
+ v1CompactionOperationList = input.getOperations().stream().map(inp ->
+ HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime())
.setFileId(inp.getFileId()).setPartitionPath(inp.getPartitionPath()).setMetrics(inp.getMetrics())
.setDataFilePath(convertToV1Path(basePath, inp.getPartitionPath(), inp.getDataFilePath()))
.setDeltaFilePaths(inp.getDeltaFilePaths().stream()
.map(s -> convertToV1Path(basePath, inp.getPartitionPath(), s)).collect(Collectors.toList()))
- .build();
- }).collect(Collectors.toList());
+ .build()).collect(Collectors.toList());
}
compactionPlan.setOperations(v1CompactionOperationList);
compactionPlan.setExtraMetadata(input.getExtraMetadata());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java
index cc73a2f..26180e4 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java
@@ -21,9 +21,9 @@ package org.apache.hudi.common.versioning.compaction;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.versioning.AbstractMigratorBase;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.Path;
import java.util.ArrayList;
@@ -48,17 +48,16 @@ public class CompactionV2MigrationHandler extends AbstractMigratorBase<HoodieCom
@Override
public HoodieCompactionPlan upgradeFrom(HoodieCompactionPlan input) {
- Preconditions.checkArgument(input.getVersion() == 1, "Input version is " + input.getVersion() + ". Must be 1");
+ ValidationUtils.checkArgument(input.getVersion() == 1, "Input version is " + input.getVersion() + ". Must be 1");
HoodieCompactionPlan compactionPlan = new HoodieCompactionPlan();
List<HoodieCompactionOperation> v2CompactionOperationList = new ArrayList<>();
if (null != input.getOperations()) {
- v2CompactionOperationList = input.getOperations().stream().map(inp -> {
- return HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime())
+ v2CompactionOperationList = input.getOperations().stream().map(inp ->
+ HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime())
.setFileId(inp.getFileId()).setPartitionPath(inp.getPartitionPath()).setMetrics(inp.getMetrics())
.setDataFilePath(new Path(inp.getDataFilePath()).getName()).setDeltaFilePaths(
inp.getDeltaFilePaths().stream().map(s -> new Path(s).getName()).collect(Collectors.toList()))
- .build();
- }).collect(Collectors.toList());
+ .build()).collect(Collectors.toList());
}
compactionPlan.setOperations(v2CompactionOperationList);
compactionPlan.setExtraMetadata(input.getExtraMetadata());
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
index a35fe50..ed4750a 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java
@@ -21,7 +21,6 @@ package org.apache.hudi.common.minicluster;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.util.FileIOUtils;
-import com.google.common.base.Preconditions;
import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -33,6 +32,7 @@ import org.apache.log4j.Logger;
import java.io.File;
import java.io.IOException;
+import java.util.Objects;
import java.net.ServerSocket;
/**
@@ -70,7 +70,7 @@ public class HdfsTestService {
}
public MiniDFSCluster start(boolean format) throws IOException {
- Preconditions.checkState(workDir != null, "The work dir must be set before starting cluster.");
+ Objects.requireNonNull(workDir, "The work dir must be set before starting cluster.");
hadoopConf = HoodieTestUtils.getDefaultHadoopConf();
// If clean, then remove the work dir so we can start fresh.
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
index 670be44..514e298 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java
@@ -18,7 +18,6 @@
package org.apache.hudi.common.minicluster;
-import com.google.common.base.Preconditions;
import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
@@ -36,6 +35,7 @@ import java.io.OutputStream;
import java.io.Reader;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.util.Objects;
/**
* A Zookeeper minicluster service implementation.
@@ -85,7 +85,7 @@ public class ZookeeperTestService {
}
public ZooKeeperServer start() throws IOException, InterruptedException {
- Preconditions.checkState(workDir != null, "The localBaseFsLocation must be set before starting cluster.");
+ Objects.requireNonNull(workDir, "The localBaseFsLocation must be set before starting cluster.");
setupTestEnv();
stop();
@@ -171,13 +171,10 @@ public class ZookeeperTestService {
long start = System.currentTimeMillis();
while (true) {
try {
- Socket sock = new Socket("localhost", port);
- try {
+ try (Socket sock = new Socket("localhost", port)) {
OutputStream outstream = sock.getOutputStream();
outstream.write("stat".getBytes());
outstream.flush();
- } finally {
- sock.close();
}
} catch (IOException e) {
return true;
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java
index b896d1f..3d47fed 100755
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java
@@ -57,8 +57,8 @@ import org.junit.runners.Parameterized;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URISyntaxException;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
index e38625a..c00c6a1 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
@@ -44,10 +44,10 @@ import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
import org.apache.hadoop.fs.Path;
@@ -201,7 +201,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
view1.sync();
Map<String, List<String>> instantsToFiles;
- /**
+ /*
* Case where incremental syncing is catching up on more than one ingestion at a time
*/
// Run 1 ingestion on MOR table (1 delta commits). View1 is now sync up to this point
@@ -222,7 +222,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
view3.sync();
areViewsConsistent(view1, view2, partitions.size() * fileIdsPerPartition.size());
- /**
+ /*
* Case where a compaction is scheduled and then unscheduled
*/
scheduleCompaction(view2, "15");
@@ -233,7 +233,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath()));
view4.sync();
- /**
+ /*
* Case where a compaction is scheduled, 2 ingestion happens and then a compaction happens
*/
scheduleCompaction(view2, "16");
@@ -247,7 +247,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath()));
view5.sync();
- /**
+ /*
* Case where a clean happened and then rounds of ingestion and compaction happened
*/
testCleans(view2, Collections.singletonList("19"),
@@ -266,7 +266,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
getFileSystemView(new HoodieTableMetaClient(metaClient.getHadoopConf(), metaClient.getBasePath()));
view6.sync();
- /**
+ /*
* Case where multiple restores and ingestions happened
*/
testRestore(view2, Collections.singletonList("25"), true, new HashMap<>(), Collections.singletonList("24"), "29", true);
@@ -528,7 +528,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
String newBaseInstant) throws IOException {
HoodieInstant instant = new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, compactionInstantTime);
boolean deleted = metaClient.getFs().delete(new Path(metaClient.getMetaPath(), instant.getFileName()), false);
- Preconditions.checkArgument(deleted, "Unable to delete compaction instant.");
+ ValidationUtils.checkArgument(deleted, "Unable to delete compaction instant.");
view.sync();
Assert.assertEquals(newLastInstant, view.getLastInstant().get().getTimestamp());
@@ -719,7 +719,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
metaClient.getActiveTimeline().createNewInstant(inflightInstant);
metaClient.getActiveTimeline().saveAsComplete(inflightInstant,
Option.of(metadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
- /**
+ /*
// Delete pending compaction if present
metaClient.getFs().delete(new Path(metaClient.getMetaPath(),
new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instant).getFileName()));
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java
index 08cb952..cd76383 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java
@@ -20,7 +20,10 @@ package org.apache.hudi.common.util;
import org.junit.Test;
+import java.util.Arrays;
+
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
/**
* Tests numeric utils.
@@ -37,6 +40,29 @@ public class TestNumericUtils {
assertEquals("27.0 GB", NumericUtils.humanReadableByteCount(28991029248L));
assertEquals("1.7 TB", NumericUtils.humanReadableByteCount(1855425871872L));
assertEquals("8.0 EB", NumericUtils.humanReadableByteCount(9223372036854775807L));
+ }
+
+ @Test
+ public void testGetMessageDigestHash() {
+ assertEquals(6808551913422584641L, NumericUtils.getMessageDigestHash("MD5", "This is a string"));
+ assertEquals(2549749777095932358L, NumericUtils.getMessageDigestHash("MD5", "This is a test string"));
+ assertNotEquals(1L, NumericUtils.getMessageDigestHash("MD5", "This"));
+ assertNotEquals(6808551913422584641L, NumericUtils.getMessageDigestHash("SHA-256", "This is a string"));
+ }
+
+ private static byte[] byteArrayWithNum(int size, int num) {
+ byte[] bytez = new byte[size];
+ Arrays.fill(bytez, (byte) num);
+ return bytez;
+ }
+ @Test
+ public void testPadToLong() {
+ assertEquals(0x0000000099999999L, NumericUtils.padToLong(byteArrayWithNum(4, 0x99)));
+ assertEquals(0x0000999999999999L, NumericUtils.padToLong(byteArrayWithNum(6, 0x99)));
+ assertEquals(0x9999999999999999L, NumericUtils.padToLong(byteArrayWithNum(8, 0x99)));
+ assertEquals(0x1111111111111111L, NumericUtils.padToLong(byteArrayWithNum(8, 0x11)));
+ assertEquals(0x0000000011111111L, NumericUtils.padToLong(byteArrayWithNum(4, 0x11)));
+ assertEquals(0x0000181818181818L, NumericUtils.padToLong(byteArrayWithNum(6, 0x18)));
}
}
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
index bc717a9..4fe041a 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
@@ -27,12 +27,12 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
import org.apache.hudi.hadoop.UseFileSplitsFromInputFormat;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -244,7 +244,7 @@ public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat i
LOG.info("Creating record reader with readCols :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR)
+ ", Ids :" + jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR));
// sanity check
- Preconditions.checkArgument(split instanceof HoodieRealtimeFileSplit,
+ ValidationUtils.checkArgument(split instanceof HoodieRealtimeFileSplit,
"HoodieRealtimeRecordReader can only work on HoodieRealtimeFileSplit and not with " + split);
return new HoodieRealtimeRecordReader((HoodieRealtimeFileSplit) split, jobConf,
diff --git a/hudi-hive/pom.xml b/hudi-hive/pom.xml
index 7ff1b3a..01e6ff1 100644
--- a/hudi-hive/pom.xml
+++ b/hudi-hive/pom.xml
@@ -56,11 +56,6 @@
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
index 8d8dd75..acbff7f 100644
--- a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
+++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
@@ -28,12 +28,12 @@ import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.InvalidTableException;
import org.apache.hudi.hive.util.SchemaUtil;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -179,7 +179,7 @@ public class HoodieHiveClient {
*/
private String getPartitionClause(String partition) {
List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partition);
- Preconditions.checkArgument(syncConfig.partitionFields.size() == partitionValues.size(),
+ ValidationUtils.checkArgument(syncConfig.partitionFields.size() == partitionValues.size(),
"Partition key parts " + syncConfig.partitionFields + " does not match with partition values " + partitionValues
+ ". Check partition strategy. ");
List<String> partBuilder = new ArrayList<>();
diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java b/hudi-hive/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
index aa6ec30..ec37a82 100644
--- a/hudi-hive/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
+++ b/hudi-hive/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
@@ -18,7 +18,7 @@
package org.apache.hudi.hive;
-import com.google.common.base.Preconditions;
+import org.apache.hudi.common.util.ValidationUtils;
import java.util.Arrays;
import java.util.List;
@@ -35,7 +35,7 @@ public class MultiPartKeysValueExtractor implements PartitionValueExtractor {
return Arrays.stream(splits).map(s -> {
if (s.contains("=")) {
String[] moreSplit = s.split("=");
- Preconditions.checkArgument(moreSplit.length == 2, "Partition Field (" + s + ") not in expected format");
+ ValidationUtils.checkArgument(moreSplit.length == 2, "Partition Field (" + s + ") not in expected format");
return moreSplit[1];
}
return s;
diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java b/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java
index 57bab64..bef40d5 100644
--- a/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java
+++ b/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java
@@ -44,7 +44,7 @@ public class SchemaDifference {
this.tableSchema = tableSchema;
this.deleteColumns = Collections.unmodifiableList(deleteColumns);
this.updateColumnTypes = Collections.unmodifiableMap(updateColumnTypes);
- this.addColumnTypes = Collections.unmodifiableMap(addColumnTypes);
+ this.addColumnTypes = Collections.unmodifiableMap(addColumnTypes);
}
public List<String> getDeleteColumns() {
diff --git a/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
index fc7675f..0cef82b 100644
--- a/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
+++ b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
@@ -21,8 +21,6 @@ package org.apache.hudi.hive.util;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.util.FileIOUtils;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -53,7 +51,9 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketException;
+import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -73,7 +73,7 @@ public class HiveTestService {
private int serverPort = 9999;
private boolean clean = true;
- private Map<String, String> sysProps = Maps.newHashMap();
+ private Map<String, String> sysProps = new HashMap<>();
private ExecutorService executorService;
private TServer tServer;
private HiveServer2 hiveServer;
@@ -87,7 +87,7 @@ public class HiveTestService {
}
public HiveServer2 start() throws IOException {
- Preconditions.checkState(workDir != null, "The work dir must be set before starting cluster.");
+ Objects.requireNonNull(workDir, "The work dir must be set before starting cluster.");
if (hadoopConf == null) {
hadoopConf = HoodieTestUtils.getDefaultHadoopConf();
diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml
index fe6f66d..e0ee9b4 100644
--- a/hudi-integ-test/pom.xml
+++ b/hudi-integ-test/pom.xml
@@ -115,13 +115,6 @@
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>20.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java
index a807e05..8c49394 100644
--- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java
+++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java
@@ -28,13 +28,13 @@ import org.apache.hudi.common.table.timeline.dto.InstantDTO;
import org.apache.hudi.common.table.timeline.dto.TimelineDTO;
import org.apache.hudi.common.table.view.FileSystemViewManager;
import org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.timeline.service.handlers.BaseFileHandler;
import org.apache.hudi.timeline.service.handlers.FileSliceHandler;
import org.apache.hudi.timeline.service.handlers.TimelineHandler;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
import io.javalin.Context;
import io.javalin.Handler;
import io.javalin.Javalin;
@@ -339,7 +339,7 @@ public class FileSystemViewHandler {
+ " but server has the following timeline "
+ viewManager.getFileSystemView(context.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM))
.getTimeline().getInstants().collect(Collectors.toList());
- Preconditions.checkArgument(!isLocalViewBehind(context), errMsg);
+ ValidationUtils.checkArgument(!isLocalViewBehind(context), errMsg);
long endFinalCheck = System.currentTimeMillis();
finalCheckTimeTaken = endFinalCheck - beginFinalCheck;
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieWithTimelineServer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieWithTimelineServer.java
index e7e5cb8..b6fe64a 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieWithTimelineServer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieWithTimelineServer.java
@@ -20,12 +20,12 @@ package org.apache.hudi.utilities;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
-import com.google.common.base.Preconditions;
import io.javalin.Javalin;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.spark.api.java.JavaSparkContext;
import java.io.BufferedReader;
@@ -87,7 +87,7 @@ public class HoodieWithTimelineServer implements Serializable {
IntStream.range(0, cfg.numPartitions).forEach(i -> messages.add("Hello World"));
List<String> gotMessages = jsc.parallelize(messages).map(msg -> sendRequest(driverHost, cfg.serverPort)).collect();
System.out.println("Got Messages :" + gotMessages);
- Preconditions.checkArgument(gotMessages.equals(messages), "Got expected reply from Server");
+ ValidationUtils.checkArgument(gotMessages.equals(messages), "Got expected reply from Server");
}
public String sendRequest(String driverHost, int port) {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
index a62c232..50254dd 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.util.DFSPropertiesConfiguration;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.TypedProperties;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -36,7 +37,6 @@ import org.apache.hudi.utilities.schema.SchemaProvider;
import org.apache.hudi.utilities.sources.Source;
import org.apache.hudi.utilities.transform.Transformer;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -137,7 +137,7 @@ public class UtilHelpers {
TypedProperties properties = new TypedProperties();
props.forEach(x -> {
String[] kv = x.split("=");
- Preconditions.checkArgument(kv.length == 2);
+ ValidationUtils.checkArgument(kv.length == 2);
properties.setProperty(kv[0], kv[1]);
});
return properties;
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
index 4b69d22..3073dfa 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
@@ -20,6 +20,7 @@ package org.apache.hudi.utilities.deltastreamer;
import org.apache.hudi.AvroConversionUtils;
import org.apache.hudi.DataSourceUtils;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.client.HoodieWriteClient;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.client.WriteStatus;
@@ -49,7 +50,6 @@ import org.apache.hudi.utilities.sources.InputBatch;
import org.apache.hudi.utilities.transform.Transformer;
import com.codahale.metrics.Timer;
-import com.google.common.base.Preconditions;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.conf.Configuration;
@@ -502,10 +502,10 @@ public class DeltaSync implements Serializable {
HoodieWriteConfig config = builder.build();
// Validate what deltastreamer assumes of write-config to be really safe
- Preconditions.checkArgument(config.isInlineCompaction() == cfg.isInlineCompactionEnabled());
- Preconditions.checkArgument(!config.shouldAutoCommit());
- Preconditions.checkArgument(config.shouldCombineBeforeInsert() == cfg.filterDupes);
- Preconditions.checkArgument(config.shouldCombineBeforeUpsert());
+ ValidationUtils.checkArgument(config.isInlineCompaction() == cfg.isInlineCompactionEnabled());
+ ValidationUtils.checkArgument(!config.shouldAutoCommit());
+ ValidationUtils.checkArgument(config.shouldCombineBeforeInsert() == cfg.filterDupes);
+ ValidationUtils.checkArgument(config.shouldCombineBeforeUpsert());
return config;
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
index b2aec69..01ab1cc 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
@@ -29,6 +29,7 @@ import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.TypedProperties;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
@@ -41,7 +42,6 @@ import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -353,7 +353,7 @@ public class HoodieDeltaStreamer implements Serializable {
new HoodieTableMetaClient(new Configuration(fs.getConf()), cfg.targetBasePath, false);
tableType = meta.getTableType();
// This will guarantee there is no surprise with table type
- Preconditions.checkArgument(tableType.equals(HoodieTableType.valueOf(cfg.tableType)),
+ ValidationUtils.checkArgument(tableType.equals(HoodieTableType.valueOf(cfg.tableType)),
"Hoodie table is of type " + tableType + " but passed in CLI argument is " + cfg.tableType);
} else {
tableType = HoodieTableType.valueOf(cfg.tableType);
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java
index 9787bab..44fd50d 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java
@@ -22,12 +22,14 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
-import com.google.common.base.Preconditions;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Row;
+import java.util.Objects;
+
public class IncrSourceHelper {
/**
@@ -37,7 +39,7 @@ public class IncrSourceHelper {
*/
private static String getStrictlyLowerTimestamp(String timestamp) {
long ts = Long.parseLong(timestamp);
- Preconditions.checkArgument(ts > 0, "Timestamp must be positive");
+ ValidationUtils.checkArgument(ts > 0, "Timestamp must be positive");
long lower = ts - 1;
return "" + lower;
}
@@ -54,7 +56,7 @@ public class IncrSourceHelper {
*/
public static Pair<String, String> calculateBeginAndEndInstants(JavaSparkContext jssc, String srcBasePath,
int numInstantsPerFetch, Option<String> beginInstant, boolean readLatestOnMissingBeginInstant) {
- Preconditions.checkArgument(numInstantsPerFetch > 0,
+ ValidationUtils.checkArgument(numInstantsPerFetch > 0,
"Make sure the config hoodie.deltastreamer.source.hoodieincr.num_instants is set to a positive value");
HoodieTableMetaClient srcMetaClient = new HoodieTableMetaClient(jssc.hadoopConfiguration(), srcBasePath, true);
@@ -85,11 +87,11 @@ public class IncrSourceHelper {
* @param endInstant end instant of the batch
*/
public static void validateInstantTime(Row row, String instantTime, String sinceInstant, String endInstant) {
- Preconditions.checkNotNull(instantTime);
- Preconditions.checkArgument(HoodieTimeline.compareTimestamps(instantTime, sinceInstant, HoodieTimeline.GREATER),
+ Objects.requireNonNull(instantTime);
+ ValidationUtils.checkArgument(HoodieTimeline.compareTimestamps(instantTime, sinceInstant, HoodieTimeline.GREATER),
"Instant time(_hoodie_commit_time) in row (" + row + ") was : " + instantTime + "but expected to be between "
+ sinceInstant + "(excl) - " + endInstant + "(incl)");
- Preconditions.checkArgument(
+ ValidationUtils.checkArgument(
HoodieTimeline.compareTimestamps(instantTime, endInstant, HoodieTimeline.LESSER_OR_EQUAL),
"Instant time(_hoodie_commit_time) in row (" + row + ") was : " + instantTime + "but expected to be between "
+ sinceInstant + "(excl) - " + endInstant + "(incl)");
[hudi] 19/40: [HUDI-742] Fix Java Math Exception (#1466)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 2f4ee0a4af3c088f825decd77c5429c822b50ad6
Author: Edwin Guo <al...@yahoo.com>
AuthorDate: Tue Mar 31 00:56:20 2020 -0400
[HUDI-742] Fix Java Math Exception (#1466)
---
.../index/bloom/BucketizedBloomCheckPartitioner.java | 2 +-
.../bloom/TestBucketizedBloomCheckPartitioner.java | 17 +++++++++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java
index 798120c..36710dc 100644
--- a/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java
+++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java
@@ -145,7 +145,7 @@ public class BucketizedBloomCheckPartitioner extends Partitioner {
final Pair<String, String> parts = (Pair<String, String>) key;
final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", parts.getRight());
final List<Integer> candidatePartitions = fileGroupToPartitions.get(parts.getLeft());
- final int idx = (int) Math.floorMod(hashOfKey, candidatePartitions.size());
+ final int idx = (int) Math.floorMod((int) hashOfKey, candidatePartitions.size());
assert idx >= 0;
return candidatePartitions.get(idx);
}
diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java
index 29d039a..3ad5a99 100644
--- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java
+++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java
@@ -30,6 +30,7 @@ import java.util.stream.IntStream;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TestBucketizedBloomCheckPartitioner {
@@ -79,4 +80,20 @@ public class TestBucketizedBloomCheckPartitioner {
BucketizedBloomCheckPartitioner p = new BucketizedBloomCheckPartitioner(10000, comparisons1, 10);
assertEquals("num partitions must equal total buckets", 100, p.numPartitions());
}
+
+ @Test
+ public void testGetPartitions() {
+ Map<String, Long> comparisons1 = new HashMap<String, Long>() {
+ {
+ IntStream.range(0, 100000).forEach(f -> put("f" + f, 100L));
+ }
+ };
+ BucketizedBloomCheckPartitioner p = new BucketizedBloomCheckPartitioner(1000, comparisons1, 10);
+
+ IntStream.range(0, 100000).forEach(f -> {
+ int partition = p.getPartition(Pair.of("f" + f, "value"));
+ assertTrue("partition is out of range: " + partition, 0 <= partition && partition <= 1000);
+ });
+ }
+
}
[hudi] 36/40: Fixing test failure in
TestHoodieClientOnCopyOnWriteStorage
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 07345bacb0b494158760fa9f4aa92a4980f1f085
Author: Sivabalan Narayanan <si...@uber.com>
AuthorDate: Mon May 25 08:28:33 2020 -0400
Fixing test failure in TestHoodieClientOnCopyOnWriteStorage
---
.../org/apache/hudi/client/HoodieWriteClient.java | 104 ++++++++++-----------
1 file changed, 52 insertions(+), 52 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
index ba39b9b..37dfe3d 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
@@ -25,8 +25,8 @@ import org.apache.hudi.avro.model.HoodieSavepointMetadata;
import org.apache.hudi.client.embedded.EmbeddedTimelineService;
import org.apache.hudi.common.HoodieRollbackStat;
import org.apache.hudi.common.model.EmptyHoodieRecordPayload;
-import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
@@ -54,8 +54,8 @@ import org.apache.hudi.exception.HoodieSavepointException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.execution.BulkInsertMapFunction;
import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.table.HoodieCommitArchiveLog;
import org.apache.hudi.metrics.HoodieMetrics;
+import org.apache.hudi.table.HoodieCommitArchiveLog;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.UserDefinedBulkInsertPartitioner;
import org.apache.hudi.table.WorkloadProfile;
@@ -102,7 +102,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
/**
* Create a write client, without cleaning up failed/inflight commits.
*
- * @param jsc Java Spark Context
+ * @param jsc Java Spark Context
* @param clientConfig instance of HoodieWriteConfig
*/
public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig) {
@@ -112,8 +112,8 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
/**
* Create a write client, with new hudi index.
*
- * @param jsc Java Spark Context
- * @param clientConfig instance of HoodieWriteConfig
+ * @param jsc Java Spark Context
+ * @param clientConfig instance of HoodieWriteConfig
* @param rollbackPending whether need to cleanup pending commits
*/
public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending) {
@@ -125,15 +125,15 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
}
/**
- * Create a write client, allows to specify all parameters.
+ * Create a write client, allows to specify all parameters.
*
- * @param jsc Java Spark Context
- * @param clientConfig instance of HoodieWriteConfig
+ * @param jsc Java Spark Context
+ * @param clientConfig instance of HoodieWriteConfig
* @param rollbackPending whether need to cleanup pending commits
* @param timelineService Timeline Service that runs as part of write client.
*/
public HoodieWriteClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig, boolean rollbackPending,
- HoodieIndex index, Option<EmbeddedTimelineService> timelineService) {
+ HoodieIndex index, Option<EmbeddedTimelineService> timelineService) {
super(jsc, index, clientConfig, timelineService);
this.metrics = new HoodieMetrics(config, config.getTableName());
this.rollbackPending = rollbackPending;
@@ -147,7 +147,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* @return SparkConf
*/
public static SparkConf registerClasses(SparkConf conf) {
- conf.registerKryoClasses(new Class[]{HoodieWriteConfig.class, HoodieRecord.class, HoodieKey.class});
+ conf.registerKryoClasses(new Class[] {HoodieWriteConfig.class, HoodieRecord.class, HoodieKey.class});
return conf;
}
@@ -169,7 +169,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
/**
* Upsert a batch of new records into Hoodie table at the supplied commitTime.
*
- * @param records JavaRDD of hoodieRecords to upsert
+ * @param records JavaRDD of hoodieRecords to upsert
* @param commitTime Instant time of the commit
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
@@ -199,7 +199,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* This implementation requires that the input records are already tagged, and de-duped if needed.
*
* @param preppedRecords Prepared HoodieRecords to upsert
- * @param commitTime Instant time of the commit
+ * @param commitTime Instant time of the commit
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
public JavaRDD<WriteStatus> upsertPreppedRecords(JavaRDD<HoodieRecord<T>> preppedRecords, final String commitTime) {
@@ -220,7 +220,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* This implementation skips the index check and is able to leverage benefits such as small file handling/blocking
* alignment, as with upsert(), by profiling the workload
*
- * @param records HoodieRecords to insert
+ * @param records HoodieRecords to insert
* @param commitTime Instant time of the commit
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
@@ -248,7 +248,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* de-duped if needed.
*
* @param preppedRecords HoodieRecords to insert
- * @param commitTime Instant time of the commit
+ * @param commitTime Instant time of the commit
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
public JavaRDD<WriteStatus> insertPreppedRecords(JavaRDD<HoodieRecord<T>> preppedRecords, final String commitTime) {
@@ -270,7 +270,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* This implementation uses sortBy (which does range partitioning based on reservoir sampling) and attempts to control
* the numbers of files with less memory compared to the {@link HoodieWriteClient#insert(JavaRDD, String)}
*
- * @param records HoodieRecords to insert
+ * @param records HoodieRecords to insert
* @param commitTime Instant time of the commit
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
@@ -287,14 +287,14 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* it allows users to specify their own partitioner. If specified then it will be used for repartitioning records. See
* {@link UserDefinedBulkInsertPartitioner}.
*
- * @param records HoodieRecords to insert
- * @param commitTime Instant time of the commit
+ * @param records HoodieRecords to insert
+ * @param commitTime Instant time of the commit
* @param bulkInsertPartitioner If specified then it will be used to partition input records before they are inserted
- * into hoodie.
+ * into hoodie.
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
public JavaRDD<WriteStatus> bulkInsert(JavaRDD<HoodieRecord<T>> records, final String commitTime,
- Option<UserDefinedBulkInsertPartitioner> bulkInsertPartitioner) {
+ Option<UserDefinedBulkInsertPartitioner> bulkInsertPartitioner) {
HoodieTable<T> table = getTableAndInitCtx(OperationType.BULK_INSERT);
try {
// De-dupe/merge if needed
@@ -320,14 +320,14 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* it allows users to specify their own partitioner. If specified then it will be used for repartitioning records. See
* {@link UserDefinedBulkInsertPartitioner}.
*
- * @param preppedRecords HoodieRecords to insert
- * @param commitTime Instant time of the commit
+ * @param preppedRecords HoodieRecords to insert
+ * @param commitTime Instant time of the commit
* @param bulkInsertPartitioner If specified then it will be used to partition input records before they are inserted
- * into hoodie.
+ * into hoodie.
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
public JavaRDD<WriteStatus> bulkInsertPreppedRecords(JavaRDD<HoodieRecord<T>> preppedRecords, final String commitTime,
- Option<UserDefinedBulkInsertPartitioner> bulkInsertPartitioner) {
+ Option<UserDefinedBulkInsertPartitioner> bulkInsertPartitioner) {
HoodieTable<T> table = getTableAndInitCtx(OperationType.BULK_INSERT_PREPPED);
try {
return bulkInsertInternal(preppedRecords, commitTime, table, bulkInsertPartitioner);
@@ -343,7 +343,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* Deletes a list of {@link HoodieKey}s from the Hoodie table, at the supplied commitTime {@link HoodieKey}s will be
* deduped and non existant keys will be removed before deleting.
*
- * @param keys {@link List} of {@link HoodieKey}s to be deleted
+ * @param keys {@link List} of {@link HoodieKey}s to be deleted
* @param commitTime Commit time handle
* @return JavaRDD[WriteStatus] - RDD of WriteStatus to inspect errors and counts
*/
@@ -380,7 +380,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
}
private JavaRDD<WriteStatus> bulkInsertInternal(JavaRDD<HoodieRecord<T>> dedupedRecords, String commitTime,
- HoodieTable<T> table, Option<UserDefinedBulkInsertPartitioner> bulkInsertPartitioner) {
+ HoodieTable<T> table, Option<UserDefinedBulkInsertPartitioner> bulkInsertPartitioner) {
final JavaRDD<HoodieRecord<T>> repartitionedRecords;
final int parallelism = config.getBulkInsertShuffleParallelism();
if (bulkInsertPartitioner.isPresent()) {
@@ -400,7 +400,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
IntStream.range(0, parallelism).mapToObj(i -> FSUtils.createNewFileIdPfx()).collect(Collectors.toList());
table.getActiveTimeline().transitionRequestedToInflight(new HoodieInstant(State.REQUESTED,
- table.getMetaClient().getCommitActionType(), commitTime), Option.empty());
+ table.getMetaClient().getCommitActionType(), commitTime), Option.empty(), config.shouldAllowMultiWriteOnSameInstant());
JavaRDD<WriteStatus> writeStatusRDD = repartitionedRecords
.mapPartitionsWithIndex(new BulkInsertMapFunction<T>(commitTime, config, table, fileIDPrefixes), true)
@@ -410,7 +410,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
}
private JavaRDD<HoodieRecord<T>> combineOnCondition(boolean condition, JavaRDD<HoodieRecord<T>> records,
- int parallelism) {
+ int parallelism) {
return condition ? deduplicateRecords(records, parallelism) : records;
}
@@ -440,14 +440,14 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
String commitActionType = table.getMetaClient().getCommitActionType();
HoodieInstant requested = new HoodieInstant(State.REQUESTED, commitActionType, commitTime);
activeTimeline.transitionRequestedToInflight(requested,
- Option.of(metadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
+ Option.of(metadata.toJsonString().getBytes(StandardCharsets.UTF_8)), config.shouldAllowMultiWriteOnSameInstant());
} catch (IOException io) {
throw new HoodieCommitException("Failed to commit " + commitTime + " unable to save inflight metadata ", io);
}
}
private JavaRDD<WriteStatus> upsertRecordsInternal(JavaRDD<HoodieRecord<T>> preppedRecords, String commitTime,
- HoodieTable<T> hoodieTable, final boolean isUpsert) {
+ HoodieTable<T> hoodieTable, final boolean isUpsert) {
// Cache the tagged records, so we don't end up computing both
// TODO: Consistent contract in HoodieWriteClient regarding preppedRecord storage level handling
@@ -494,7 +494,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
@Override
protected void postCommit(HoodieCommitMetadata metadata, String instantTime,
- Option<Map<String, String>> extraMetadata) throws IOException {
+ Option<Map<String, String>> extraMetadata) throws IOException {
// Do an inline compaction if enabled
if (config.isInlineCompaction()) {
@@ -524,7 +524,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* <p>
* Savepoint should be on a commit that could not have been cleaned.
*
- * @param user - User creating the savepoint
+ * @param user - User creating the savepoint
* @param comment - Comment for the savepoint
* @return true if the savepoint was created successfully
*/
@@ -552,8 +552,8 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* Savepoint should be on a commit that could not have been cleaned.
*
* @param commitTime - commit that should be savepointed
- * @param user - User creating the savepoint
- * @param comment - Comment for the savepoint
+ * @param user - User creating the savepoint
+ * @param comment - Comment for the savepoint
* @return true if the savepoint was created successfully
*/
public boolean savepoint(String commitTime, String user, String comment) {
@@ -638,7 +638,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
/**
* Delete a compaction request that is pending.
- *
+ * <p>
* NOTE - This is an Admin operation. With async compaction, this is expected to be called with async compaction and
* write shutdown. Otherwise, async compactor could fail with errors
*
@@ -777,7 +777,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
}
private void finishRestore(final Timer.Context context, Map<String, List<HoodieRollbackStat>> commitToStats,
- List<String> commitsToRollback, final String startRestoreTime, final String restoreToInstant) throws IOException {
+ List<String> commitsToRollback, final String startRestoreTime, final String restoreToInstant) throws IOException {
HoodieTable<T> table = HoodieTable.getHoodieTable(createMetaClient(true), config, jsc);
Option<Long> durationInMs = Option.empty();
long numFilesDeleted = 0L;
@@ -870,8 +870,8 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
metaClient.getActiveTimeline().filterPendingCompactionTimeline().lastInstant().ifPresent(latestPending ->
ValidationUtils.checkArgument(
HoodieTimeline.compareTimestamps(latestPending.getTimestamp(), instantTime, HoodieTimeline.LESSER),
- "Latest pending compaction instant time must be earlier than this instant time. Latest Compaction :"
- + latestPending + ", Ingesting at " + instantTime));
+ "Latest pending compaction instant time must be earlier than this instant time. Latest Compaction :"
+ + latestPending + ", Ingesting at " + instantTime));
metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(State.REQUESTED, metaClient.getCommitActionType(),
instantTime));
}
@@ -891,7 +891,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
/**
* Schedules a new compaction instant with passed-in instant time.
*
- * @param instantTime Compaction Instant Time
+ * @param instantTime Compaction Instant Time
* @param extraMetadata Extra Metadata to be stored
*/
public boolean scheduleCompactionAtInstant(String instantTime, Option<Map<String, String>> extraMetadata)
@@ -939,11 +939,11 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file.
*
* @param compactionInstantTime Compaction Instant Time
- * @param writeStatuses RDD of WriteStatus to inspect errors and counts
- * @param extraMetadata Extra Metadata to be stored
+ * @param writeStatuses RDD of WriteStatus to inspect errors and counts
+ * @param extraMetadata Extra Metadata to be stored
*/
public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> writeStatuses,
- Option<Map<String, String>> extraMetadata) throws IOException {
+ Option<Map<String, String>> extraMetadata) throws IOException {
HoodieTableMetaClient metaClient = createMetaClient(true);
HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
@@ -966,7 +966,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
/**
* Deduplicate Hoodie records, using the given deduplication function.
*
- * @param records hoodieRecords to deduplicate
+ * @param records hoodieRecords to deduplicate
* @param parallelism parallelism or partitions to be used while reducing/deduplicating
* @return RDD of HoodieRecord already be deduplicated
*/
@@ -1051,12 +1051,12 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* Perform compaction operations as specified in the compaction commit file.
*
* @param compactionInstant Compaction Instant time
- * @param activeTimeline Active Timeline
- * @param autoCommit Commit after compaction
+ * @param activeTimeline Active Timeline
+ * @param autoCommit Commit after compaction
* @return RDD of Write Status
*/
private JavaRDD<WriteStatus> runCompaction(HoodieInstant compactionInstant, HoodieActiveTimeline activeTimeline,
- boolean autoCommit) throws IOException {
+ boolean autoCommit) throws IOException {
HoodieTableMetaClient metaClient = createMetaClient(true);
HoodieCompactionPlan compactionPlan =
CompactionUtils.getCompactionPlan(metaClient, compactionInstant.getTimestamp());
@@ -1077,14 +1077,14 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
/**
* Commit Compaction and track metrics.
*
- * @param compactedStatuses Compaction Write status
- * @param table Hoodie Table
+ * @param compactedStatuses Compaction Write status
+ * @param table Hoodie Table
* @param compactionCommitTime Compaction Commit Time
- * @param autoCommit Auto Commit
- * @param extraMetadata Extra Metadata to store
+ * @param autoCommit Auto Commit
+ * @param extraMetadata Extra Metadata to store
*/
protected void commitCompaction(JavaRDD<WriteStatus> compactedStatuses, HoodieTable<T> table,
- String compactionCommitTime, boolean autoCommit, Option<Map<String, String>> extraMetadata) {
+ String compactionCommitTime, boolean autoCommit, Option<Map<String, String>> extraMetadata) {
if (autoCommit) {
HoodieCommitMetadata metadata = doCompactionCommit(table, compactedStatuses, compactionCommitTime, extraMetadata);
if (compactionTimer != null) {
@@ -1107,7 +1107,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* Rollback failed compactions. Inflight rollbacks for compactions revert the .inflight file to the .requested file
*
* @param inflightInstant Inflight Compaction Instant
- * @param table Hoodie Table
+ * @param table Hoodie Table
*/
public void rollbackInflightCompaction(HoodieInstant inflightInstant, HoodieTable table) throws IOException {
table.rollback(jsc, inflightInstant, false);
@@ -1116,7 +1116,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
}
private HoodieCommitMetadata doCompactionCommit(HoodieTable<T> table, JavaRDD<WriteStatus> writeStatuses,
- String compactionCommitTime, Option<Map<String, String>> extraMetadata) {
+ String compactionCommitTime, Option<Map<String, String>> extraMetadata) {
HoodieTableMetaClient metaClient = table.getMetaClient();
List<HoodieWriteStat> updateStatusMap = writeStatuses.map(WriteStatus::getStat).collect();
[hudi] 31/40: HUDI-528 Handle empty commit in incremental pulling
(#1612)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 7c766c33ddc5171743181767d1c6ce8d237ab5db
Author: Gary Li <ya...@gmail.com>
AuthorDate: Thu May 14 22:55:25 2020 -0700
HUDI-528 Handle empty commit in incremental pulling (#1612)
---
.../hudi/common/table/TableSchemaResolver.java | 412 +++++++++++++++++++++
.../apache/hudi/common/util/HoodieAvroUtils.java | 14 +
.../org/apache/hudi/IncrementalRelation.scala | 42 +--
hudi-spark/src/test/scala/TestDataSource.scala | 8 +
4 files changed, 452 insertions(+), 24 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
new file mode 100644
index 0000000..d623b5a
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table;
+
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.table.log.HoodieLogFormat;
+import org.apache.hudi.common.table.log.HoodieLogFormat.Reader;
+import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock;
+import org.apache.hudi.common.table.log.block.HoodieLogBlock;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.util.HoodieAvroUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.InvalidTableException;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.SchemaCompatibility;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.parquet.avro.AvroSchemaConverter;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.schema.MessageType;
+
+import java.io.IOException;
+
+/**
+ * Helper class to read schema from data files and log files and to convert it between different formats.
+ */
+public class TableSchemaResolver {
+
+ private static final Logger LOG = LogManager.getLogger(TableSchemaResolver.class);
+ private HoodieTableMetaClient metaClient;
+
+ public TableSchemaResolver(HoodieTableMetaClient metaClient) {
+ this.metaClient = metaClient;
+ }
+
+ /**
+ * Gets the schema for a hoodie table. Depending on the type of table, read from any file written in the latest
+ * commit. We will assume that the schema has not changed within a single atomic write.
+ *
+ * @return Parquet schema for this table
+ * @throws Exception
+ */
+ private MessageType getTableParquetSchemaFromDataFile() throws Exception {
+ HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
+
+ try {
+ switch (metaClient.getTableType()) {
+ case COPY_ON_WRITE:
+ // If this is COW, get the last commit and read the schema from a file written in the
+ // last commit
+ HoodieInstant lastCommit =
+ activeTimeline.getCommitsTimeline().filterCompletedInstants().lastInstant().orElseThrow(() -> new InvalidTableException(metaClient.getBasePath()));
+ HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
+ .fromBytes(activeTimeline.getInstantDetails(lastCommit).get(), HoodieCommitMetadata.class);
+ String filePath = commitMetadata.getFileIdAndFullPaths(metaClient.getBasePath()).values().stream().findAny()
+ .orElseThrow(() -> new IllegalArgumentException("Could not find any data file written for commit "
+ + lastCommit + ", could not get schema for table " + metaClient.getBasePath() + ", Metadata :"
+ + commitMetadata));
+ return readSchemaFromBaseFile(new Path(filePath));
+ case MERGE_ON_READ:
+ // If this is MOR, depending on whether the latest commit is a delta commit or
+ // compaction commit
+ // Get a datafile written and get the schema from that file
+ Option<HoodieInstant> lastCompactionCommit =
+ metaClient.getActiveTimeline().getCommitTimeline().filterCompletedInstants().lastInstant();
+ LOG.info("Found the last compaction commit as " + lastCompactionCommit);
+
+ Option<HoodieInstant> lastDeltaCommit;
+ if (lastCompactionCommit.isPresent()) {
+ lastDeltaCommit = metaClient.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants()
+ .findInstantsAfter(lastCompactionCommit.get().getTimestamp(), Integer.MAX_VALUE).lastInstant();
+ } else {
+ lastDeltaCommit =
+ metaClient.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants().lastInstant();
+ }
+ LOG.info("Found the last delta commit " + lastDeltaCommit);
+
+ if (lastDeltaCommit.isPresent()) {
+ HoodieInstant lastDeltaInstant = lastDeltaCommit.get();
+ // read from the log file wrote
+ commitMetadata = HoodieCommitMetadata.fromBytes(activeTimeline.getInstantDetails(lastDeltaInstant).get(),
+ HoodieCommitMetadata.class);
+ Pair<String, HoodieFileFormat> filePathWithFormat =
+ commitMetadata.getFileIdAndFullPaths(metaClient.getBasePath()).values().stream()
+ .filter(s -> s.contains(HoodieLogFile.DELTA_EXTENSION)).findAny()
+ .map(f -> Pair.of(f, HoodieFileFormat.HOODIE_LOG)).orElseGet(() -> {
+ // No Log files in Delta-Commit. Check if there are any parquet files
+ return commitMetadata.getFileIdAndFullPaths(metaClient.getBasePath()).values().stream()
+ .filter(s -> s.contains((metaClient.getTableConfig().getBaseFileFormat().getFileExtension())))
+ .findAny().map(f -> Pair.of(f, HoodieFileFormat.PARQUET)).orElseThrow(() ->
+ new IllegalArgumentException("Could not find any data file written for commit "
+ + lastDeltaInstant + ", could not get schema for table " + metaClient.getBasePath()
+ + ", CommitMetadata :" + commitMetadata));
+ });
+ switch (filePathWithFormat.getRight()) {
+ case HOODIE_LOG:
+ return readSchemaFromLogFile(lastCompactionCommit, new Path(filePathWithFormat.getLeft()));
+ case PARQUET:
+ return readSchemaFromBaseFile(new Path(filePathWithFormat.getLeft()));
+ default:
+ throw new IllegalArgumentException("Unknown file format :" + filePathWithFormat.getRight()
+ + " for file " + filePathWithFormat.getLeft());
+ }
+ } else {
+ return readSchemaFromLastCompaction(lastCompactionCommit);
+ }
+ default:
+ LOG.error("Unknown table type " + metaClient.getTableType());
+ throw new InvalidTableException(metaClient.getBasePath());
+ }
+ } catch (IOException e) {
+ throw new HoodieException("Failed to read data schema", e);
+ }
+ }
+
+ private Schema getTableAvroSchemaFromDataFile() throws Exception {
+ return convertParquetSchemaToAvro(getTableParquetSchemaFromDataFile());
+ }
+
+ /**
+ * Gets full schema (user + metadata) for a hoodie table in Avro format.
+ *
+ * @return Avro schema for this table
+ * @throws Exception
+ */
+ public Schema getTableAvroSchema() throws Exception {
+ Option<Schema> schemaFromCommitMetadata = getTableSchemaFromCommitMetadata(true);
+ return schemaFromCommitMetadata.isPresent() ? schemaFromCommitMetadata.get() : getTableAvroSchemaFromDataFile();
+ }
+
+ /**
+ * Gets full schema (user + metadata) for a hoodie table in Parquet format.
+ *
+ * @return Parquet schema for the table
+ * @throws Exception
+ */
+ public MessageType getTableParquetSchema() throws Exception {
+ Option<Schema> schemaFromCommitMetadata = getTableSchemaFromCommitMetadata(true);
+ return schemaFromCommitMetadata.isPresent() ? convertAvroSchemaToParquet(schemaFromCommitMetadata.get()) :
+ getTableParquetSchemaFromDataFile();
+ }
+
+ /**
+ * Gets users data schema for a hoodie table in Avro format.
+ *
+ * @return Avro user data schema
+ * @throws Exception
+ */
+ public Schema getTableAvroSchemaWithoutMetadataFields() throws Exception {
+ Option<Schema> schemaFromCommitMetadata = getTableSchemaFromCommitMetadata(false);
+ return schemaFromCommitMetadata.isPresent() ? schemaFromCommitMetadata.get() :
+ HoodieAvroUtils.removeMetadataFields(getTableAvroSchemaFromDataFile());
+ }
+
+ /**
+ * Gets the schema for a hoodie table in Avro format from the HoodieCommitMetadata of the last commit.
+ *
+ * @return Avro schema for this table
+ */
+ private Option<Schema> getTableSchemaFromCommitMetadata(boolean includeMetadataFields) {
+ try {
+ HoodieTimeline timeline = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
+ byte[] data = timeline.getInstantDetails(timeline.lastInstant().get()).get();
+ HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(data, HoodieCommitMetadata.class);
+ String existingSchemaStr = metadata.getMetadata(HoodieCommitMetadata.SCHEMA_KEY);
+
+ if (StringUtils.isNullOrEmpty(existingSchemaStr)) {
+ return Option.empty();
+ }
+
+ Schema schema = new Schema.Parser().parse(existingSchemaStr);
+ if (includeMetadataFields) {
+ schema = HoodieAvroUtils.addMetadataFields(schema);
+ }
+ return Option.of(schema);
+ } catch (Exception e) {
+ throw new HoodieException("Failed to read schema from commit metadata", e);
+ }
+ }
+
+ /**
+ * Convert a parquet scheme to the avro format.
+ *
+ * @param parquetSchema The parquet schema to convert
+ * @return The converted avro schema
+ */
+ public Schema convertParquetSchemaToAvro(MessageType parquetSchema) {
+ AvroSchemaConverter avroSchemaConverter = new AvroSchemaConverter(metaClient.getHadoopConf());
+ return avroSchemaConverter.convert(parquetSchema);
+ }
+
+ /**
+ * Convert a avro scheme to the parquet format.
+ *
+ * @param schema The avro schema to convert
+ * @return The converted parquet schema
+ */
+ public MessageType convertAvroSchemaToParquet(Schema schema) {
+ AvroSchemaConverter avroSchemaConverter = new AvroSchemaConverter(metaClient.getHadoopConf());
+ return avroSchemaConverter.convert(schema);
+ }
+
+ /**
+ * HUDI specific validation of schema evolution. Ensures that a newer schema can be used for the dataset by
+ * checking if the data written using the old schema can be read using the new schema.
+ * <p>
+ * HUDI requires a Schema to be specified in HoodieWriteConfig and is used by the HoodieWriteClient to
+ * create the records. The schema is also saved in the data files (parquet format) and log files (avro format).
+ * Since a schema is required each time new data is ingested into a HUDI dataset, schema can be evolved over time.
+ * <p>
+ * New Schema is compatible only if:
+ * A1. There is no change in schema
+ * A2. A field has been added and it has a default value specified
+ * <p>
+ * New Schema is incompatible if:
+ * B1. A field has been deleted
+ * B2. A field has been renamed (treated as delete + add)
+ * B3. A field's type has changed to be incompatible with the older type
+ * <p>
+ * Issue with org.apache.avro.SchemaCompatibility:
+ * org.apache.avro.SchemaCompatibility checks schema compatibility between a writer schema (which originally wrote
+ * the AVRO record) and a readerSchema (with which we are reading the record). It ONLY guarantees that that each
+ * field in the reader record can be populated from the writer record. Hence, if the reader schema is missing a
+ * field, it is still compatible with the writer schema.
+ * <p>
+ * In other words, org.apache.avro.SchemaCompatibility was written to guarantee that we can read the data written
+ * earlier. It does not guarantee schema evolution for HUDI (B1 above).
+ * <p>
+ * Implementation: This function implements specific HUDI specific checks (listed below) and defers the remaining
+ * checks to the org.apache.avro.SchemaCompatibility code.
+ * <p>
+ * Checks:
+ * C1. If there is no change in schema: success
+ * C2. If a field has been deleted in new schema: failure
+ * C3. If a field has been added in new schema: it should have default value specified
+ * C4. If a field has been renamed(treated as delete + add): failure
+ * C5. If a field type has changed: failure
+ *
+ * @param oldSchema Older schema to check.
+ * @param newSchema Newer schema to check.
+ * @return True if the schema validation is successful
+ */
+ public static boolean isSchemaCompatible(Schema oldSchema, Schema newSchema) {
+ if (oldSchema.getType() == newSchema.getType() && newSchema.getType() == Schema.Type.RECORD) {
+ // record names must match:
+ if (!SchemaCompatibility.schemaNameEquals(oldSchema, newSchema)) {
+ return false;
+ }
+
+ // Check that each field in the oldSchema can populated the newSchema
+ for (final Field oldSchemaField : oldSchema.getFields()) {
+ final Field newSchemaField = SchemaCompatibility.lookupWriterField(newSchema, oldSchemaField);
+ if (newSchemaField == null) {
+ // C4 or C2: newSchema does not correspond to any field in the oldSchema
+ return false;
+ } else {
+ if (!isSchemaCompatible(oldSchemaField.schema(), newSchemaField.schema())) {
+ // C5: The fields do not have a compatible type
+ return false;
+ }
+ }
+ }
+
+ // Check that new fields added in newSchema have default values as they will not be
+ // present in oldSchema and hence cannot be populated on reading records from existing data.
+ for (final Field newSchemaField : newSchema.getFields()) {
+ final Field oldSchemaField = SchemaCompatibility.lookupWriterField(oldSchema, newSchemaField);
+ if (oldSchemaField == null) {
+ if (newSchemaField.defaultValue() == null) {
+ // C3: newly added field in newSchema does not have a default value
+ return false;
+ }
+ }
+ }
+
+ // All fields in the newSchema record can be populated from the oldSchema record
+ return true;
+ } else {
+ // Use the checks implemented by
+ org.apache.avro.SchemaCompatibility.SchemaPairCompatibility compatResult =
+ org.apache.avro.SchemaCompatibility.checkReaderWriterCompatibility(oldSchema, newSchema);
+ return compatResult.getType() == org.apache.avro.SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE;
+ }
+ }
+
+ public static boolean isSchemaCompatible(String oldSchema, String newSchema) {
+ return isSchemaCompatible(new Schema.Parser().parse(oldSchema), new Schema.Parser().parse(newSchema));
+ }
+
+ /**
+ * Read the parquet schema from a parquet File.
+ */
+ public MessageType readSchemaFromBaseFile(Path parquetFilePath) throws IOException {
+ LOG.info("Reading schema from " + parquetFilePath);
+
+ FileSystem fs = metaClient.getRawFs();
+ if (!fs.exists(parquetFilePath)) {
+ throw new IllegalArgumentException(
+ "Failed to read schema from data file " + parquetFilePath + ". File does not exist.");
+ }
+ ParquetMetadata fileFooter =
+ ParquetFileReader.readFooter(fs.getConf(), parquetFilePath, ParquetMetadataConverter.NO_FILTER);
+ return fileFooter.getFileMetaData().getSchema();
+ }
+
+ /**
+ * Read schema from a data file from the last compaction commit done.
+ *
+ * @throws Exception
+ */
+ public MessageType readSchemaFromLastCompaction(Option<HoodieInstant> lastCompactionCommitOpt) throws Exception {
+ HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
+
+ HoodieInstant lastCompactionCommit = lastCompactionCommitOpt.orElseThrow(() -> new Exception(
+ "Could not read schema from last compaction, no compaction commits found on path " + metaClient));
+
+ // Read from the compacted file wrote
+ HoodieCommitMetadata compactionMetadata = HoodieCommitMetadata
+ .fromBytes(activeTimeline.getInstantDetails(lastCompactionCommit).get(), HoodieCommitMetadata.class);
+ String filePath = compactionMetadata.getFileIdAndFullPaths(metaClient.getBasePath()).values().stream().findAny()
+ .orElseThrow(() -> new IllegalArgumentException("Could not find any data file written for compaction "
+ + lastCompactionCommit + ", could not get schema for table " + metaClient.getBasePath()));
+ return readSchemaFromBaseFile(new Path(filePath));
+ }
+
+ /**
+ * Read the schema from the log file on path.
+ *
+ * @return
+ */
+ public MessageType readSchemaFromLogFile(Path path) throws IOException {
+ FileSystem fs = metaClient.getRawFs();
+ Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(path), null);
+ HoodieAvroDataBlock lastBlock = null;
+ while (reader.hasNext()) {
+ HoodieLogBlock block = reader.next();
+ if (block instanceof HoodieAvroDataBlock) {
+ lastBlock = (HoodieAvroDataBlock) block;
+ }
+ }
+ reader.close();
+ if (lastBlock != null) {
+ return new AvroSchemaConverter().convert(lastBlock.getSchema());
+ }
+ return null;
+ }
+
+ /**
+ * Read the schema from the log file on path.
+ *
+ * @throws Exception
+ */
+ public MessageType readSchemaFromLogFile(Option<HoodieInstant> lastCompactionCommitOpt, Path path)
+ throws Exception {
+ MessageType messageType = readSchemaFromLogFile(path);
+ // Fall back to read the schema from last compaction
+ if (messageType == null) {
+ LOG.info("Falling back to read the schema from last compaction " + lastCompactionCommitOpt);
+ return readSchemaFromLastCompaction(lastCompactionCommitOpt);
+ }
+ return messageType;
+ }
+
+ /**
+ * Read the schema from the log file on path.
+ *
+ * @return
+ */
+ public static MessageType readSchemaFromLogFile(FileSystem fs, Path path) throws IOException {
+ Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(path), null);
+ HoodieAvroDataBlock lastBlock = null;
+ while (reader.hasNext()) {
+ HoodieLogBlock block = reader.next();
+ if (block instanceof HoodieAvroDataBlock) {
+ lastBlock = (HoodieAvroDataBlock) block;
+ }
+ }
+ reader.close();
+ if (lastBlock != null) {
+ return new AvroSchemaConverter().convert(lastBlock.getSchema());
+ }
+ return null;
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
index 32ea71e..e9dddff 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
@@ -97,6 +97,10 @@ public class HoodieAvroUtils {
|| HoodieRecord.FILENAME_METADATA_FIELD.equals(fieldName);
}
+ public static Schema createHoodieWriteSchema(Schema originalSchema) {
+ return HoodieAvroUtils.addMetadataFields(originalSchema);
+ }
+
/**
* Adds the Hoodie metadata fields to the given schema.
*/
@@ -177,6 +181,16 @@ public class HoodieAvroUtils {
return newSchema;
}
+ public static Schema removeMetadataFields(Schema schema) {
+ List<Schema.Field> filteredFields = schema.getFields()
+ .stream()
+ .filter(field -> !HoodieRecord.HOODIE_META_COLUMNS.contains(field.name()))
+ .collect(Collectors.toList());
+ Schema filteredSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), false);
+ filteredSchema.setFields(filteredFields);
+ return filteredSchema;
+ }
+
/**
* Adds the Hoodie commit metadata into the provided Generic Record.
*/
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/IncrementalRelation.scala b/hudi-spark/src/main/scala/org/apache/hudi/IncrementalRelation.scala
index a9e7389..9fca419 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/IncrementalRelation.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/IncrementalRelation.scala
@@ -18,10 +18,9 @@
package org.apache.hudi
import org.apache.hadoop.fs.GlobPattern
-import org.apache.hadoop.fs.Path
import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieRecord, HoodieTableType}
-import org.apache.hudi.common.table.HoodieTableMetaClient
-import org.apache.hudi.common.util.ParquetUtils
+import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
+import org.apache.hudi.common.util.HoodieAvroUtils
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.table.HoodieTable
@@ -35,11 +34,11 @@ import scala.collection.JavaConversions._
import scala.collection.mutable
/**
- * Relation, that implements the Hoodie incremental view.
- *
- * Implemented for Copy_on_write storage.
- *
- */
+ * Relation, that implements the Hoodie incremental view.
+ *
+ * Implemented for Copy_on_write storage.
+ *
+ */
class IncrementalRelation(val sqlContext: SQLContext,
val basePath: String,
val optParams: Map[String, String],
@@ -47,8 +46,7 @@ class IncrementalRelation(val sqlContext: SQLContext,
private val log = LogManager.getLogger(classOf[IncrementalRelation])
- val fs = new Path(basePath).getFileSystem(sqlContext.sparkContext.hadoopConfiguration)
- val metaClient = new HoodieTableMetaClient(sqlContext.sparkContext.hadoopConfiguration, basePath, true)
+ private val metaClient = new HoodieTableMetaClient(sqlContext.sparkContext.hadoopConfiguration, basePath, true)
// MOR tables not supported yet
if (metaClient.getTableType.equals(HoodieTableType.MERGE_ON_READ)) {
throw new HoodieException("Incremental view not implemented yet, for merge-on-read tables")
@@ -56,7 +54,7 @@ class IncrementalRelation(val sqlContext: SQLContext,
// TODO : Figure out a valid HoodieWriteConfig
private val hoodieTable = HoodieTable.getHoodieTable(metaClient, HoodieWriteConfig.newBuilder().withPath(basePath).build(),
sqlContext.sparkContext)
- val commitTimeline = hoodieTable.getMetaClient.getCommitTimeline.filterCompletedInstants()
+ private val commitTimeline = hoodieTable.getMetaClient.getCommitTimeline.filterCompletedInstants()
if (commitTimeline.empty()) {
throw new HoodieException("No instants to incrementally pull")
}
@@ -65,25 +63,21 @@ class IncrementalRelation(val sqlContext: SQLContext,
s"option ${DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY}")
}
- val lastInstant = commitTimeline.lastInstant().get()
+ private val lastInstant = commitTimeline.lastInstant().get()
- val commitsToReturn = commitTimeline.findInstantsInRange(
+ private val commitsToReturn = commitTimeline.findInstantsInRange(
optParams(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY),
optParams.getOrElse(DataSourceReadOptions.END_INSTANTTIME_OPT_KEY, lastInstant.getTimestamp))
.getInstants.iterator().toList
- // use schema from a file produced in the latest instant
- val latestSchema = {
- // use last instant if instant range is empty
- val instant = commitsToReturn.lastOption.getOrElse(lastInstant)
- val latestMeta = HoodieCommitMetadata
- .fromBytes(commitTimeline.getInstantDetails(instant).get, classOf[HoodieCommitMetadata])
- val metaFilePath = latestMeta.getFileIdAndFullPaths(basePath).values().iterator().next()
- AvroConversionUtils.convertAvroSchemaToStructType(ParquetUtils.readAvroSchema(
- sqlContext.sparkContext.hadoopConfiguration, new Path(metaFilePath)))
+ // use schema from latest metadata, if not present, read schema from the data file
+ private val latestSchema = {
+ val schemaUtil = new TableSchemaResolver(metaClient)
+ val tableSchema = HoodieAvroUtils.createHoodieWriteSchema(schemaUtil.getTableAvroSchemaWithoutMetadataFields);
+ AvroConversionUtils.convertAvroSchemaToStructType(tableSchema)
}
- val filters = {
+ private val filters = {
if (optParams.contains(DataSourceReadOptions.PUSH_DOWN_INCR_FILTERS_OPT_KEY)) {
val filterStr = optParams.getOrElse(
DataSourceReadOptions.PUSH_DOWN_INCR_FILTERS_OPT_KEY,
@@ -106,7 +100,7 @@ class IncrementalRelation(val sqlContext: SQLContext,
val pathGlobPattern = optParams.getOrElse(
DataSourceReadOptions.INCR_PATH_GLOB_OPT_KEY,
DataSourceReadOptions.DEFAULT_INCR_PATH_GLOB_OPT_VAL)
- val filteredFullPath = if(!pathGlobPattern.equals(DataSourceReadOptions.DEFAULT_INCR_PATH_GLOB_OPT_VAL)) {
+ val filteredFullPath = if (!pathGlobPattern.equals(DataSourceReadOptions.DEFAULT_INCR_PATH_GLOB_OPT_VAL)) {
val globMatcher = new GlobPattern("*" + pathGlobPattern)
fileIdToFullPath.filter(p => globMatcher.matches(p._2))
} else {
diff --git a/hudi-spark/src/test/scala/TestDataSource.scala b/hudi-spark/src/test/scala/TestDataSource.scala
index 6bd32b4..ab44c76 100644
--- a/hudi-spark/src/test/scala/TestDataSource.scala
+++ b/hudi-spark/src/test/scala/TestDataSource.scala
@@ -126,6 +126,14 @@ class TestDataSource extends AssertionsForJUnit {
assertEquals(1, countsPerCommit.length)
assertEquals(firstCommit, countsPerCommit(0).get(0))
+ // Upsert an empty dataFrame
+ val emptyRecords = DataSourceTestUtils.convertToStringList(dataGen.generateUpdates("002", 0)).toList
+ val emptyDF: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(emptyRecords, 1))
+ emptyDF.write.format("org.apache.hudi")
+ .options(commonOpts)
+ .mode(SaveMode.Append)
+ .save(basePath)
+
// pull the latest commit
val hoodieIncViewDF2 = spark.read.format("org.apache.hudi")
.option(DataSourceReadOptions.QUERY_TYPE_OPT_KEY, DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL)
[hudi] 02/40: [HUDI-652] Decouple HoodieReadClient and
AbstractHoodieClient to break the inheritance chain (#1372)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit dd7952ec9de7f9858ded98b49923e8978fd1df78
Author: vinoyang <ya...@gmail.com>
AuthorDate: Sat Mar 7 01:59:35 2020 +0800
[HUDI-652] Decouple HoodieReadClient and AbstractHoodieClient to break the inheritance chain (#1372)
* Removed timeline server support
* Removed try-with-resource
---
.../org/apache/hudi/client/HoodieReadClient.java | 9 ++-
.../apache/hudi/client/TestHoodieReadClient.java | 63 ++++++++---------
.../apache/hudi/table/TestMergeOnReadTable.java | 82 +++++++++++-----------
.../hudi/table/compact/TestAsyncCompaction.java | 25 +++----
.../main/java/org/apache/hudi/DataSourceUtils.java | 3 +-
5 files changed, 88 insertions(+), 94 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
index e08ec34..33d661b 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
@@ -25,7 +25,6 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
@@ -46,6 +45,7 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.types.StructType;
+import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -56,7 +56,7 @@ import scala.Tuple2;
/**
* Provides an RDD based API for accessing/filtering Hoodie tables, based on keys.
*/
-public class HoodieReadClient<T extends HoodieRecordPayload> extends AbstractHoodieClient {
+public class HoodieReadClient<T extends HoodieRecordPayload> implements Serializable {
private static final Logger LOG = LogManager.getLogger(HoodieReadClient.class);
@@ -65,9 +65,9 @@ public class HoodieReadClient<T extends HoodieRecordPayload> extends AbstractHoo
* basepath pointing to the table. Until, then just always assume a BloomIndex
*/
private final transient HoodieIndex<T> index;
- private final HoodieTimeline commitTimeline;
private HoodieTable hoodieTable;
private transient Option<SQLContext> sqlContextOpt;
+ private final transient JavaSparkContext jsc;
/**
* @param basePath path to Hoodie table
@@ -108,12 +108,11 @@ public class HoodieReadClient<T extends HoodieRecordPayload> extends AbstractHoo
*/
public HoodieReadClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig,
Option<EmbeddedTimelineService> timelineService) {
- super(jsc, clientConfig, timelineService);
+ this.jsc = jsc;
final String basePath = clientConfig.getBasePath();
// Create a Hoodie table which encapsulated the commits and files visible
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true);
this.hoodieTable = HoodieTable.getHoodieTable(metaClient, clientConfig, jsc);
- this.commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
this.index = HoodieIndex.createIndex(clientConfig, jsc);
this.sqlContextOpt = Option.empty();
}
diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java
index c57da14..6329e08 100644
--- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java
+++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieReadClient.java
@@ -96,8 +96,8 @@ public class TestHoodieReadClient extends TestHoodieClientBase {
*/
private void testReadFilterExist(HoodieWriteConfig config,
Function3<JavaRDD<WriteStatus>, HoodieWriteClient, JavaRDD<HoodieRecord>, String> writeFn) throws Exception {
- try (HoodieWriteClient writeClient = getHoodieWriteClient(config);
- HoodieReadClient readClient = getHoodieReadClient(config.getBasePath());) {
+ try (HoodieWriteClient writeClient = getHoodieWriteClient(config);) {
+ HoodieReadClient readClient = getHoodieReadClient(config.getBasePath());
String newCommitTime = writeClient.startCommit();
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
JavaRDD<HoodieRecord> recordsRDD = jsc.parallelize(records, 1);
@@ -113,37 +113,36 @@ public class TestHoodieReadClient extends TestHoodieClientBase {
// Verify there are no errors
assertNoWriteErrors(statuses);
- try (HoodieReadClient anotherReadClient = getHoodieReadClient(config.getBasePath());) {
- filteredRDD = anotherReadClient.filterExists(recordsRDD);
- List<HoodieRecord> result = filteredRDD.collect();
- // Check results
- assertEquals(25, result.size());
-
- // check path exists for written keys
- JavaPairRDD<HoodieKey, Option<String>> keyToPathPair =
- anotherReadClient.checkExists(recordsRDD.map(r -> r.getKey()));
- JavaRDD<HoodieKey> keysWithPaths = keyToPathPair.filter(keyPath -> keyPath._2.isPresent())
- .map(keyPath -> keyPath._1);
- assertEquals(75, keysWithPaths.count());
-
- // verify rows match inserted records
- Dataset<Row> rows = anotherReadClient.readROView(keysWithPaths, 1);
- assertEquals(75, rows.count());
-
- JavaRDD<HoodieKey> keysWithoutPaths = keyToPathPair.filter(keyPath -> !keyPath._2.isPresent())
- .map(keyPath -> keyPath._1);
-
- try {
- anotherReadClient.readROView(keysWithoutPaths, 1);
- } catch (Exception e) {
- // data frame reader throws exception for empty records. ignore the error.
- assertEquals(e.getClass(), AnalysisException.class);
- }
-
- // Actual tests of getPendingCompactions method are in TestAsyncCompaction
- // This is just testing empty list
- assertEquals(0, anotherReadClient.getPendingCompactions().size());
+ HoodieReadClient anotherReadClient = getHoodieReadClient(config.getBasePath());
+ filteredRDD = anotherReadClient.filterExists(recordsRDD);
+ List<HoodieRecord> result = filteredRDD.collect();
+ // Check results
+ assertEquals(25, result.size());
+
+ // check path exists for written keys
+ JavaPairRDD<HoodieKey, Option<String>> keyToPathPair =
+ anotherReadClient.checkExists(recordsRDD.map(r -> r.getKey()));
+ JavaRDD<HoodieKey> keysWithPaths = keyToPathPair.filter(keyPath -> keyPath._2.isPresent())
+ .map(keyPath -> keyPath._1);
+ assertEquals(75, keysWithPaths.count());
+
+ // verify rows match inserted records
+ Dataset<Row> rows = anotherReadClient.readROView(keysWithPaths, 1);
+ assertEquals(75, rows.count());
+
+ JavaRDD<HoodieKey> keysWithoutPaths = keyToPathPair.filter(keyPath -> !keyPath._2.isPresent())
+ .map(keyPath -> keyPath._1);
+
+ try {
+ anotherReadClient.readROView(keysWithoutPaths, 1);
+ } catch (Exception e) {
+ // data frame reader throws exception for empty records. ignore the error.
+ assertEquals(e.getClass(), AnalysisException.class);
}
+
+ // Actual tests of getPendingCompactions method are in TestAsyncCompaction
+ // This is just testing empty list
+ assertEquals(0, anotherReadClient.getPendingCompactions().size());
}
}
diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java
index ab27920..740caf2 100644
--- a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java
+++ b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java
@@ -759,54 +759,54 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
List<HoodieRecord> updatedRecords = dataGen.generateUpdates(newCommitTime, records);
JavaRDD<HoodieRecord> updatedRecordsRDD = jsc.parallelize(updatedRecords, 1);
- try (HoodieReadClient readClient = new HoodieReadClient(jsc, config);) {
- updatedRecords = readClient.tagLocation(updatedRecordsRDD).collect();
- // Write them to corresponding avro logfiles
- HoodieTestUtils.writeRecordsToLogFiles(metaClient.getFs(), metaClient.getBasePath(),
- HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS, updatedRecords);
+ HoodieReadClient readClient = new HoodieReadClient(jsc, config);
+ updatedRecords = readClient.tagLocation(updatedRecordsRDD).collect();
- // Verify that all data file has one log file
- metaClient = HoodieTableMetaClient.reload(metaClient);
- HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
- // In writeRecordsToLogFiles, no commit files are getting added, so resetting file-system view state
- ((SyncableFileSystemView) (table.getSliceView())).reset();
-
- for (String partitionPath : dataGen.getPartitionPaths()) {
- List<FileSlice> groupedLogFiles =
- table.getSliceView().getLatestFileSlices(partitionPath).collect(Collectors.toList());
- for (FileSlice fileSlice : groupedLogFiles) {
- assertEquals("There should be 1 log file written for every data file", 1, fileSlice.getLogFiles().count());
- }
+ // Write them to corresponding avro logfiles
+ HoodieTestUtils.writeRecordsToLogFiles(metaClient.getFs(), metaClient.getBasePath(),
+ HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS, updatedRecords);
+
+ // Verify that all data file has one log file
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
+ // In writeRecordsToLogFiles, no commit files are getting added, so resetting file-system view state
+ ((SyncableFileSystemView) (table.getSliceView())).reset();
+
+ for (String partitionPath : dataGen.getPartitionPaths()) {
+ List<FileSlice> groupedLogFiles =
+ table.getSliceView().getLatestFileSlices(partitionPath).collect(Collectors.toList());
+ for (FileSlice fileSlice : groupedLogFiles) {
+ assertEquals("There should be 1 log file written for every data file", 1, fileSlice.getLogFiles().count());
}
+ }
- // Mark 2nd delta-instant as completed
- metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(State.INFLIGHT,
- HoodieTimeline.DELTA_COMMIT_ACTION, newCommitTime));
- metaClient.getActiveTimeline().saveAsComplete(
- new HoodieInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, newCommitTime), Option.empty());
+ // Mark 2nd delta-instant as completed
+ metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(State.INFLIGHT,
+ HoodieTimeline.DELTA_COMMIT_ACTION, newCommitTime));
+ metaClient.getActiveTimeline().saveAsComplete(
+ new HoodieInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, newCommitTime), Option.empty());
- // Do a compaction
- String compactionInstantTime = writeClient.scheduleCompaction(Option.empty()).get().toString();
- JavaRDD<WriteStatus> result = writeClient.compact(compactionInstantTime);
+ // Do a compaction
+ String compactionInstantTime = writeClient.scheduleCompaction(Option.empty()).get().toString();
+ JavaRDD<WriteStatus> result = writeClient.compact(compactionInstantTime);
- // Verify that recently written compacted data file has no log file
- metaClient = HoodieTableMetaClient.reload(metaClient);
- table = HoodieTable.getHoodieTable(metaClient, config, jsc);
- HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
-
- assertTrue("Compaction commit should be > than last insert", HoodieTimeline
- .compareTimestamps(timeline.lastInstant().get().getTimestamp(), newCommitTime, HoodieTimeline.GREATER));
-
- for (String partitionPath : dataGen.getPartitionPaths()) {
- List<FileSlice> groupedLogFiles =
- table.getSliceView().getLatestFileSlices(partitionPath).collect(Collectors.toList());
- for (FileSlice slice : groupedLogFiles) {
- assertEquals("After compaction there should be no log files visible on a full view", 0, slice.getLogFiles().count());
- }
- List<WriteStatus> writeStatuses = result.collect();
- assertTrue(writeStatuses.stream().anyMatch(writeStatus -> writeStatus.getStat().getPartitionPath().contentEquals(partitionPath)));
+ // Verify that recently written compacted data file has no log file
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ table = HoodieTable.getHoodieTable(metaClient, config, jsc);
+ HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
+
+ assertTrue("Compaction commit should be > than last insert", HoodieTimeline
+ .compareTimestamps(timeline.lastInstant().get().getTimestamp(), newCommitTime, HoodieTimeline.GREATER));
+
+ for (String partitionPath : dataGen.getPartitionPaths()) {
+ List<FileSlice> groupedLogFiles =
+ table.getSliceView().getLatestFileSlices(partitionPath).collect(Collectors.toList());
+ for (FileSlice slice : groupedLogFiles) {
+ assertEquals("After compaction there should be no log files visible on a full view", 0, slice.getLogFiles().count());
}
+ List<WriteStatus> writeStatuses = result.collect();
+ assertTrue(writeStatuses.stream().anyMatch(writeStatus -> writeStatus.getStat().getPartitionPath().contentEquals(partitionPath)));
}
}
}
diff --git a/hudi-client/src/test/java/org/apache/hudi/table/compact/TestAsyncCompaction.java b/hudi-client/src/test/java/org/apache/hudi/table/compact/TestAsyncCompaction.java
index e81fa99..1a366a4 100644
--- a/hudi-client/src/test/java/org/apache/hudi/table/compact/TestAsyncCompaction.java
+++ b/hudi-client/src/test/java/org/apache/hudi/table/compact/TestAsyncCompaction.java
@@ -92,9 +92,8 @@ public class TestAsyncCompaction extends TestHoodieClientBase {
public void testRollbackForInflightCompaction() throws Exception {
// Rollback inflight compaction
HoodieWriteConfig cfg = getConfig(false);
- try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);
- HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());) {
-
+ try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);) {
+ HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());
String firstInstantTime = "001";
String secondInstantTime = "004";
String compactionInstantTime = "005";
@@ -155,9 +154,8 @@ public class TestAsyncCompaction extends TestHoodieClientBase {
int numRecs = 2000;
- try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);
- HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());) {
-
+ try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);) {
+ HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());
List<HoodieRecord> records = dataGen.generateInserts(firstInstantTime, numRecs);
records = runNextDeltaCommits(client, readClient, Arrays.asList(firstInstantTime, secondInstantTime), records, cfg, true,
new ArrayList<>());
@@ -197,9 +195,8 @@ public class TestAsyncCompaction extends TestHoodieClientBase {
public void testInflightCompaction() throws Exception {
// There is inflight compaction. Subsequent compaction run must work correctly
HoodieWriteConfig cfg = getConfig(true);
- try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);
- HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());) {
-
+ try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);) {
+ HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());
String firstInstantTime = "001";
String secondInstantTime = "004";
String compactionInstantTime = "005";
@@ -351,9 +348,8 @@ public class TestAsyncCompaction extends TestHoodieClientBase {
public void testCompactionAfterTwoDeltaCommits() throws Exception {
// No Delta Commits after compaction request
HoodieWriteConfig cfg = getConfig(true);
- try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);
- HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());) {
-
+ try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);) {
+ HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());
String firstInstantTime = "001";
String secondInstantTime = "004";
String compactionInstantTime = "005";
@@ -373,9 +369,8 @@ public class TestAsyncCompaction extends TestHoodieClientBase {
public void testInterleavedCompaction() throws Exception {
// Case: Two delta commits before and after compaction schedule
HoodieWriteConfig cfg = getConfig(true);
- try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);
- HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());) {
-
+ try (HoodieWriteClient client = getHoodieWriteClient(cfg, true);) {
+ HoodieReadClient readClient = getHoodieReadClient(cfg.getBasePath());
String firstInstantTime = "001";
String secondInstantTime = "004";
String compactionInstantTime = "005";
diff --git a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
index 1158fa2..35c5955 100644
--- a/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
+++ b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
@@ -185,7 +185,8 @@ public class DataSourceUtils {
@SuppressWarnings("unchecked")
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords,
HoodieWriteConfig writeConfig, Option<EmbeddedTimelineService> timelineService) {
- try (HoodieReadClient client = new HoodieReadClient<>(jssc, writeConfig, timelineService)) {
+ try {
+ HoodieReadClient client = new HoodieReadClient<>(jssc, writeConfig, timelineService);
return client.tagLocation(incomingHoodieRecords)
.filter(r -> !((HoodieRecord<HoodieRecordPayload>) r).isCurrentLocationKnown());
} catch (TableNotFoundException e) {
[hudi] 33/40: [HUDI-858] Allow multiple operations to be executed
within a single commit (#1633)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 555ff1aea4d7cc5947e890866b2a384b5409fb3a
Author: Balaji Varadarajan <va...@uber.com>
AuthorDate: Mon May 18 19:27:24 2020 -0700
[HUDI-858] Allow multiple operations to be executed within a single commit (#1633)
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 26 +++++++++++++++-
.../TestHoodieClientOnCopyOnWriteStorage.java | 36 +++++++++++++++++++++-
.../table/timeline/HoodieActiveTimeline.java | 20 ++++++++++--
3 files changed, 77 insertions(+), 5 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index f88d96a..24984db 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -96,6 +96,20 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
private static final String MAX_CONSISTENCY_CHECKS_PROP = "hoodie.consistency.check.max_checks";
private static int DEFAULT_MAX_CONSISTENCY_CHECKS = 7;
+ /**
+ * HUDI-858 : There are users who had been directly using RDD APIs and have relied on a behavior in 0.4.x to allow
+ * multiple write operations (upsert/buk-insert/...) to be executed within a single commit.
+ *
+ * Given Hudi commit protocol, these are generally unsafe operations and user need to handle failure scenarios. It
+ * only works with COW table. Hudi 0.5.x had stopped this behavior.
+ *
+ * Given the importance of supporting such cases for the user's migration to 0.5.x, we are proposing a safety flag
+ * (disabled by default) which will allow this old behavior.
+ */
+ private static final String ALLOW_MULTI_WRITE_ON_SAME_INSTANT =
+ "_.hoodie.allow.multi.write.on.same.instant";
+ private static final String DEFAULT_ALLOW_MULTI_WRITE_ON_SAME_INSTANT = "false";
+
private ConsistencyGuardConfig consistencyGuardConfig;
// Hoodie Write Client transparently rewrites File System View config when embedded mode is enabled
@@ -187,6 +201,10 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return StorageLevel.fromString(props.getProperty(WRITE_STATUS_STORAGE_LEVEL));
}
+ public boolean shouldAllowMultiWriteOnSameInstant() {
+ return Boolean.parseBoolean(props.getProperty(ALLOW_MULTI_WRITE_ON_SAME_INSTANT));
+ }
+
public String getWriteStatusClassName() {
return props.getProperty(HOODIE_WRITE_STATUS_CLASS_PROP);
}
@@ -706,6 +724,11 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return this;
}
+ public Builder withAllowMultiWriteOnSameInstant(boolean allow) {
+ props.setProperty(ALLOW_MULTI_WRITE_ON_SAME_INSTANT, String.valueOf(allow));
+ return this;
+ }
+
public HoodieWriteConfig build() {
// Check for mandatory properties
setDefaultOnCondition(props, !props.containsKey(INSERT_PARALLELISM), INSERT_PARALLELISM, DEFAULT_PARALLELISM);
@@ -721,6 +744,8 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
DEFAULT_COMBINE_BEFORE_UPSERT);
setDefaultOnCondition(props, !props.containsKey(COMBINE_BEFORE_DELETE_PROP), COMBINE_BEFORE_DELETE_PROP,
DEFAULT_COMBINE_BEFORE_DELETE);
+ setDefaultOnCondition(props, !props.containsKey(ALLOW_MULTI_WRITE_ON_SAME_INSTANT),
+ ALLOW_MULTI_WRITE_ON_SAME_INSTANT, DEFAULT_ALLOW_MULTI_WRITE_ON_SAME_INSTANT);
setDefaultOnCondition(props, !props.containsKey(WRITE_STATUS_STORAGE_LEVEL), WRITE_STATUS_STORAGE_LEVEL,
DEFAULT_WRITE_STATUS_STORAGE_LEVEL);
setDefaultOnCondition(props, !props.containsKey(HOODIE_AUTO_COMMIT_PROP), HOODIE_AUTO_COMMIT_PROP,
@@ -760,7 +785,6 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
// Ensure Layout Version is good
new TimelineLayoutVersion(Integer.parseInt(layoutVersion));
-
// Build WriteConfig at the end
HoodieWriteConfig config = new HoodieWriteConfig(props);
Objects.requireNonNull(config.getBasePath());
diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
index c7da7a7..bba8ecd 100644
--- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
@@ -20,8 +20,8 @@ package org.apache.hudi.client;
import org.apache.hudi.common.HoodieClientTestUtils;
import org.apache.hudi.common.HoodieTestDataGenerator;
-import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRollingStat;
@@ -981,6 +981,40 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase {
return Pair.of(markerFilePath, result);
}
+ @Test
+ public void testMultiOperationsPerCommit() throws IOException {
+ HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false)
+ .withAllowMultiWriteOnSameInstant(true)
+ .build();
+ HoodieWriteClient client = getHoodieWriteClient(cfg);
+ String firstInstantTime = "0000";
+ client.startCommitWithTime(firstInstantTime);
+ int numRecords = 200;
+ JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(dataGen.generateInserts(firstInstantTime, numRecords), 1);
+ JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, firstInstantTime);
+ assertTrue("Commit should succeed", client.commit(firstInstantTime, result));
+ assertTrue("After explicit commit, commit file should be created", HoodieTestUtils.doesCommitExist(basePath, firstInstantTime));
+
+ // Check the entire dataset has all records still
+ String[] fullPartitionPaths = new String[dataGen.getPartitionPaths().length];
+ for (int i = 0; i < fullPartitionPaths.length; i++) {
+ fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
+ }
+ assertEquals("Must contain " + numRecords + " records", numRecords,
+ HoodieClientTestUtils.read(jsc, basePath, sqlContext, fs, fullPartitionPaths).count());
+
+ String nextInstantTime = "0001";
+ client.startCommitWithTime(nextInstantTime);
+ JavaRDD<HoodieRecord> updateRecords = jsc.parallelize(dataGen.generateUpdates(nextInstantTime, numRecords), 1);
+ JavaRDD<HoodieRecord> insertRecords = jsc.parallelize(dataGen.generateInserts(nextInstantTime, numRecords), 1);
+ JavaRDD<WriteStatus> inserts = client.bulkInsert(insertRecords, nextInstantTime);
+ JavaRDD<WriteStatus> upserts = client.upsert(updateRecords, nextInstantTime);
+ assertTrue("Commit should succeed", client.commit(nextInstantTime, inserts.union(upserts)));
+ assertTrue("After explicit commit, commit file should be created", HoodieTestUtils.doesCommitExist(basePath, firstInstantTime));
+ int totalRecords = 2 * numRecords;
+ assertEquals("Must contain " + totalRecords + " records", totalRecords, HoodieClientTestUtils.read(jsc, basePath, sqlContext, fs, fullPartitionPaths).count());
+ }
+
/**
* Build Hoodie Write Config for small data file sizes.
*/
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
index 389314f..ab16428 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
@@ -307,11 +307,16 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
}
private void transitionState(HoodieInstant fromInstant, HoodieInstant toInstant, Option<byte[]> data) {
+ transitionState(fromInstant, toInstant, data, false);
+ }
+
+ private void transitionState(HoodieInstant fromInstant, HoodieInstant toInstant, Option<byte[]> data,
+ boolean allowRedundantTransitions) {
ValidationUtils.checkArgument(fromInstant.getTimestamp().equals(toInstant.getTimestamp()));
try {
if (metaClient.getTimelineLayoutVersion().isNullVersion()) {
// Re-create the .inflight file by opening a new file and write the commit metadata in
- createFileInMetaPath(fromInstant.getFileName(), data, false);
+ createFileInMetaPath(fromInstant.getFileName(), data, allowRedundantTransitions);
Path fromInstantPath = new Path(metaClient.getMetaPath(), fromInstant.getFileName());
Path toInstantPath = new Path(metaClient.getMetaPath(), toInstant.getFileName());
boolean success = metaClient.getFs().rename(fromInstantPath, toInstantPath);
@@ -324,7 +329,11 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
ValidationUtils.checkArgument(metaClient.getFs().exists(new Path(metaClient.getMetaPath(),
fromInstant.getFileName())));
// Use Write Once to create Target File
- createImmutableFileInPath(new Path(metaClient.getMetaPath(), toInstant.getFileName()), data);
+ if (allowRedundantTransitions) {
+ createFileInPath(new Path(metaClient.getMetaPath(), toInstant.getFileName()), data);
+ } else {
+ createImmutableFileInPath(new Path(metaClient.getMetaPath(), toInstant.getFileName()), data);
+ }
LOG.info("Create new file for toInstant ?" + new Path(metaClient.getMetaPath(), toInstant.getFileName()));
}
} catch (IOException e) {
@@ -367,9 +376,14 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
}
public void transitionRequestedToInflight(HoodieInstant requested, Option<byte[]> content) {
+ transitionRequestedToInflight(requested, content, false);
+ }
+
+ public void transitionRequestedToInflight(HoodieInstant requested, Option<byte[]> content,
+ boolean allowRedundantTransitions) {
HoodieInstant inflight = new HoodieInstant(State.INFLIGHT, requested.getAction(), requested.getTimestamp());
ValidationUtils.checkArgument(requested.isRequested(), "Instant " + requested + " in wrong state");
- transitionState(requested, inflight, content);
+ transitionState(requested, inflight, content, allowRedundantTransitions);
}
public void saveToCompactionRequested(HoodieInstant instant, Option<byte[]> content) {
[hudi] 28/40: [HUDI-789]Adjust logic of upsert in
HDFSParquetImporter (#1511)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8324f6ecb030227beca87d92031cefb08c21ae6e
Author: hongdd <jn...@163.com>
AuthorDate: Tue Apr 21 14:21:30 2020 +0800
[HUDI-789]Adjust logic of upsert in HDFSParquetImporter (#1511)
---
.../apache/hudi/utilities/HDFSParquetImporter.java | 22 +-
.../hudi/utilities/TestHDFSParquetImporter.java | 255 ++++++++++++++++-----
2 files changed, 217 insertions(+), 60 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java
index e566a3e..8f2b068 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java
@@ -99,6 +99,10 @@ public class HDFSParquetImporter implements Serializable {
}
+ private boolean isUpsert() {
+ return "upsert".equals(cfg.command.toLowerCase());
+ }
+
public int dataImport(JavaSparkContext jsc, int retry) {
this.fs = FSUtils.getFs(cfg.targetPath, jsc.hadoopConfiguration());
this.props = cfg.propsFilePath == null ? UtilHelpers.buildProperties(cfg.configs)
@@ -107,7 +111,7 @@ public class HDFSParquetImporter implements Serializable {
int ret = -1;
try {
// Verify that targetPath is not present.
- if (fs.exists(new Path(cfg.targetPath))) {
+ if (fs.exists(new Path(cfg.targetPath)) && !isUpsert()) {
throw new HoodieIOException(String.format("Make sure %s is not present.", cfg.targetPath));
}
do {
@@ -121,20 +125,22 @@ public class HDFSParquetImporter implements Serializable {
protected int dataImport(JavaSparkContext jsc) throws IOException {
try {
- if (fs.exists(new Path(cfg.targetPath))) {
+ if (fs.exists(new Path(cfg.targetPath)) && !isUpsert()) {
// cleanup target directory.
fs.delete(new Path(cfg.targetPath), true);
}
+ if (!fs.exists(new Path(cfg.targetPath))) {
+ // Initialize target hoodie table.
+ Properties properties = new Properties();
+ properties.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, cfg.tableName);
+ properties.put(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, cfg.tableType);
+ HoodieTableMetaClient.initTableAndGetMetaClient(jsc.hadoopConfiguration(), cfg.targetPath, properties);
+ }
+
// Get schema.
String schemaStr = UtilHelpers.parseSchema(fs, cfg.schemaFile);
- // Initialize target hoodie table.
- Properties properties = new Properties();
- properties.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, cfg.tableName);
- properties.put(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, cfg.tableType);
- HoodieTableMetaClient.initTableAndGetMetaClient(jsc.hadoopConfiguration(), cfg.targetPath, properties);
-
HoodieWriteClient client =
UtilHelpers.createHoodieClient(jsc, cfg.targetPath, schemaStr, cfg.parallelism, Option.empty(), props);
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java
index e4aac06..060e1c6 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java
@@ -20,6 +20,7 @@ package org.apache.hudi.utilities;
import org.apache.hudi.client.HoodieReadClient;
import org.apache.hudi.client.HoodieWriteClient;
+import org.apache.hudi.common.HoodieClientTestUtils;
import org.apache.hudi.common.HoodieTestDataGenerator;
import org.apache.hudi.common.minicluster.HdfsTestService;
import org.apache.hudi.common.model.HoodieTestUtils;
@@ -37,8 +38,13 @@ import org.apache.parquet.avro.AvroParquetWriter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
+
+import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -50,8 +56,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -75,34 +83,43 @@ public class TestHDFSParquetImporter implements Serializable {
}
@AfterClass
- public static void cleanupClass() throws Exception {
+ public static void cleanupClass() {
if (hdfsTestService != null) {
hdfsTestService.stop();
}
}
+ private String basePath;
+ private transient Path hoodieFolder;
+ private transient Path srcFolder;
+ private transient List<GenericRecord> insertData;
+
+ @Before
+ public void init() throws IOException, ParseException {
+ basePath = (new Path(dfsBasePath, Thread.currentThread().getStackTrace()[1].getMethodName())).toString();
+
+ // Hoodie root folder.
+ hoodieFolder = new Path(basePath, "testTarget");
+
+ // Create generic records.
+ srcFolder = new Path(basePath, "testSrc");
+ insertData = createInsertRecords(srcFolder);
+ }
+
+ @After
+ public void clean() throws IOException {
+ dfs.delete(new Path(basePath), true);
+ }
+
/**
* Test successful data import with retries.
*/
@Test
public void testImportWithRetries() throws Exception {
- JavaSparkContext jsc = null;
- try {
- jsc = getJavaSparkContext();
-
- // Test root folder.
- String basePath = (new Path(dfsBasePath, Thread.currentThread().getStackTrace()[1].getMethodName())).toString();
-
- // Hoodie root folder
- Path hoodieFolder = new Path(basePath, "testTarget");
-
+ try (JavaSparkContext jsc = getJavaSparkContext()) {
// Create schema file.
String schemaFile = new Path(basePath, "file.schema").toString();
- // Create generic records.
- Path srcFolder = new Path(basePath, "testSrc");
- createRecords(srcFolder);
-
HDFSParquetImporter.Config cfg = getHDFSParquetImporterConfig(srcFolder.toString(), hoodieFolder.toString(),
"testTable", "COPY_ON_WRITE", "_row_key", "timestamp", 1, schemaFile);
AtomicInteger retry = new AtomicInteger(3);
@@ -150,14 +167,104 @@ public class TestHDFSParquetImporter implements Serializable {
for (Entry<String, Long> e : recordCounts.entrySet()) {
assertEquals("missing records", 24, e.getValue().longValue());
}
- } finally {
- if (jsc != null) {
- jsc.stop();
- }
}
}
- private void createRecords(Path srcFolder) throws ParseException, IOException {
+ private void insert(JavaSparkContext jsc) throws IOException {
+ // Create schema file.
+ String schemaFile = new Path(basePath, "file.schema").toString();
+ createSchemaFile(schemaFile);
+
+ HDFSParquetImporter.Config cfg = getHDFSParquetImporterConfig(srcFolder.toString(), hoodieFolder.toString(),
+ "testTable", "COPY_ON_WRITE", "_row_key", "timestamp", 1, schemaFile);
+ HDFSParquetImporter dataImporter = new HDFSParquetImporter(cfg);
+
+ dataImporter.dataImport(jsc, 0);
+ }
+
+ /**
+ * Test successful insert and verify data consistency.
+ */
+ @Test
+ public void testImportWithInsert() throws IOException, ParseException {
+ try (JavaSparkContext jsc = getJavaSparkContext()) {
+ insert(jsc);
+ SQLContext sqlContext = new SQLContext(jsc);
+ Dataset<Row> ds = HoodieClientTestUtils.read(jsc, basePath + "/testTarget", sqlContext, dfs, basePath + "/testTarget/*/*/*/*");
+
+ List<Row> readData = ds.select("timestamp", "_row_key", "rider", "driver", "begin_lat", "begin_lon", "end_lat", "end_lon").collectAsList();
+ List<HoodieTripModel> result = readData.stream().map(row ->
+ new HoodieTripModel(row.getDouble(0), row.getString(1), row.getString(2), row.getString(3), row.getDouble(4),
+ row.getDouble(5), row.getDouble(6), row.getDouble(7)))
+ .collect(Collectors.toList());
+
+ List<HoodieTripModel> expected = insertData.stream().map(g ->
+ new HoodieTripModel(Double.parseDouble(g.get("timestamp").toString()),
+ g.get("_row_key").toString(),
+ g.get("rider").toString(),
+ g.get("driver").toString(),
+ Double.parseDouble(g.get("begin_lat").toString()),
+ Double.parseDouble(g.get("begin_lon").toString()),
+ Double.parseDouble(g.get("end_lat").toString()),
+ Double.parseDouble(g.get("end_lon").toString())))
+ .collect(Collectors.toList());
+
+ assertTrue(result.containsAll(expected) && expected.containsAll(result) && result.size() == expected.size());
+ }
+ }
+
+ /**
+ * Test upsert data and verify data consistency.
+ */
+ @Test
+ public void testImportWithUpsert() throws IOException, ParseException {
+ try (JavaSparkContext jsc = getJavaSparkContext()) {
+ insert(jsc);
+
+ // Create schema file.
+ String schemaFile = new Path(basePath, "file.schema").toString();
+
+ Path upsertFolder = new Path(basePath, "testUpsertSrc");
+ List<GenericRecord> upsertData = createUpsertRecords(upsertFolder);
+
+ HDFSParquetImporter.Config cfg = getHDFSParquetImporterConfig(upsertFolder.toString(), hoodieFolder.toString(),
+ "testTable", "COPY_ON_WRITE", "_row_key", "timestamp", 1, schemaFile);
+ cfg.command = "upsert";
+ HDFSParquetImporter dataImporter = new HDFSParquetImporter(cfg);
+
+ dataImporter.dataImport(jsc, 0);
+
+ // construct result, remove top 10 and add upsert data.
+ List<GenericRecord> expectData = insertData.subList(11, 96);
+ expectData.addAll(upsertData);
+
+ // read latest data
+ SQLContext sqlContext = new SQLContext(jsc);
+ Dataset<Row> ds = HoodieClientTestUtils.read(jsc, basePath + "/testTarget", sqlContext, dfs, basePath + "/testTarget/*/*/*/*");
+
+ List<Row> readData = ds.select("timestamp", "_row_key", "rider", "driver", "begin_lat", "begin_lon", "end_lat", "end_lon").collectAsList();
+ List<HoodieTripModel> result = readData.stream().map(row ->
+ new HoodieTripModel(row.getDouble(0), row.getString(1), row.getString(2), row.getString(3), row.getDouble(4),
+ row.getDouble(5), row.getDouble(6), row.getDouble(7)))
+ .collect(Collectors.toList());
+
+ // get expected result.
+ List<HoodieTripModel> expected = expectData.stream().map(g ->
+ new HoodieTripModel(Double.parseDouble(g.get("timestamp").toString()),
+ g.get("_row_key").toString(),
+ g.get("rider").toString(),
+ g.get("driver").toString(),
+ Double.parseDouble(g.get("begin_lat").toString()),
+ Double.parseDouble(g.get("begin_lon").toString()),
+ Double.parseDouble(g.get("end_lat").toString()),
+ Double.parseDouble(g.get("end_lon").toString())))
+ .collect(Collectors.toList());
+
+ assertTrue(result.containsAll(expected) && expected.containsAll(result) && result.size() == expected.size());
+ }
+ }
+
+ private List<GenericRecord> createInsertRecords(Path srcFolder) throws ParseException, IOException {
Path srcFile = new Path(srcFolder.toString(), "file1.parquet");
long startTime = HoodieActiveTimeline.COMMIT_FORMATTER.parse("20170203000000").getTime() / 1000;
List<GenericRecord> records = new ArrayList<GenericRecord>();
@@ -165,12 +272,36 @@ public class TestHDFSParquetImporter implements Serializable {
records.add(HoodieTestDataGenerator.generateGenericRecord(Long.toString(recordNum), "rider-" + recordNum,
"driver-" + recordNum, startTime + TimeUnit.HOURS.toSeconds(recordNum)));
}
- ParquetWriter<GenericRecord> writer = AvroParquetWriter.<GenericRecord>builder(srcFile)
- .withSchema(HoodieTestDataGenerator.AVRO_SCHEMA).withConf(HoodieTestUtils.getDefaultHadoopConf()).build();
- for (GenericRecord record : records) {
- writer.write(record);
+ try (ParquetWriter<GenericRecord> writer = AvroParquetWriter.<GenericRecord>builder(srcFile)
+ .withSchema(HoodieTestDataGenerator.AVRO_SCHEMA).withConf(HoodieTestUtils.getDefaultHadoopConf()).build()) {
+ for (GenericRecord record : records) {
+ writer.write(record);
+ }
}
- writer.close();
+ return records;
+ }
+
+ private List<GenericRecord> createUpsertRecords(Path srcFolder) throws ParseException, IOException {
+ Path srcFile = new Path(srcFolder.toString(), "file1.parquet");
+ long startTime = HoodieActiveTimeline.COMMIT_FORMATTER.parse("20170203000000").getTime() / 1000;
+ List<GenericRecord> records = new ArrayList<GenericRecord>();
+ // 10 for update
+ for (long recordNum = 0; recordNum < 11; recordNum++) {
+ records.add(HoodieTestDataGenerator.generateGenericRecord(Long.toString(recordNum), "rider-upsert-" + recordNum,
+ "driver-upsert" + recordNum, startTime + TimeUnit.HOURS.toSeconds(recordNum)));
+ }
+ // 4 for insert
+ for (long recordNum = 96; recordNum < 100; recordNum++) {
+ records.add(HoodieTestDataGenerator.generateGenericRecord(Long.toString(recordNum), "rider-upsert-" + recordNum,
+ "driver-upsert" + recordNum, startTime + TimeUnit.HOURS.toSeconds(recordNum)));
+ }
+ try (ParquetWriter<GenericRecord> writer = AvroParquetWriter.<GenericRecord>builder(srcFile)
+ .withSchema(HoodieTestDataGenerator.AVRO_SCHEMA).withConf(HoodieTestUtils.getDefaultHadoopConf()).build()) {
+ for (GenericRecord record : records) {
+ writer.write(record);
+ }
+ }
+ return records;
}
private void createSchemaFile(String schemaFile) throws IOException {
@@ -184,12 +315,7 @@ public class TestHDFSParquetImporter implements Serializable {
*/
@Test
public void testSchemaFile() throws Exception {
- JavaSparkContext jsc = null;
- try {
- jsc = getJavaSparkContext();
-
- // Test root folder.
- String basePath = (new Path(dfsBasePath, Thread.currentThread().getStackTrace()[1].getMethodName())).toString();
+ try (JavaSparkContext jsc = getJavaSparkContext()) {
// Hoodie root folder
Path hoodieFolder = new Path(basePath, "testTarget");
Path srcFolder = new Path(basePath.toString(), "srcTest");
@@ -204,10 +330,6 @@ public class TestHDFSParquetImporter implements Serializable {
// Should fail - return : -1.
assertEquals(-1, dataImporter.dataImport(jsc, 0));
- } finally {
- if (jsc != null) {
- jsc.stop();
- }
}
}
@@ -216,19 +338,7 @@ public class TestHDFSParquetImporter implements Serializable {
*/
@Test
public void testRowAndPartitionKey() throws Exception {
- JavaSparkContext jsc = null;
- try {
- jsc = getJavaSparkContext();
-
- // Test root folder.
- String basePath = (new Path(dfsBasePath, Thread.currentThread().getStackTrace()[1].getMethodName())).toString();
- // Hoodie root folder
- Path hoodieFolder = new Path(basePath, "testTarget");
-
- // Create generic records.
- Path srcFolder = new Path(basePath, "testSrc");
- createRecords(srcFolder);
-
+ try (JavaSparkContext jsc = getJavaSparkContext()) {
// Create schema file.
Path schemaFile = new Path(basePath.toString(), "missingFile.schema");
createSchemaFile(schemaFile.toString());
@@ -248,10 +358,6 @@ public class TestHDFSParquetImporter implements Serializable {
dataImporter = new HDFSParquetImporter(cfg);
assertEquals(-1, dataImporter.dataImport(jsc, 0));
- } finally {
- if (jsc != null) {
- jsc.stop();
- }
}
}
@@ -275,4 +381,49 @@ public class TestHDFSParquetImporter implements Serializable {
sparkConf = HoodieWriteClient.registerClasses(sparkConf);
return new JavaSparkContext(HoodieReadClient.addHoodieSupport(sparkConf));
}
+
+ /**
+ * Class used for compare result and expected.
+ */
+ public static class HoodieTripModel {
+ double timestamp;
+ String rowKey;
+ String rider;
+ String driver;
+ double beginLat;
+ double beginLon;
+ double endLat;
+ double endLon;
+
+ private HoodieTripModel(double timestamp, String rowKey, String rider, String driver, double beginLat,
+ double beginLon, double endLat, double endLon) {
+ this.timestamp = timestamp;
+ this.rowKey = rowKey;
+ this.rider = rider;
+ this.driver = driver;
+ this.beginLat = beginLat;
+ this.beginLon = beginLon;
+ this.endLat = endLat;
+ this.endLon = endLon;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ HoodieTripModel other = (HoodieTripModel) o;
+ return timestamp == other.timestamp && rowKey.equals(other.rowKey) && rider.equals(other.rider)
+ && driver.equals(other.driver) && beginLat == other.beginLat && beginLon == other.beginLon
+ && endLat == other.endLat && endLon == other.endLon;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestamp, rowKey, rider, driver, beginLat, beginLon, endLat, endLon);
+ }
+ }
}
[hudi] 08/40: [HUDI-656][Performance] Return a dummy Spark relation
after writing the DataFrame (#1394)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 9a2b3e3c93f670c79adb3d5cba69cdd3f551005a
Author: Udit Mehrotra <um...@illinois.edu>
AuthorDate: Wed Mar 11 20:27:46 2020 -0700
[HUDI-656][Performance] Return a dummy Spark relation after writing the DataFrame (#1394)
Co-authored-by: Mehrotra <ud...@amazon.com>
---
.../main/scala/org/apache/hudi/DefaultSource.scala | 20 ++++++++++++-
.../scala/org/apache/hudi/HudiEmptyRelation.scala | 35 ++++++++++++++++++++++
2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala b/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala
index 931f23b..fbdd4ea 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala
@@ -22,6 +22,7 @@ import org.apache.hudi.exception.HoodieException
import org.apache.hudi.hadoop.HoodieROTablePathFilter
import org.apache.log4j.LogManager
import org.apache.spark.sql.execution.datasources.DataSource
+import org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand
import org.apache.spark.sql.execution.streaming.Sink
import org.apache.spark.sql.sources._
import org.apache.spark.sql.streaming.OutputMode
@@ -82,6 +83,22 @@ class DefaultSource extends RelationProvider
}
}
+ /**
+ * This DataSource API is used for writing the DataFrame at the destination. For now, we are returning a dummy
+ * relation here because Spark does not really make use of the relation returned, and just returns an empty
+ * dataset at [[SaveIntoDataSourceCommand.run()]]. This saves us the cost of creating and returning a parquet
+ * relation here.
+ *
+ * TODO: Revisit to return a concrete relation here when we support CREATE TABLE AS for Hudi with DataSource API.
+ * That is the only case where Spark seems to actually need a relation to be returned here
+ * [[DataSource.writeAndRead()]]
+ *
+ * @param sqlContext Spark SQL Context
+ * @param mode Mode for saving the DataFrame at the destination
+ * @param optParams Parameters passed as part of the DataFrame write operation
+ * @param df Spark DataFrame to be written
+ * @return Spark Relation
+ */
override def createRelation(sqlContext: SQLContext,
mode: SaveMode,
optParams: Map[String, String],
@@ -89,7 +106,8 @@ class DefaultSource extends RelationProvider
val parameters = HoodieSparkSqlWriter.parametersWithWriteDefaults(optParams)
HoodieSparkSqlWriter.write(sqlContext, mode, parameters, df)
- createRelation(sqlContext, parameters, df.schema)
+
+ new HudiEmptyRelation(sqlContext, df.schema)
}
override def createSink(sqlContext: SQLContext,
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/HudiEmptyRelation.scala b/hudi-spark/src/main/scala/org/apache/hudi/HudiEmptyRelation.scala
new file mode 100644
index 0000000..8ddbe46
--- /dev/null
+++ b/hudi-spark/src/main/scala/org/apache/hudi/HudiEmptyRelation.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi
+
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.sources.BaseRelation
+import org.apache.spark.sql.types.StructType
+
+/**
+ * This is a dummy Spark relation that can be used if needed to return a place holder relation that does not get used.
+ *
+ * @param sqlContext Spark SQL Context
+ * @param userSchema Users data schema
+ */
+class HudiEmptyRelation(val sqlContext: SQLContext,
+ val userSchema: StructType) extends BaseRelation {
+
+ override def schema: StructType = userSchema
+}
[hudi] 18/40: [MINOR] Update DOAP with 0.5.2 Release (#1448)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 45efa695c15f64294e84506a8e8cb19aba1af7df
Author: Suneel Marthi <sm...@apache.org>
AuthorDate: Wed Mar 25 23:37:32 2020 -0400
[MINOR] Update DOAP with 0.5.2 Release (#1448)
---
doap_HUDI.rdf | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/doap_HUDI.rdf b/doap_HUDI.rdf
index c33d201..af45a41 100644
--- a/doap_HUDI.rdf
+++ b/doap_HUDI.rdf
@@ -46,6 +46,11 @@
<created>2020-01-31</created>
<revision>0.5.1</revision>
</Version>
+ <Version>
+ <name>Apache Hudi-incubating 0.5.2</name>
+ <created>2020-03-26</created>
+ <revision>0.5.2</revision>
+ </Version>
</release>
<repository>
<GitRepository>
[hudi] 15/40: [HUDI-782] Add support of Aliyun object storage
service. (#1506)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit cc4aa45ae28bbd4135d1c65f38a2dcd5de749418
Author: Shen Hong <sh...@126.com>
AuthorDate: Sun Apr 12 10:06:30 2020 +0800
[HUDI-782] Add support of Aliyun object storage service. (#1506)
---
.../src/main/java/org/apache/hudi/common/storage/StorageSchemes.java | 2 ++
.../test/java/org/apache/hudi/common/storage/TestStorageSchemes.java | 1 +
2 files changed, 3 insertions(+)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/storage/StorageSchemes.java b/hudi-common/src/main/java/org/apache/hudi/common/storage/StorageSchemes.java
index 316ff0d..7286683 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/storage/StorageSchemes.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/storage/StorageSchemes.java
@@ -42,6 +42,8 @@ public enum StorageSchemes {
ADL("adl", false),
// Azure ADLS Gen2
ABFS("abfs", false), ABFSS("abfss", false),
+ // Aliyun OSS
+ OSS("oss", false),
// View FS for federated setups. If federating across cloud stores, then append support is false
VIEWFS("viewfs", true);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java b/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java
index ebd3d78..f6c229f 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java
@@ -38,6 +38,7 @@ public class TestStorageSchemes {
assertFalse(StorageSchemes.isAppendSupported("wasb"));
assertFalse(StorageSchemes.isAppendSupported("adl"));
assertFalse(StorageSchemes.isAppendSupported("abfs"));
+ assertFalse(StorageSchemes.isAppendSupported("oss"));
assertTrue(StorageSchemes.isAppendSupported("viewfs"));
try {
StorageSchemes.isAppendSupported("s2");
[hudi] 30/40: [HUDI-820] cleaner repair command should only inspect
clean metadata files (#1542)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 4095de4d45f174b45dd1310a6a99c4fecfb2466a
Author: Balaji Varadarajan <va...@uber.com>
AuthorDate: Sun May 10 18:25:54 2020 -0700
[HUDI-820] cleaner repair command should only inspect clean metadata files (#1542)
---
hudi-cli/pom.xml | 21 +++
.../apache/hudi/cli/HoodieTableHeaderFields.java | 36 ++++
.../apache/hudi/cli/commands/RepairsCommand.java | 21 ++-
.../cli/commands/AbstractShellIntegrationTest.java | 61 ++++++
.../HoodieTestCommitMetadataGenerator.java | 137 ++++++++++++++
.../hudi/cli/commands/TestRepairsCommand.java | 206 +++++++++++++++++++++
.../hudi/common/HoodieTestDataGenerator.java | 17 +-
.../apache/hudi/common/util/CollectionUtils.java | 111 +++++++++++
8 files changed, 601 insertions(+), 9 deletions(-)
diff --git a/hudi-cli/pom.xml b/hudi-cli/pom.xml
index a5e358c..7b63e23 100644
--- a/hudi-cli/pom.xml
+++ b/hudi-cli/pom.xml
@@ -150,6 +150,27 @@
<artifactId>hudi-utilities_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-client</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-utilities_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
<!-- Logging -->
<dependency>
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
new file mode 100644
index 0000000..2e3bc01
--- /dev/null
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli;
+
+/**
+ * Fields of print table header.
+ */
+public class HoodieTableHeaderFields {
+
+ public static final String HEADER_PARTITION = "Partition";
+ public static final String HEADER_PARTITION_PATH = HEADER_PARTITION + " Path";
+ /**
+ * Fields of Repair.
+ */
+ public static final String HEADER_METADATA_PRESENT = "Metadata Present?";
+ public static final String HEADER_REPAIR_ACTION = "Action";
+ public static final String HEADER_HOODIE_PROPERTY = "Property";
+ public static final String HEADER_OLD_VALUE = "Old Value";
+ public static final String HEADER_NEW_VALUE = "New Value";
+}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 7a65336..7feebed 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -25,10 +25,12 @@ import org.apache.hudi.cli.utils.SparkUtil;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.avro.AvroRuntimeException;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import org.apache.spark.launcher.SparkLauncher;
@@ -147,14 +149,21 @@ public class RepairsCommand implements CommandMarker {
public void removeCorruptedPendingCleanAction() {
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
- HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
-
- activeTimeline.filterInflightsAndRequested().getInstants().forEach(instant -> {
+ HoodieTimeline cleanerTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline().getCleanerTimeline();
+ LOG.info("Inspecting pending clean metadata in timeline for corrupted files");
+ cleanerTimeline.filterInflightsAndRequested().getInstants().forEach(instant -> {
try {
CleanerUtils.getCleanerPlan(client, instant);
- } catch (IOException e) {
- LOG.warn("try to remove corrupted instant file: " + instant);
+ } catch (AvroRuntimeException e) {
+ LOG.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant);
FSUtils.deleteInstantFile(client.getFs(), client.getMetaPath(), instant);
+ } catch (IOException ioe) {
+ if (ioe.getMessage().contains("Not an Avro data file")) {
+ LOG.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant);
+ FSUtils.deleteInstantFile(client.getFs(), client.getMetaPath(), instant);
+ } else {
+ throw new HoodieIOException(ioe.getMessage(), ioe);
+ }
}
});
}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/AbstractShellIntegrationTest.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/AbstractShellIntegrationTest.java
new file mode 100644
index 0000000..ad81af5
--- /dev/null
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/AbstractShellIntegrationTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.common.HoodieClientTestHarness;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.springframework.shell.Bootstrap;
+import org.springframework.shell.core.JLineShellComponent;
+
+/**
+ * Class to start Bootstrap and JLineShellComponent.
+ */
+public abstract class AbstractShellIntegrationTest extends HoodieClientTestHarness {
+
+ private static JLineShellComponent shell;
+
+ @BeforeClass
+ public static void startup() {
+ Bootstrap bootstrap = new Bootstrap();
+ shell = bootstrap.getJLineShellComponent();
+ }
+
+ @AfterClass
+ public static void shutdown() {
+ shell.stop();
+ }
+
+ @Before
+ public void setup() throws Exception {
+ initResources();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ cleanupResources();
+ }
+
+ protected static JLineShellComponent getShell() {
+ return shell;
+ }
+}
\ No newline at end of file
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/HoodieTestCommitMetadataGenerator.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/HoodieTestCommitMetadataGenerator.java
new file mode 100644
index 0000000..7abad66
--- /dev/null
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/HoodieTestCommitMetadataGenerator.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.common.HoodieTestDataGenerator;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieTestUtils;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTimeline;
+import org.apache.hudi.common.util.CollectionUtils;
+import org.apache.hudi.common.util.FSUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.exception.HoodieIOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to be used in tests to keep generating test inserts and updates against a corpus.
+ */
+public class HoodieTestCommitMetadataGenerator extends HoodieTestDataGenerator {
+
+ // default commit metadata value
+ public static final String DEFAULT_PATH = "path";
+ public static final String DEFAULT_FILEID = "fileId";
+ public static final int DEFAULT_TOTAL_WRITE_BYTES = 50;
+ public static final String DEFAULT_PRE_COMMIT = "commit-1";
+ public static final int DEFAULT_NUM_WRITES = 10;
+ public static final int DEFAULT_NUM_UPDATE_WRITES = 15;
+ public static final int DEFAULT_TOTAL_LOG_BLOCKS = 1;
+ public static final int DEFAULT_TOTAL_LOG_RECORDS = 10;
+ public static final int DEFAULT_OTHER_VALUE = 0;
+ public static final String DEFAULT_NULL_VALUE = "null";
+
+ /**
+ * Create a commit file with default CommitMetadata.
+ */
+ public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) {
+ createCommitFileWithMetadata(basePath, commitTime, configuration, Option.empty(), Option.empty());
+ }
+
+ public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
+ Option<Integer> writes, Option<Integer> updates) {
+ Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
+ HoodieTimeline.makeRequestedCommitFileName(commitTime))
+ .forEach(f -> {
+ Path commitFile = new Path(
+ basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + f);
+ FSDataOutputStream os = null;
+ try {
+ FileSystem fs = FSUtils.getFs(basePath, configuration);
+ os = fs.create(commitFile, true);
+ // Generate commitMetadata
+ HoodieCommitMetadata commitMetadata = generateCommitMetadata(basePath, commitTime, writes, updates);
+ // Write empty commit metadata
+ os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
+ } catch (IOException ioe) {
+ throw new HoodieIOException(ioe.getMessage(), ioe);
+ } finally {
+ if (null != os) {
+ try {
+ os.close();
+ } catch (IOException e) {
+ throw new HoodieIOException(e.getMessage(), e);
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * Generate commitMetadata in path.
+ */
+ public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws IOException {
+ return generateCommitMetadata(basePath, commitTime, Option.empty(), Option.empty());
+ }
+
+ public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime,
+ Option<Integer> writes, Option<Integer> updates) throws IOException {
+ String file1P0C0 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime);
+ String file1P1C0 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime);
+ return generateCommitMetadata(new HashMap<String, List<String>>() {
+ {
+ put(DEFAULT_FIRST_PARTITION_PATH, CollectionUtils.createImmutableList(file1P0C0));
+ put(DEFAULT_SECOND_PARTITION_PATH, CollectionUtils.createImmutableList(file1P1C0));
+ }
+ }, writes, updates);
+ }
+
+ /**
+ * Method to generate commit metadata.
+ */
+ private static HoodieCommitMetadata generateCommitMetadata(Map<String, List<String>> partitionToFilePaths,
+ Option<Integer> writes, Option<Integer> updates) {
+ HoodieCommitMetadata metadata = new HoodieCommitMetadata();
+ partitionToFilePaths.forEach((key, value) -> value.forEach(f -> {
+ HoodieWriteStat writeStat = new HoodieWriteStat();
+ writeStat.setPartitionPath(key);
+ writeStat.setPath(DEFAULT_PATH);
+ writeStat.setFileId(DEFAULT_FILEID);
+ writeStat.setTotalWriteBytes(DEFAULT_TOTAL_WRITE_BYTES);
+ writeStat.setPrevCommit(DEFAULT_PRE_COMMIT);
+ writeStat.setNumWrites(writes.orElse(DEFAULT_NUM_WRITES));
+ writeStat.setNumUpdateWrites(updates.orElse(DEFAULT_NUM_UPDATE_WRITES));
+ writeStat.setTotalLogBlocks(DEFAULT_TOTAL_LOG_BLOCKS);
+ writeStat.setTotalLogRecords(DEFAULT_TOTAL_LOG_RECORDS);
+ metadata.addWriteStat(key, writeStat);
+ }));
+ return metadata;
+ }
+}
\ No newline at end of file
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
new file mode 100644
index 0000000..1192915
--- /dev/null
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.HoodiePrintHelper;
+import org.apache.hudi.cli.HoodieTableHeaderFields;
+import org.apache.hudi.common.HoodieTestDataGenerator;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.TimelineLayoutVersion;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.FSUtils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.shell.core.CommandResult;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Test class for {@link RepairsCommand}.
+ */
+public class TestRepairsCommand extends AbstractShellIntegrationTest {
+
+ private String tablePath;
+
+ @Before
+ public void init() throws IOException {
+ String tableName = "test_table";
+ tablePath = basePath + File.separator + tableName;
+
+ // Create table and connect
+ new TableCommand().createTable(
+ tablePath, "test_table", HoodieTableType.COPY_ON_WRITE.name(),
+ "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
+ }
+
+ /**
+ * Test case for dry run 'repair addpartitionmeta'.
+ */
+ @Test
+ public void testAddPartitionMetaWithDryRun() throws IOException {
+ // create commit instant
+ Files.createFile(Paths.get(tablePath + "/.hoodie/100.commit"));
+
+ // create partition path
+ String partition1 = tablePath + File.separator + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+ String partition2 = tablePath + File.separator + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+ String partition3 = tablePath + File.separator + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
+ assertTrue(fs.mkdirs(new Path(partition1)));
+ assertTrue(fs.mkdirs(new Path(partition2)));
+ assertTrue(fs.mkdirs(new Path(partition3)));
+
+ // default is dry run.
+ CommandResult cr = getShell().executeCommand("repair addpartitionmeta");
+ assertTrue(cr.isSuccess());
+
+ // expected all 'No'.
+ String[][] rows = FSUtils.getAllPartitionFoldersThreeLevelsDown(fs, tablePath)
+ .stream()
+ .map(partition -> new String[] {partition, "No", "None"})
+ .toArray(String[][]::new);
+ String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
+ HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows);
+
+ assertEquals(expected, cr.getResult().toString());
+ }
+
+ /**
+ * Test case for real run 'repair addpartitionmeta'.
+ */
+ @Test
+ public void testAddPartitionMetaWithRealRun() throws IOException {
+ // create commit instant
+ Files.createFile(Paths.get(tablePath + "/.hoodie/100.commit"));
+
+ // create partition path
+ String partition1 = tablePath + File.separator + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+ String partition2 = tablePath + File.separator + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+ String partition3 = tablePath + File.separator + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
+ assertTrue(fs.mkdirs(new Path(partition1)));
+ assertTrue(fs.mkdirs(new Path(partition2)));
+ assertTrue(fs.mkdirs(new Path(partition3)));
+
+ CommandResult cr = getShell().executeCommand("repair addpartitionmeta --dryrun false");
+ assertTrue(cr.isSuccess());
+
+ List<String> paths = FSUtils.getAllPartitionFoldersThreeLevelsDown(fs, tablePath);
+ // after dry run, the action will be 'Repaired'
+ String[][] rows = paths.stream()
+ .map(partition -> new String[] {partition, "No", "Repaired"})
+ .toArray(String[][]::new);
+ String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
+ HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows);
+
+ assertEquals(expected, cr.getResult().toString());
+
+ cr = getShell().executeCommand("repair addpartitionmeta");
+
+ // after real run, Metadata is present now.
+ rows = paths.stream()
+ .map(partition -> new String[] {partition, "Yes", "None"})
+ .toArray(String[][]::new);
+ expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
+ HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows);
+ assertEquals(expected, cr.getResult().toString());
+ }
+
+ /**
+ * Test case for 'repair overwrite-hoodie-props'.
+ */
+ @Test
+ public void testOverwriteHoodieProperties() throws IOException {
+ URL newProps = this.getClass().getClassLoader().getResource("table-config.properties");
+ assertNotNull("New property file must exist", newProps);
+
+ CommandResult cr = getShell().executeCommand("repair overwrite-hoodie-props --new-props-file " + newProps.getPath());
+ assertTrue(cr.isSuccess());
+
+ Map<String, String> oldProps = HoodieCLI.getTableMetaClient().getTableConfig().getProps();
+
+ // after overwrite, the stored value in .hoodie is equals to which read from properties.
+ Map<String, String> result = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()).getTableConfig().getProps();
+ Properties expectProps = new Properties();
+ expectProps.load(new FileInputStream(new File(newProps.getPath())));
+
+ Map<String, String> expected = expectProps.entrySet().stream()
+ .collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue())));
+ assertEquals(expected, result);
+
+ // check result
+ List<String> allPropsStr = Arrays.asList("hoodie.table.name", "hoodie.table.type",
+ "hoodie.archivelog.folder", "hoodie.timeline.layout.version");
+ String[][] rows = allPropsStr.stream().sorted().map(key -> new String[] {key,
+ oldProps.getOrDefault(key, null), result.getOrDefault(key, null)})
+ .toArray(String[][]::new);
+ String expect = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY,
+ HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows);
+
+ assertEquals(expect, cr.getResult().toString());
+ }
+
+ /**
+ * Test case for 'repair corrupted clean files'.
+ */
+ @Test
+ public void testRemoveCorruptedPendingCleanAction() throws IOException {
+ HoodieCLI.conf = jsc.hadoopConfiguration();
+
+ Configuration conf = HoodieCLI.conf;
+
+ metaClient = HoodieCLI.getTableMetaClient();
+
+ // Create four requested files
+ for (int i = 100; i < 104; i++) {
+ String timestamp = String.valueOf(i);
+ // Write corrupted requested Compaction
+ HoodieTestCommitMetadataGenerator.createCompactionRequestedFile(tablePath, timestamp, conf);
+ }
+
+ // reload meta client
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ // first, there are four instants
+ assertEquals(4, metaClient.getActiveTimeline().filterInflightsAndRequested().getInstants().count());
+
+ CommandResult cr = getShell().executeCommand("repair corrupted clean files");
+ assertTrue(cr.isSuccess());
+
+ // reload meta client
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ assertEquals(0, metaClient.getActiveTimeline().filterInflightsAndRequested().getInstants().count());
+ }
+}
diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
index 99bea63..0ec3bc8 100644
--- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
+++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
@@ -216,12 +216,23 @@ public class HoodieTestDataGenerator {
});
}
- public static void createCompactionRequestedFile(String basePath, String commitTime, Configuration configuration)
+ public static void createEmptyCleanRequestedFile(String basePath, String instantTime, Configuration configuration)
throws IOException {
Path commitFile = new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
- + HoodieTimeline.makeRequestedCompactionFileName(commitTime));
+ + HoodieTimeline.makeRequestedCleanerFileName(instantTime));
+ createEmptyFile(basePath, commitFile, configuration);
+ }
+
+ public static void createCompactionRequestedFile(String basePath, String instantTime, Configuration configuration)
+ throws IOException {
+ Path commitFile = new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ + HoodieTimeline.makeRequestedCompactionFileName(instantTime));
+ createEmptyFile(basePath, commitFile, configuration);
+ }
+
+ private static void createEmptyFile(String basePath, Path filePath, Configuration configuration) throws IOException {
FileSystem fs = FSUtils.getFs(basePath, configuration);
- FSDataOutputStream os = fs.create(commitFile, true);
+ FSDataOutputStream os = fs.create(filePath, true);
os.close();
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
new file mode 100644
index 0000000..15e8bea
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util;
+
+import org.apache.hudi.common.util.collection.Pair;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class CollectionUtils {
+ /**
+ * Determines whether two iterators contain equal elements in the same order. More specifically,
+ * this method returns {@code true} if {@code iterator1} and {@code iterator2} contain the same
+ * number of elements and every element of {@code iterator1} is equal to the corresponding element
+ * of {@code iterator2}.
+ *
+ * <p>Note that this will modify the supplied iterators, since they will have been advanced some
+ * number of elements forward.
+ */
+ public static boolean elementsEqual(Iterator<?> iterator1, Iterator<?> iterator2) {
+ while (iterator1.hasNext()) {
+ if (!iterator2.hasNext()) {
+ return false;
+ }
+ Object o1 = iterator1.next();
+ Object o2 = iterator2.next();
+ if (!Objects.equals(o1, o2)) {
+ return false;
+ }
+ }
+ return !iterator2.hasNext();
+ }
+
+ @SafeVarargs
+ public static <T> Set<T> createSet(final T... elements) {
+ return Stream.of(elements).collect(Collectors.toSet());
+ }
+
+ public static <K,V> Map<K, V> createImmutableMap(final K key, final V value) {
+ return Collections.unmodifiableMap(Collections.singletonMap(key, value));
+ }
+
+ @SafeVarargs
+ public static <T> List<T> createImmutableList(final T... elements) {
+ return Collections.unmodifiableList(Stream.of(elements).collect(Collectors.toList()));
+ }
+
+ public static <K,V> Map<K,V> createImmutableMap(final Map<K,V> map) {
+ return Collections.unmodifiableMap(map);
+ }
+
+ @SafeVarargs
+ public static <K,V> Map<K,V> createImmutableMap(final Pair<K,V>... elements) {
+ Map<K,V> map = new HashMap<>();
+ for (Pair<K,V> pair: elements) {
+ map.put(pair.getLeft(), pair.getRight());
+ }
+ return Collections.unmodifiableMap(map);
+ }
+
+ @SafeVarargs
+ public static <T> Set<T> createImmutableSet(final T... elements) {
+ return Collections.unmodifiableSet(createSet(elements));
+ }
+
+ public static <T> Set<T> createImmutableSet(final Set<T> set) {
+ return Collections.unmodifiableSet(set);
+ }
+
+ public static <T> List<T> createImmutableList(final List<T> list) {
+ return Collections.unmodifiableList(list);
+ }
+
+ private static Object[] checkElementsNotNull(Object... array) {
+ return checkElementsNotNull(array, array.length);
+ }
+
+ private static Object[] checkElementsNotNull(Object[] array, int length) {
+ for (int i = 0; i < length; i++) {
+ checkElementNotNull(array[i], i);
+ }
+ return array;
+ }
+
+ private static Object checkElementNotNull(Object element, int index) {
+ return Objects.requireNonNull(element, "Element is null at index " + index);
+ }
+}
\ No newline at end of file
[hudi] 24/40: [HUDI-852] adding check for table name for Append
Save mode (#1580)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 288bc4dd809521af896d0538ec2ae87bc034df7b
Author: AakashPradeep <em...@gmail.com>
AuthorDate: Sun May 3 23:09:17 2020 -0700
[HUDI-852] adding check for table name for Append Save mode (#1580)
* adding check for table name for Append Save mode
* adding existing table validation for delete and upsert operation
Co-authored-by: Aakash Pradeep <ap...@twilio.com>
---
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 7 +++
.../apache/hudi/HoodieSparkSqlWriterSuite.scala | 60 ++++++++++++++++++++--
2 files changed, 63 insertions(+), 4 deletions(-)
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 326595f..e5886eb 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -82,6 +82,13 @@ private[hudi] object HoodieSparkSqlWriter {
val fs = basePath.getFileSystem(sparkContext.hadoopConfiguration)
var exists = fs.exists(new Path(basePath, HoodieTableMetaClient.METAFOLDER_NAME))
+ if (exists && mode == SaveMode.Append) {
+ val existingTableName = new HoodieTableMetaClient(sparkContext.hadoopConfiguration, path.get).getTableConfig.getTableName
+ if (!existingTableName.equals(tblName.get)) {
+ throw new HoodieException(s"hoodie table with name $existingTableName already exist at $basePath")
+ }
+ }
+
val (writeStatuses, writeClient: HoodieWriteClient[HoodieRecordPayload[Nothing]]) =
if (!operation.equalsIgnoreCase(DELETE_OPERATION_OPT_VAL)) {
// register classes & schemas
diff --git a/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala b/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala
index 58ca984..bb82f8d 100644
--- a/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala
+++ b/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala
@@ -17,6 +17,9 @@
package org.apache.hudi
+import java.util.{Date, UUID}
+
+import org.apache.commons.io.FileUtils
import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.HoodieException
@@ -43,10 +46,59 @@ class HoodieSparkSqlWriterSuite extends FunSuite with Matchers {
test("throw hoodie exception when invalid serializer") {
val session = SparkSession.builder().appName("hoodie_test").master("local").getOrCreate()
- val sqlContext = session.sqlContext
- val options = Map("path" -> "hoodie/test/path", HoodieWriteConfig.TABLE_NAME -> "hoodie_test_tbl")
- val e = intercept[HoodieException](HoodieSparkSqlWriter.write(sqlContext, SaveMode.ErrorIfExists, options, session.emptyDataFrame))
- assert(e.getMessage.contains("spark.serializer"))
+ try {
+ val sqlContext = session.sqlContext
+ val options = Map("path" -> "hoodie/test/path", HoodieWriteConfig.TABLE_NAME -> "hoodie_test_tbl")
+ val e = intercept[HoodieException](HoodieSparkSqlWriter.write(sqlContext, SaveMode.ErrorIfExists, options, session.emptyDataFrame))
+ assert(e.getMessage.contains("spark.serializer"))
+ } finally {
+ session.stop()
+ }
+ }
+
+
+ test("throw hoodie exception when there already exist a table with different name with Append Save mode") {
+
+ val session = SparkSession.builder()
+ .appName("test_append_mode")
+ .master("local[2]")
+ .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+ .getOrCreate()
+ val path = java.nio.file.Files.createTempDirectory("hoodie_test_path")
+ try {
+
+ val sqlContext = session.sqlContext
+ val hoodieFooTableName = "hoodie_foo_tbl"
+
+ //create a new table
+ val fooTableModifier = Map("path" -> path.toAbsolutePath.toString,
+ HoodieWriteConfig.TABLE_NAME -> hoodieFooTableName,
+ "hoodie.insert.shuffle.parallelism" -> "4",
+ "hoodie.upsert.shuffle.parallelism" -> "4")
+ val fooTableParams = HoodieSparkSqlWriter.parametersWithWriteDefaults(fooTableModifier)
+ val dataFrame = session.createDataFrame(Seq(Test(UUID.randomUUID().toString, new Date().getTime)))
+ HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, fooTableParams, dataFrame)
+
+ //on same path try append with different("hoodie_bar_tbl") table name which should throw an exception
+ val barTableModifier = Map("path" -> path.toAbsolutePath.toString,
+ HoodieWriteConfig.TABLE_NAME -> "hoodie_bar_tbl",
+ "hoodie.insert.shuffle.parallelism" -> "4",
+ "hoodie.upsert.shuffle.parallelism" -> "4")
+ val barTableParams = HoodieSparkSqlWriter.parametersWithWriteDefaults(barTableModifier)
+ val dataFrame2 = session.createDataFrame(Seq(Test(UUID.randomUUID().toString, new Date().getTime)))
+ val tableAlreadyExistException = intercept[HoodieException](HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, barTableParams, dataFrame2))
+ assert(tableAlreadyExistException.getMessage.contains("hoodie table with name " + hoodieFooTableName + " already exist"))
+
+ //on same path try append with delete operation and different("hoodie_bar_tbl") table name which should throw an exception
+ val deleteTableParams = barTableParams ++ Map(OPERATION_OPT_KEY -> "delete")
+ val deleteCmdException = intercept[HoodieException](HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, deleteTableParams, dataFrame2))
+ assert(deleteCmdException.getMessage.contains("hoodie table with name " + hoodieFooTableName + " already exist"))
+ } finally {
+ session.stop()
+ FileUtils.deleteDirectory(path.toFile)
+ }
}
+ case class Test(uuid: String, ts: Long)
+
}
[hudi] 07/40: [HUDI-713] Fix conversion of Spark array of struct
type to Avro schema (#1406)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 18453a456db0f5be0b935e5a04d8c0cdb3958936
Author: wenningd <we...@gmail.com>
AuthorDate: Mon Mar 30 15:52:15 2020 -0700
[HUDI-713] Fix conversion of Spark array of struct type to Avro schema (#1406)
Co-authored-by: Wenning Ding <we...@amazon.com>
---
.../hudi/common/HoodieTestDataGenerator.java | 32 ++++++++++++++++++----
.../org/apache/hudi/AvroConversionHelper.scala | 8 +++---
.../org/apache/hudi/AvroConversionUtils.scala | 10 -------
.../resources/delta-streamer-config/source.avsc | 27 ++++++++++++++++++
.../sql-transformer.properties | 2 +-
.../resources/delta-streamer-config/target.avsc | 26 ++++++++++++++++++
6 files changed, 84 insertions(+), 21 deletions(-)
diff --git a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
index e0d2a53..99bea63 100644
--- a/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
+++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java
@@ -34,6 +34,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.conf.Configuration;
@@ -46,6 +47,7 @@ import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -74,17 +76,25 @@ public class HoodieTestDataGenerator {
public static final String[] DEFAULT_PARTITION_PATHS =
{DEFAULT_FIRST_PARTITION_PATH, DEFAULT_SECOND_PARTITION_PATH, DEFAULT_THIRD_PARTITION_PATH};
public static final int DEFAULT_PARTITION_DEPTH = 3;
- public static final String TRIP_EXAMPLE_SCHEMA = "{\"type\": \"record\"," + "\"name\": \"triprec\"," + "\"fields\": [ "
+ public static final String TRIP_SCHEMA_PREFIX = "{\"type\": \"record\"," + "\"name\": \"triprec\"," + "\"fields\": [ "
+ "{\"name\": \"timestamp\",\"type\": \"double\"}," + "{\"name\": \"_row_key\", \"type\": \"string\"},"
+ "{\"name\": \"rider\", \"type\": \"string\"}," + "{\"name\": \"driver\", \"type\": \"string\"},"
+ "{\"name\": \"begin_lat\", \"type\": \"double\"}," + "{\"name\": \"begin_lon\", \"type\": \"double\"},"
- + "{\"name\": \"end_lat\", \"type\": \"double\"}," + "{\"name\": \"end_lon\", \"type\": \"double\"},"
- + "{\"name\": \"fare\",\"type\": {\"type\":\"record\", \"name\":\"fare\",\"fields\": ["
- + "{\"name\": \"amount\",\"type\": \"double\"},{\"name\": \"currency\", \"type\": \"string\"}]}},"
- + "{\"name\": \"_hoodie_is_deleted\", \"type\": \"boolean\", \"default\": false} ]}";
+ + "{\"name\": \"end_lat\", \"type\": \"double\"}," + "{\"name\": \"end_lon\", \"type\": \"double\"},";
+ public static final String TRIP_SCHEMA_SUFFIX = "{\"name\": \"_hoodie_is_deleted\", \"type\": \"boolean\", \"default\": false} ]}";
+ public static final String FARE_NESTED_SCHEMA = "{\"name\": \"fare\",\"type\": {\"type\":\"record\", \"name\":\"fare\",\"fields\": ["
+ + "{\"name\": \"amount\",\"type\": \"double\"},{\"name\": \"currency\", \"type\": \"string\"}]}},";
+ public static final String TIP_NESTED_SCHEMA = "{\"name\": \"tip_history\", \"type\": {\"type\": \"array\", \"items\": {\"type\": \"record\", \"name\": \"tip_history\", \"fields\": ["
+ + "{\"name\": \"amount\", \"type\": \"double\"}, {\"name\": \"currency\", \"type\": \"string\"}]}}},";
+ public static final String MAP_TYPE_SCHEMA = "{\"name\": \"city_to_state\", \"type\": {\"type\": \"map\", \"values\": \"string\"}},";
+
+ public static final String TRIP_EXAMPLE_SCHEMA =
+ TRIP_SCHEMA_PREFIX + MAP_TYPE_SCHEMA + FARE_NESTED_SCHEMA + TIP_NESTED_SCHEMA + TRIP_SCHEMA_SUFFIX;
+
public static final String NULL_SCHEMA = Schema.create(Schema.Type.NULL).toString();
public static final String TRIP_HIVE_COLUMN_TYPES = "double,string,string,string,double,double,double,double,"
- + "struct<amount:double,currency:string>,boolean";
+ + "map<string,string>,struct<amount:double,currency:string>,array<struct<amount:double,currency:string>>,boolean";
+
public static final Schema AVRO_SCHEMA = new Schema.Parser().parse(TRIP_EXAMPLE_SCHEMA);
public static final Schema AVRO_SCHEMA_WITH_METADATA_FIELDS =
HoodieAvroUtils.addMetadataFields(AVRO_SCHEMA);
@@ -156,11 +166,21 @@ public class HoodieTestDataGenerator {
rec.put("end_lat", RAND.nextDouble());
rec.put("end_lon", RAND.nextDouble());
+ rec.put("city_to_state", Collections.singletonMap("LA", "CA"));
+
GenericRecord fareRecord = new GenericData.Record(AVRO_SCHEMA.getField("fare").schema());
fareRecord.put("amount", RAND.nextDouble() * 100);
fareRecord.put("currency", "USD");
rec.put("fare", fareRecord);
+ GenericArray<GenericRecord> tipHistoryArray = new GenericData.Array<>(1, AVRO_SCHEMA.getField("tip_history").schema());
+ Schema tipSchema = new Schema.Parser().parse(AVRO_SCHEMA.getField("tip_history").schema().toString()).getElementType();
+ GenericRecord tipRecord = new GenericData.Record(tipSchema);
+ tipRecord.put("amount", RAND.nextDouble() * 100);
+ tipRecord.put("currency", "USD");
+ tipHistoryArray.add(tipRecord);
+ rec.put("tip_history", tipHistoryArray);
+
if (isDeleteRecord) {
rec.put("_hoodie_is_deleted", true);
} else {
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala
index b61bef3..43225bc 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionHelper.scala
@@ -27,7 +27,6 @@ import org.apache.avro.{LogicalTypes, Schema}
import org.apache.avro.Schema.Type._
import org.apache.avro.generic.GenericData.{Fixed, Record}
import org.apache.avro.generic.{GenericData, GenericFixed, GenericRecord}
-import org.apache.hudi.AvroConversionUtils.getNewRecordNamespace
import org.apache.spark.sql.Row
import org.apache.spark.sql.avro.{IncompatibleSchemaException, SchemaConverters}
import org.apache.spark.sql.catalyst.expressions.GenericRow
@@ -303,7 +302,7 @@ object AvroConversionHelper {
avroSchema,
elementType,
structName,
- getNewRecordNamespace(elementType, recordNamespace, structName))
+ recordNamespace)
(item: Any) => {
if (item == null) {
null
@@ -324,7 +323,7 @@ object AvroConversionHelper {
avroSchema,
valueType,
structName,
- getNewRecordNamespace(valueType, recordNamespace, structName))
+ recordNamespace)
(item: Any) => {
if (item == null) {
null
@@ -338,12 +337,13 @@ object AvroConversionHelper {
}
case structType: StructType =>
val schema: Schema = SchemaConverters.toAvroType(structType, nullable = false, structName, recordNamespace)
+ val childNameSpace = if (recordNamespace != "") s"$recordNamespace.$structName" else structName
val fieldConverters = structType.fields.map(field =>
createConverterToAvro(
avroSchema,
field.dataType,
field.name,
- getNewRecordNamespace(field.dataType, recordNamespace, structName)))
+ childNameSpace))
(item: Any) => {
if (item == null) {
null
diff --git a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala
index decab9c..04de1c7 100644
--- a/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala
+++ b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala
@@ -71,16 +71,6 @@ object AvroConversionUtils {
}
}
- def getNewRecordNamespace(elementDataType: DataType,
- currentRecordNamespace: String,
- elementName: String): String = {
-
- elementDataType match {
- case StructType(_) => s"$currentRecordNamespace.$elementName"
- case _ => currentRecordNamespace
- }
- }
-
def convertStructTypeToAvroSchema(structType: StructType,
structName: String,
recordNamespace: String): Schema = {
diff --git a/hudi-utilities/src/test/resources/delta-streamer-config/source.avsc b/hudi-utilities/src/test/resources/delta-streamer-config/source.avsc
index a02d48d..8d01820 100644
--- a/hudi-utilities/src/test/resources/delta-streamer-config/source.avsc
+++ b/hudi-utilities/src/test/resources/delta-streamer-config/source.avsc
@@ -45,6 +45,13 @@
"type" : "double"
},
{
+ "name" :"city_to_state",
+ "type" : {
+ "type" : "map",
+ "values": "string"
+ }
+ },
+ {
"name" : "fare",
"type" : {
"type" : "record",
@@ -62,6 +69,26 @@
}
},
{
+ "name" : "tip_history",
+ "type" : {
+ "type" : "array",
+ "items" : {
+ "type" : "record",
+ "name" : "tip_history",
+ "fields" : [
+ {
+ "name" : "amount",
+ "type" : "double"
+ },
+ {
+ "name" : "currency",
+ "type" : "string"
+ }
+ ]
+ }
+ }
+ },
+ {
"name" : "_hoodie_is_deleted",
"type" : "boolean",
"default" : false
diff --git a/hudi-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties b/hudi-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties
index e8b2857..569b417 100644
--- a/hudi-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties
+++ b/hudi-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties
@@ -16,4 +16,4 @@
# limitations under the License.
###
include=base.properties
-hoodie.deltastreamer.transformer.sql=SELECT a.timestamp, a._row_key, a.rider, a.driver, a.begin_lat, a.begin_lon, a.end_lat, a.end_lon, a.fare, a.`_hoodie_is_deleted`, CAST(1.0 AS DOUBLE) AS haversine_distance FROM <SRC> a
+hoodie.deltastreamer.transformer.sql=SELECT a.timestamp, a._row_key, a.rider, a.driver, a.begin_lat, a.begin_lon, a.end_lat, a.end_lon, a.city_to_state, a.fare, a.tip_history, a.`_hoodie_is_deleted`, CAST(1.0 AS DOUBLE) AS haversine_distance FROM <SRC> a
diff --git a/hudi-utilities/src/test/resources/delta-streamer-config/target.avsc b/hudi-utilities/src/test/resources/delta-streamer-config/target.avsc
index b64fe4d..4fbb5c5 100644
--- a/hudi-utilities/src/test/resources/delta-streamer-config/target.avsc
+++ b/hudi-utilities/src/test/resources/delta-streamer-config/target.avsc
@@ -44,6 +44,12 @@
"name" : "end_lon",
"type" : "double"
}, {
+ "name" :"city_to_state",
+ "type" : {
+ "type" : "map",
+ "values": "string"
+ }
+ }, {
"name" : "fare",
"type" : {
"type" : "record",
@@ -61,6 +67,26 @@
}
},
{
+ "name" : "tip_history",
+ "type" : {
+ "type" : "array",
+ "items" : {
+ "type" : "record",
+ "name" : "tip_history",
+ "fields" : [
+ {
+ "name" : "amount",
+ "type" : "double"
+ },
+ {
+ "name" : "currency",
+ "type" : "string"
+ }
+ ]
+ }
+ }
+ },
+ {
"name" : "_hoodie_is_deleted",
"type" : "boolean",
"default" : false
[hudi] 17/40: [HUDI-400] Check upgrade from old plan to new plan
for compaction (#1422)
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.5.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 2e5f6011575cc9fb6adbae822718ea08ac00c4be
Author: Zhiyuan Zhao <49...@users.noreply.github.com>
AuthorDate: Fri Mar 20 15:13:17 2020 +0800
[HUDI-400] Check upgrade from old plan to new plan for compaction (#1422)
* Fix NPE when DataFile is null
* Check from old plan upgrade to new plan
---
.../hudi/common/versioning/compaction/CompactionV2MigrationHandler.java | 2 +-
.../src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java
index 26180e4..0c07b7c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java
@@ -55,7 +55,7 @@ public class CompactionV2MigrationHandler extends AbstractMigratorBase<HoodieCom
v2CompactionOperationList = input.getOperations().stream().map(inp ->
HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime())
.setFileId(inp.getFileId()).setPartitionPath(inp.getPartitionPath()).setMetrics(inp.getMetrics())
- .setDataFilePath(new Path(inp.getDataFilePath()).getName()).setDeltaFilePaths(
+ .setDataFilePath(inp.getDataFilePath() == null ? null : new Path(inp.getDataFilePath()).getName()).setDeltaFilePaths(
inp.getDeltaFilePaths().stream().map(s -> new Path(s).getName()).collect(Collectors.toList()))
.build()).collect(Collectors.toList());
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
index 2d3cf2e..034ac3c 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
@@ -84,6 +84,8 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
HoodieCompactionPlan newPlan = migrator.upgradeToLatest(plan, plan.getVersion());
Assert.assertEquals(LATEST_COMPACTION_METADATA_VERSION, newPlan.getVersion());
testFileSlicesCompactionPlanEquality(inputAndPlan.getKey(), newPlan);
+ HoodieCompactionPlan latestPlan = migrator.migrateToVersion(oldPlan, oldPlan.getVersion(), newPlan.getVersion());
+ testFileSlicesCompactionPlanEquality(inputAndPlan.getKey(), latestPlan);
}
@Test