You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by xu...@apache.org on 2022/04/19 10:30:08 UTC
[hudi] branch release-0.11.0 updated (fbc6595b34 -> c0cadbca65)
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a change to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from fbc6595b34 [HOTFIX] add missing license (#5322)
new d886e9eb8d [MINOR] Removing invalid code to close parquet reader iterator (#5182)
new 7bc940c7e4 [HUDI-3835] Add UT for delete in java client (#5270)
new 4a4407288a [MINOR] Fix typos in log4j-surefire.properties (#5212)
new 23b0dfc337 Fixing async clustering job test in TestHoodieDeltaStreamer (#5317)
new 06adf27051 [HUDI-3886] Adding default null for some of the fields in col stats in MDT schema (#5329)
new d979feee34 [HUDI-3707] Fix target schema handling in HoodieSparkUtils while creating RDD (#5347)
new bdc0f9336a [HUDI-3895] Fixing file-partitioning seq for base-file only views to make sure we bucket the files efficiently (#5337)
new 52cb67bf48 [HUDI-3894] Fix datahub to include HBase dependencies and shading (#5338)
new 54753a5a30 [HUDI-3903] Fix NoClassDefFoundError with Kafka Connect bundle (#5353)
new 46deea05c4 [HUDI-3899] Drop index to delete pending index instants from timeline if applicable (#5342)
new c0cadbca65 [HUDI-3894] Fix gcp bundle to include HBase dependencies and shading (#5349)
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../metadata/HoodieBackedTableMetadataWriter.java | 39 ++++-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/main/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 86 +++++++++-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 19 ++-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-common/src/main/avro/HoodieMetadata.avsc | 15 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 27 ++--
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 8 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 2 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 176 +++++++++++++++++++--
.../functional/TestHoodieDeltaStreamer.java | 30 ++--
...er.properties => indexer-only-bloom.properties} | 2 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 118 ++++++++++++++
packaging/hudi-gcp-bundle/pom.xml | 117 ++++++++++++++
packaging/hudi-kafka-connect-bundle/pom.xml | 2 +-
34 files changed, 619 insertions(+), 106 deletions(-)
copy hudi-utilities/src/test/resources/delta-streamer-config/{indexer.properties => indexer-only-bloom.properties} (93%)
[hudi] 04/11: Fixing async clustering job test in TestHoodieDeltaStreamer (#5317)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 23b0dfc337737e2d1e5b792f999c311df62f9498
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Mon Apr 18 08:08:33 2022 -0400
Fixing async clustering job test in TestHoodieDeltaStreamer (#5317)
---
.../functional/TestHoodieDeltaStreamer.java | 25 ++++++++++++++--------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
index 2db72cbd41..804676f0ff 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
@@ -104,6 +104,7 @@ import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
@@ -123,6 +124,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -380,7 +382,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
ret = false;
}
}
- return true;
+ return ret;
});
res.get(timeoutInSecs, TimeUnit.SECONDS);
}
@@ -1028,17 +1030,20 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
});
}
- @Disabled("HUDI-3710 to fix the ConcurrentModificationException")
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testHoodieAsyncClusteringJob(boolean shouldPassInClusteringInstantTime) throws Exception {
String tableBasePath = dfsBasePath + "/asyncClusteringJob";
-
- HoodieDeltaStreamer ds = initialHoodieDeltaStreamer(tableBasePath, 3000, "true");
+ HoodieDeltaStreamer ds = initialHoodieDeltaStreamer(tableBasePath, 3000, "false");
+ CountDownLatch countDownLatch = new CountDownLatch(1);
deltaStreamerTestRunner(ds, (r) -> {
TestHelpers.assertAtLeastNCommits(2, tableBasePath, dfs);
+ countDownLatch.countDown();
+ return true;
+ });
+ if (countDownLatch.await(2, TimeUnit.MINUTES)) {
Option<String> scheduleClusteringInstantTime = Option.empty();
try {
HoodieClusteringJob scheduleClusteringJob =
@@ -1046,7 +1051,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
scheduleClusteringInstantTime = scheduleClusteringJob.doSchedule();
} catch (Exception e) {
LOG.warn("Schedule clustering failed", e);
- return false;
+ Assertions.fail("Schedule clustering failed", e);
}
if (scheduleClusteringInstantTime.isPresent()) {
LOG.info("Schedule clustering success, now cluster with instant time " + scheduleClusteringInstantTime.get());
@@ -1054,13 +1059,15 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
shouldPassInClusteringInstantTime ? scheduleClusteringInstantTime.get() : null, false);
HoodieClusteringJob clusterClusteringJob = new HoodieClusteringJob(jsc, clusterClusteringConfig);
clusterClusteringJob.cluster(clusterClusteringConfig.retry);
+ TestHelpers.assertAtLeastNReplaceCommits(1, tableBasePath, dfs);
LOG.info("Cluster success");
} else {
- LOG.warn("Schedule clustering failed");
+ LOG.warn("Clustering execution failed");
+ Assertions.fail("Clustering execution failed");
}
- TestHelpers.assertAtLeastNReplaceCommits(2, tableBasePath, dfs);
- return true;
- });
+ } else {
+ Assertions.fail("Deltastreamer should have completed 2 commits.");
+ }
}
@Test
[hudi] 10/11: [HUDI-3899] Drop index to delete pending index instants from timeline if applicable (#5342)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 46deea05c402e986395875b35f8bdab6cd8bbda5
Author: Sagar Sumit <sa...@gmail.com>
AuthorDate: Tue Apr 19 07:58:46 2022 +0530
[HUDI-3899] Drop index to delete pending index instants from timeline if applicable (#5342)
Co-authored-by: sivabalan <n....@gmail.com>
---
.../metadata/HoodieBackedTableMetadataWriter.java | 39 ++++-
.../org/apache/hudi/utilities/HoodieIndexer.java | 2 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 176 +++++++++++++++++++--
.../indexer-only-bloom.properties | 25 +++
4 files changed, 225 insertions(+), 17 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index be1aed75e2..d080d14a69 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -18,13 +18,9 @@
package org.apache.hudi.metadata;
-import org.apache.avro.specific.SpecificRecordBase;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieIndexPartitionInfo;
+import org.apache.hudi.avro.model.HoodieIndexPlan;
import org.apache.hudi.avro.model.HoodieInstantInfo;
import org.apache.hudi.avro.model.HoodieMetadataRecord;
import org.apache.hudi.avro.model.HoodieRestoreMetadata;
@@ -70,6 +66,12 @@ import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.exception.HoodieMetadataException;
+
+import org.apache.avro.specific.SpecificRecordBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -88,6 +90,9 @@ import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.getIndexInflightInstant;
+import static org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeIndexPlan;
import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING;
import static org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX;
import static org.apache.hudi.metadata.HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP;
@@ -727,9 +732,33 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
HoodieTableConfig.update(dataMetaClient.getFs(), new Path(dataMetaClient.getMetaPath()), dataMetaClient.getTableConfig().getProps());
LOG.warn("Deleting Metadata Table partitions: " + partitionPath);
dataMetaClient.getFs().delete(new Path(metadataWriteConfig.getBasePath(), partitionPath), true);
+ // delete corresponding pending indexing instant file in the timeline
+ LOG.warn("Deleting pending indexing instant from the timeline for partition: " + partitionPath);
+ deletePendingIndexingInstant(dataMetaClient, partitionPath);
}
}
+ /**
+ * Deletes any pending indexing instant, if it exists.
+ * It reads the plan from indexing.requested file and deletes both requested and inflight instants,
+ * if the partition path in the plan matches with the given partition path.
+ */
+ private static void deletePendingIndexingInstant(HoodieTableMetaClient metaClient, String partitionPath) {
+ metaClient.reloadActiveTimeline().filterPendingIndexTimeline().getInstants().filter(instant -> REQUESTED.equals(instant.getState()))
+ .forEach(instant -> {
+ try {
+ HoodieIndexPlan indexPlan = deserializeIndexPlan(metaClient.getActiveTimeline().readIndexPlanAsBytes(instant).get());
+ if (indexPlan.getIndexPartitionInfos().stream()
+ .anyMatch(indexPartitionInfo -> indexPartitionInfo.getMetadataPartitionPath().equals(partitionPath))) {
+ metaClient.getActiveTimeline().deleteInstantFileIfExists(instant);
+ metaClient.getActiveTimeline().deleteInstantFileIfExists(getIndexInflightInstant(instant.getTimestamp()));
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to delete the instant file corresponding to " + instant);
+ }
+ });
+ }
+
private MetadataRecordsGenerationParams getRecordsGenerationParams() {
return new MetadataRecordsGenerationParams(
dataMetaClient,
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
index 501f9296b4..96f6ce38cd 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
@@ -85,7 +85,7 @@ import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE_AND_EXECUTE;
public class HoodieIndexer {
private static final Logger LOG = LogManager.getLogger(HoodieIndexer.class);
- private static final String DROP_INDEX = "dropindex";
+ static final String DROP_INDEX = "dropindex";
private final HoodieIndexer.Config cfg;
private TypedProperties props;
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
index 695ca88413..9312a26b4f 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
@@ -28,14 +28,17 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.metadata.HoodieTableMetadataUtil;
import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.testutils.providers.SparkProvider;
+import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@@ -50,10 +53,16 @@ import java.util.List;
import java.util.Objects;
import static org.apache.hudi.common.table.HoodieTableMetaClient.reload;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
+import static org.apache.hudi.metadata.HoodieTableMetadataUtil.getCompletedMetadataPartitions;
+import static org.apache.hudi.metadata.HoodieTableMetadataUtil.metadataPartitionExists;
import static org.apache.hudi.metadata.MetadataPartitionType.BLOOM_FILTERS;
import static org.apache.hudi.metadata.MetadataPartitionType.COLUMN_STATS;
import static org.apache.hudi.metadata.MetadataPartitionType.FILES;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.apache.hudi.utilities.HoodieIndexer.DROP_INDEX;
+import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE;
+import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE_AND_EXECUTE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -81,6 +90,14 @@ public class TestHoodieIndexer extends HoodieCommonTestHarness implements SparkP
initMetaClient();
}
+ protected void initMetaClient() throws IOException {
+ String rootPathStr = "file://" + tempDir.toAbsolutePath().toString();
+ Path rootPath = new Path(rootPathStr);
+ rootPath.getFileSystem(jsc.hadoopConfiguration()).mkdirs(rootPath);
+ metaClient = HoodieTestUtils.init(rootPathStr, getTableType());
+ basePath = metaClient.getBasePath();
+ }
+
@Test
public void testGetRequestedPartitionTypes() {
HoodieIndexer.Config config = new HoodieIndexer.Config();
@@ -132,29 +149,166 @@ public class TestHoodieIndexer extends HoodieCommonTestHarness implements SparkP
assertNoWriteErrors(statuses);
// validate table config
- assertTrue(HoodieTableMetadataUtil.getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
- assertTrue(HoodieTableMetadataUtil.getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
// build indexer config which has only column_stats enabled (files is enabled by default)
HoodieIndexer.Config config = new HoodieIndexer.Config();
String propsPath = Objects.requireNonNull(getClass().getClassLoader().getResource("delta-streamer-config/indexer.properties")).getPath();
config.basePath = basePath;
config.tableName = tableName;
- config.indexTypes = "COLUMN_STATS";
- config.runningMode = "scheduleAndExecute";
+ config.indexTypes = COLUMN_STATS.name();
+ config.runningMode = SCHEDULE_AND_EXECUTE;
config.propsFilePath = propsPath;
// start the indexer and validate column_stats index is also complete
HoodieIndexer indexer = new HoodieIndexer(jsc, config);
assertEquals(0, indexer.start(0));
// validate table config
- assertTrue(HoodieTableMetadataUtil.getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
- assertTrue(HoodieTableMetadataUtil.getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
- assertTrue(HoodieTableMetadataUtil.getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(COLUMN_STATS.getPartitionPath()));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(COLUMN_STATS.getPartitionPath()));
// validate metadata partitions actually exist
- assertTrue(HoodieTableMetadataUtil.metadataPartitionExists(basePath, context, FILES));
- assertTrue(HoodieTableMetadataUtil.metadataPartitionExists(basePath, context, COLUMN_STATS));
- assertTrue(HoodieTableMetadataUtil.metadataPartitionExists(basePath, context, BLOOM_FILTERS));
+ assertTrue(metadataPartitionExists(basePath, context, FILES));
+ assertTrue(metadataPartitionExists(basePath, context, COLUMN_STATS));
+ assertTrue(metadataPartitionExists(basePath, context, BLOOM_FILTERS));
+ }
+
+ @Test
+ public void testIndexerDropPartitionDeletesInstantFromTimeline() {
+ initTestDataGenerator();
+ String tableName = "indexer_test";
+ HoodieWriteConfig.Builder writeConfigBuilder = getWriteConfigBuilder(basePath, tableName);
+ // enable files on the regular write client
+ HoodieMetadataConfig.Builder metadataConfigBuilder = getMetadataConfigBuilder(true, false).withMetadataIndexBloomFilter(true);
+ HoodieWriteConfig writeConfig = writeConfigBuilder.withMetadataConfig(metadataConfigBuilder.build()).build();
+ // do one upsert with synchronous metadata update
+ SparkRDDWriteClient writeClient = new SparkRDDWriteClient(context, writeConfig);
+ String instant = "0001";
+ writeClient.startCommitWithTime(instant);
+ List<HoodieRecord> records = dataGen.generateInserts(instant, 100);
+ JavaRDD<WriteStatus> result = writeClient.upsert(jsc.parallelize(records, 1), instant);
+ List<WriteStatus> statuses = result.collect();
+ assertNoWriteErrors(statuses);
+
+ // validate partitions built successfully
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, FILES));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, BLOOM_FILTERS));
+
+ // build indexer config which has only column_stats enabled (files is enabled by default)
+ HoodieIndexer.Config config = new HoodieIndexer.Config();
+ String propsPath = Objects.requireNonNull(getClass().getClassLoader().getResource("delta-streamer-config/indexer.properties")).getPath();
+ config.basePath = basePath;
+ config.tableName = tableName;
+ config.indexTypes = COLUMN_STATS.name();
+ config.runningMode = SCHEDULE;
+ config.propsFilePath = propsPath;
+
+ // schedule indexing and validate column_stats index is also initialized
+ HoodieIndexer indexer = new HoodieIndexer(jsc, config);
+ assertEquals(0, indexer.start(0));
+ Option<HoodieInstant> indexInstantInTimeline = metaClient.reloadActiveTimeline().filterPendingIndexTimeline().lastInstant();
+ assertTrue(indexInstantInTimeline.isPresent());
+ assertEquals(REQUESTED, indexInstantInTimeline.get().getState());
+ assertTrue(metadataPartitionExists(basePath, context, COLUMN_STATS));
+
+ // drop column_stats and validate indexing.requested is also removed from the timeline
+ config.runningMode = DROP_INDEX;
+ indexer = new HoodieIndexer(jsc, config);
+ assertEquals(0, indexer.start(0));
+ indexInstantInTimeline = metaClient.reloadActiveTimeline().filterPendingIndexTimeline().lastInstant();
+ assertFalse(indexInstantInTimeline.isPresent());
+ assertFalse(metadataPartitionExists(basePath, context, COLUMN_STATS));
+
+ // check other partitions are intact
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, FILES));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, BLOOM_FILTERS));
+ }
+
+ @Test
+ public void testTwoIndexersOneCreateOneDropPartition() {
+ initTestDataGenerator();
+ String tableName = "indexer_test";
+ HoodieWriteConfig.Builder writeConfigBuilder = getWriteConfigBuilder(basePath, tableName);
+ // enable files on the regular write client
+ HoodieMetadataConfig.Builder metadataConfigBuilder = getMetadataConfigBuilder(true, false);
+ HoodieWriteConfig writeConfig = writeConfigBuilder.withMetadataConfig(metadataConfigBuilder.build()).build();
+ // do one upsert with synchronous metadata update
+ SparkRDDWriteClient writeClient = new SparkRDDWriteClient(context, writeConfig);
+ String instant = "0001";
+ writeClient.startCommitWithTime(instant);
+ List<HoodieRecord> records = dataGen.generateInserts(instant, 100);
+ JavaRDD<WriteStatus> result = writeClient.upsert(jsc.parallelize(records, 1), instant);
+ List<WriteStatus> statuses = result.collect();
+ assertNoWriteErrors(statuses);
+
+ // validate files partition built successfully
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, FILES));
+
+ // build indexer config which has only bloom_filters enabled
+ HoodieIndexer.Config config = getHoodieIndexConfig(BLOOM_FILTERS.name(), SCHEDULE_AND_EXECUTE, "delta-streamer-config/indexer-only-bloom.properties");
+ // start the indexer and validate bloom_filters index is also complete
+ HoodieIndexer indexer = new HoodieIndexer(jsc, config);
+ assertEquals(0, indexer.start(0));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, BLOOM_FILTERS));
+
+ // completed index timeline for later validation
+ Option<HoodieInstant> bloomIndexInstant = metaClient.reloadActiveTimeline().filterCompletedIndexTimeline().lastInstant();
+ assertTrue(bloomIndexInstant.isPresent());
+
+ // build indexer config which has only column_stats enabled
+ config = getHoodieIndexConfig(COLUMN_STATS.name(), SCHEDULE, "delta-streamer-config/indexer.properties");
+
+ // schedule indexing and validate column_stats index is also initialized
+ // and indexing.requested instant is present
+ indexer = new HoodieIndexer(jsc, config);
+ assertEquals(0, indexer.start(0));
+ Option<HoodieInstant> columnStatsIndexInstant = metaClient.reloadActiveTimeline().filterPendingIndexTimeline().lastInstant();
+ assertTrue(columnStatsIndexInstant.isPresent());
+ assertEquals(REQUESTED, columnStatsIndexInstant.get().getState());
+ assertTrue(metadataPartitionExists(basePath, context, COLUMN_STATS));
+
+ // drop column_stats and validate indexing.requested is also removed from the timeline
+ // and completed indexing instant corresponding to bloom_filters index is still present
+ dropIndexAndAssert(COLUMN_STATS, "delta-streamer-config/indexer.properties", Option.empty());
+
+ // check other partitions are intact
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(FILES.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, FILES));
+ assertTrue(getCompletedMetadataPartitions(reload(metaClient).getTableConfig()).contains(BLOOM_FILTERS.getPartitionPath()));
+ assertTrue(metadataPartitionExists(basePath, context, BLOOM_FILTERS));
+
+ // drop bloom filter partition. timeline files should not be deleted since the index building is complete.
+ dropIndexAndAssert(BLOOM_FILTERS, "delta-streamer-config/indexer-only-bloom.properties", bloomIndexInstant);
+ }
+
+ private void dropIndexAndAssert(MetadataPartitionType indexType, String resourceFilePath, Option<HoodieInstant> completedIndexInstant) {
+ HoodieIndexer.Config config = getHoodieIndexConfig(indexType.name(), DROP_INDEX, resourceFilePath);
+ HoodieIndexer indexer = new HoodieIndexer(jsc, config);
+ assertEquals(0, indexer.start(0));
+ Option<HoodieInstant> pendingFlights = metaClient.reloadActiveTimeline().filterPendingIndexTimeline().lastInstant();
+ assertFalse(pendingFlights.isPresent());
+ assertFalse(metadataPartitionExists(basePath, context, indexType));
+ if (completedIndexInstant.isPresent()) {
+ assertEquals(completedIndexInstant, metaClient.reloadActiveTimeline().filterCompletedIndexTimeline().lastInstant());
+ }
+ }
+
+ private HoodieIndexer.Config getHoodieIndexConfig(String indexType, String runMode, String resourceFilePath) {
+ HoodieIndexer.Config config = new HoodieIndexer.Config();
+ String propsPath = Objects.requireNonNull(getClass().getClassLoader().getResource(resourceFilePath)).getPath();
+ config.basePath = basePath;
+ config.tableName = tableName;
+ config.indexTypes = indexType;
+ config.runningMode = runMode;
+ config.propsFilePath = propsPath;
+ return config;
}
private static HoodieWriteConfig.Builder getWriteConfigBuilder(String basePath, String tableName) {
diff --git a/hudi-utilities/src/test/resources/delta-streamer-config/indexer-only-bloom.properties b/hudi-utilities/src/test/resources/delta-streamer-config/indexer-only-bloom.properties
new file mode 100644
index 0000000000..6035077437
--- /dev/null
+++ b/hudi-utilities/src/test/resources/delta-streamer-config/indexer-only-bloom.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+hoodie.metadata.enable=true
+hoodie.metadata.index.async=true
+hoodie.metadata.index.bloom.filter.enable=true
+hoodie.metadata.index.check.timeout.seconds=60
+hoodie.write.concurrency.mode=optimistic_concurrency_control
+hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.InProcessLockProvider
\ No newline at end of file
[hudi] 09/11: [HUDI-3903] Fix NoClassDefFoundError with Kafka Connect bundle (#5353)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 54753a5a3089bffb9922f1182ee59bd6b68db632
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Mon Apr 18 18:17:53 2022 -0700
[HUDI-3903] Fix NoClassDefFoundError with Kafka Connect bundle (#5353)
---
packaging/hudi-kafka-connect-bundle/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packaging/hudi-kafka-connect-bundle/pom.xml b/packaging/hudi-kafka-connect-bundle/pom.xml
index 0e568a58e6..4377fe6b90 100644
--- a/packaging/hudi-kafka-connect-bundle/pom.xml
+++ b/packaging/hudi-kafka-connect-bundle/pom.xml
@@ -85,7 +85,7 @@
<include>org.apache.hudi:hudi-aws</include>
<!-- NOTE: This is temp (SchemaProvide dep) until PR3162 lands -->
- <include>org.apache.hudi:hudi-flink_${scala.binary.version}</include>
+ <include>org.apache.hudi:hudi-flink</include>
<include>org.apache.hudi:flink-core</include>
<include>org.apache.hudi:hudi-flink-client</include>
<include>org.apache.flink:flink-core</include>
[hudi] 07/11: [HUDI-3895] Fixing file-partitioning seq for base-file only views to make sure we bucket the files efficiently (#5337)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit bdc0f9336a1fcd86d8fd216124ae4f7f7a6ea64b
Author: Alexey Kudinkin <al...@infinilake.com>
AuthorDate: Mon Apr 18 13:06:52 2022 -0700
[HUDI-3895] Fixing file-partitioning seq for base-file only views to make sure we bucket the files efficiently (#5337)
---
.../org/apache/hudi/BaseFileOnlyRelation.scala | 27 ++++++++++++----------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
index 525292da6d..f46b31b036 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
@@ -84,21 +84,24 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
protected def collectFileSplits(partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): Seq[HoodieBaseFileSplit] = {
val partitions = listLatestBaseFiles(globPaths, partitionFilters, dataFilters)
- val fileSplits = partitions.values.toSeq.flatMap { files =>
- files.flatMap { file =>
- // TODO move to adapter
- // TODO fix, currently assuming parquet as underlying format
- HoodieDataSourceHelper.splitFiles(
- sparkSession = sparkSession,
- file = file,
- // TODO clarify why this is required
- partitionValues = getPartitionColumnsAsInternalRow(file)
- )
+ val fileSplits = partitions.values.toSeq
+ .flatMap { files =>
+ files.flatMap { file =>
+ // TODO fix, currently assuming parquet as underlying format
+ HoodieDataSourceHelper.splitFiles(
+ sparkSession = sparkSession,
+ file = file,
+ partitionValues = getPartitionColumnsAsInternalRow(file)
+ )
+ }
}
- }
+ // NOTE: It's important to order the splits in the reverse order of their
+ // size so that we can subsequently bucket them in an efficient manner
+ .sortBy(_.length)(implicitly[Ordering[Long]].reverse)
val maxSplitBytes = sparkSession.sessionState.conf.filesMaxPartitionBytes
- sparkAdapter.getFilePartitions(sparkSession, fileSplits, maxSplitBytes).map(HoodieBaseFileSplit.apply)
+ sparkAdapter.getFilePartitions(sparkSession, fileSplits, maxSplitBytes)
+ .map(HoodieBaseFileSplit.apply)
}
}
[hudi] 02/11: [HUDI-3835] Add UT for delete in java client (#5270)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 7bc940c7e4fb26dfa850689161dec754005b406b
Author: 董可伦 <do...@inspur.com>
AuthorDate: Sat Apr 16 03:03:48 2022 +0800
[HUDI-3835] Add UT for delete in java client (#5270)
---
.../commit/TestJavaCopyOnWriteActionExecutor.java | 86 +++++++++++++++++++++-
1 file changed, 85 insertions(+), 1 deletion(-)
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
index 1bf1b4cccb..518414d614 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
@@ -318,7 +318,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestBase
}
@Test
- public void testInsertRecords() throws Exception {
+ public void testInsertRecords() throws Exception {
HoodieWriteConfig config = makeHoodieClientConfig();
String instantTime = makeNewCommitTime();
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -465,6 +465,90 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestBase
verifyStatusResult(returnedStatuses, generateExpectedPartitionNumRecords(inputRecords));
}
+ @Test
+ public void testDeleteRecords() throws Exception {
+ // Prepare the AvroParquetIO
+ HoodieWriteConfig config = makeHoodieClientConfig();
+ int startInstant = 1;
+ String firstCommitTime = makeNewCommitTime(startInstant++, "%09d");
+ HoodieJavaWriteClient writeClient = getHoodieWriteClient(config);
+ writeClient.startCommitWithTime(firstCommitTime);
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ BaseFileUtils fileUtils = BaseFileUtils.getInstance(metaClient);
+
+ String partitionPath = "2022/04/09";
+
+ // Get some records belong to the same partition (2016/01/31)
+ String recordStr1 = "{\"_row_key\":\"8eb5b87a-1feh-4edd-87b4-6ec96dc405a0\","
+ + "\"time\":\"2022-04-09T03:16:41.415Z\",\"number\":1}";
+ String recordStr2 = "{\"_row_key\":\"8eb5b87b-1feu-4edd-87b4-6ec96dc405a0\","
+ + "\"time\":\"2022-04-09T03:20:41.415Z\",\"number\":2}";
+ String recordStr3 = "{\"_row_key\":\"8eb5b87c-1fej-4edd-87b4-6ec96dc405a0\","
+ + "\"time\":\"2022-04-09T03:16:41.415Z\",\"number\":3}";
+
+ List<HoodieRecord> records = new ArrayList<>();
+ RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1);
+ records.add(new HoodieAvroRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1));
+ RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2);
+ records.add(new HoodieAvroRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2));
+ RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3);
+ records.add(new HoodieAvroRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3));
+
+ // Insert new records
+ writeClient.insert(records, firstCommitTime);
+
+ FileStatus[] allFiles = getIncrementalFiles(partitionPath, "0", -1);
+ assertEquals(1, allFiles.length);
+
+ // Read out the bloom filter and make sure filter can answer record exist or not
+ Path filePath = allFiles[0].getPath();
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, filePath);
+ for (HoodieRecord record : records) {
+ assertTrue(filter.mightContain(record.getRecordKey()));
+ }
+
+ // Read the base file, check the record content
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ int index = 0;
+ for (GenericRecord record : fileRecords) {
+ assertEquals(records.get(index).getRecordKey(), record.get("_row_key").toString());
+ index++;
+ }
+
+ String newCommitTime = makeNewCommitTime(startInstant++, "%09d");
+ writeClient.startCommitWithTime(newCommitTime);
+
+ // Test delete two records
+ List<HoodieKey> keysForDelete = new ArrayList(Arrays.asList(records.get(0).getKey(), records.get(2).getKey()));
+ writeClient.delete(keysForDelete, newCommitTime);
+
+ allFiles = getIncrementalFiles(partitionPath, "0", -1);
+ assertEquals(1, allFiles.length);
+
+ filePath = allFiles[0].getPath();
+ // Read the base file, check the record content
+ fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ // Check that the two records are deleted successfully
+ assertEquals(1, fileRecords.size());
+ assertEquals(records.get(1).getRecordKey(), fileRecords.get(0).get("_row_key").toString());
+
+ newCommitTime = makeNewCommitTime(startInstant++, "%09d");
+ writeClient.startCommitWithTime(newCommitTime);
+
+ // Test delete last record
+ keysForDelete = new ArrayList(Arrays.asList(records.get(1).getKey()));
+ writeClient.delete(keysForDelete, newCommitTime);
+
+ allFiles = getIncrementalFiles(partitionPath, "0", -1);
+ assertEquals(1, allFiles.length);
+
+ filePath = allFiles[0].getPath();
+ // Read the base file, check the record content
+ fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ // Check whether all records have been deleted
+ assertEquals(0, fileRecords.size());
+ }
+
public static Map<String, Long> generateExpectedPartitionNumRecords(List<HoodieRecord> records) {
return records.stream().map(record -> Pair.of(record.getPartitionPath(), 1))
.collect(Collectors.groupingBy(Pair::getLeft, Collectors.counting()));
[hudi] 08/11: [HUDI-3894] Fix datahub to include HBase dependencies and shading (#5338)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 52cb67bf48411b4c8e7da8df12ed00b72d4e186c
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Mon Apr 18 16:20:50 2022 -0700
[HUDI-3894] Fix datahub to include HBase dependencies and shading (#5338)
Co-authored-by: Raymond Xu <27...@users.noreply.github.com>
---
packaging/hudi-datahub-sync-bundle/pom.xml | 118 +++++++++++++++++++++++++++++
1 file changed, 118 insertions(+)
diff --git a/packaging/hudi-datahub-sync-bundle/pom.xml b/packaging/hudi-datahub-sync-bundle/pom.xml
index f5f1fa401c..ac4602bd2a 100644
--- a/packaging/hudi-datahub-sync-bundle/pom.xml
+++ b/packaging/hudi-datahub-sync-bundle/pom.xml
@@ -63,6 +63,7 @@
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<artifactSet>
<includes>
@@ -70,9 +71,23 @@
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.hudi:hudi-sync-common</include>
<include>org.apache.hudi:hudi-datahub-sync</include>
+ <include>org.apache.parquet:parquet-avro</include>
<include>io.acryl:datahub-client</include>
<include>com.beust:jcommander</include>
+ <include>commons-io:commons-io</include>
+ <include>org.apache.hbase:hbase-common</include>
+ <include>org.apache.hbase:hbase-client</include>
+ <include>org.apache.hbase:hbase-hadoop-compat</include>
+ <include>org.apache.hbase:hbase-hadoop2-compat</include>
+ <include>org.apache.hbase:hbase-metrics</include>
+ <include>org.apache.hbase:hbase-metrics-api</include>
+ <include>org.apache.hbase:hbase-protocol-shaded</include>
+ <include>org.apache.hbase:hbase-server</include>
+ <include>org.apache.hbase.thirdparty:hbase-shaded-miscellaneous</include>
+ <include>org.apache.hbase.thirdparty:hbase-shaded-netty</include>
+ <include>org.apache.hbase.thirdparty:hbase-shaded-protobuf</include>
+ <include>org.apache.htrace:htrace-core4</include>
<include>org.apache.httpcomponents:fluent-hc</include>
<include>org.apache.httpcomponents:httpcore</include>
<include>org.apache.httpcomponents:httpclient</include>
@@ -80,6 +95,107 @@
<include>org.apache.httpcomponents:httpcore-nio</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.esotericsoftware.kryo.</pattern>
+ <shadedPattern>org.apache.hudi.com.esotericsoftware.kryo.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.esotericsoftware.minlog.</pattern>
+ <shadedPattern>org.apache.hudi.com.esotericsoftware.minlog.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.io.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.commons.io.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.hbase.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.hbase.</shadedPattern>
+ <excludes>
+ <exclude>org.apache.hadoop.hbase.KeyValue$KeyComparator</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hbase.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hbase.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.htrace.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.htrace.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.objenesis.</pattern>
+ <shadedPattern>org.apache.hudi.org.objenesis.</shadedPattern>
+ </relocation>
+ <!-- The classes below in org.apache.hadoop.metrics2 package come from
+ hbase-hadoop-compat and hbase-hadoop2-compat, which have to be shaded one by one,
+ instead of shading all classes under org.apache.hadoop.metrics2 including ones
+ from hadoop. -->
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.MetricHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.MetricHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.MetricsExecutor</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.MetricsExecutor
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.impl.JmxCacheBuster</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.impl.JmxCacheBuster</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MetricsExecutorImpl</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MetricsExecutorImpl
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableFastCounter</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableFastCounter
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableRangeHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableRangeHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableSizeHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableSizeHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableTimeHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableTimeHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.util.MetricQuantile</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.util.MetricQuantile
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.util.MetricSampleQuantiles</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.util.MetricSampleQuantiles
+ </shadedPattern>
+ </relocation>
+ </relocations>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
@@ -89,6 +205,8 @@
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/services/javax.*</exclude>
+ <exclude>**/*.proto</exclude>
+ <exclude>hbase-webapps/**</exclude>
</excludes>
</filter>
</filters>
[hudi] 05/11: [HUDI-3886] Adding default null for some of the fields in col stats in MDT schema (#5329)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 06adf27051da79fafe132acff004a365692a760a
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Mon Apr 18 10:37:03 2022 -0400
[HUDI-3886] Adding default null for some of the fields in col stats in MDT schema (#5329)
---
hudi-common/src/main/avro/HoodieMetadata.avsc | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/hudi-common/src/main/avro/HoodieMetadata.avsc b/hudi-common/src/main/avro/HoodieMetadata.avsc
index a8d7ca72bd..a47cbf3784 100644
--- a/hudi-common/src/main/avro/HoodieMetadata.avsc
+++ b/hudi-common/src/main/avro/HoodieMetadata.avsc
@@ -107,7 +107,8 @@
"type": [
"null",
"string"
- ]
+ ],
+ "default" : null
},
{
"doc": "Column name for which this column statistics applies",
@@ -318,7 +319,8 @@
"type": [
"null",
"long"
- ]
+ ],
+ "default": null
},
{
"doc": "Total count of null values",
@@ -326,7 +328,8 @@
"type": [
"null",
"long"
- ]
+ ],
+ "default": null
},
{
"doc": "Total storage size on disk",
@@ -334,7 +337,8 @@
"type": [
"null",
"long"
- ]
+ ],
+ "default": null
},
{
"doc": "Total uncompressed storage size on disk",
@@ -342,7 +346,8 @@
"type": [
"null",
"long"
- ]
+ ],
+ "default": null
},
{
"doc": "Column range entry valid/deleted flag",
[hudi] 06/11: [HUDI-3707] Fix target schema handling in HoodieSparkUtils while creating RDD (#5347)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d979feee3483debe7c8aa0f3d25ebed18e41bcfe
Author: Sagar Sumit <sa...@gmail.com>
AuthorDate: Mon Apr 18 23:04:04 2022 +0530
[HUDI-3707] Fix target schema handling in HoodieSparkUtils while creating RDD (#5347)
---
.../main/scala/org/apache/hudi/HoodieSparkUtils.scala | 19 +++++++++++++------
.../utilities/functional/TestHoodieDeltaStreamer.java | 5 -----
2 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
index 57eb32fce3..54bc06bd76 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
@@ -26,6 +26,9 @@ import org.apache.hudi.client.utils.SparkRowSerDe
import org.apache.hudi.common.config.TypedProperties
import org.apache.hudi.common.model.HoodieRecord
import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.internal.schema.InternalSchema
+import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter
+import org.apache.hudi.internal.schema.utils.InternalSchemaUtils
import org.apache.hudi.keygen.constant.KeyGeneratorOptions
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory
import org.apache.hudi.keygen.{BaseKeyGenerator, CustomAvroKeyGenerator, CustomKeyGenerator, KeyGenerator}
@@ -36,12 +39,8 @@ import org.apache.spark.sql.catalyst.encoders.RowEncoder
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, Literal}
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{StringType, StructField, StructType}
-import java.util.Properties
-
-import org.apache.hudi.internal.schema.InternalSchema
-import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter
-import org.apache.hudi.internal.schema.utils.InternalSchemaUtils
+import java.util.Properties
import scala.collection.JavaConverters._
object HoodieSparkUtils extends SparkAdapterSupport {
@@ -130,7 +129,15 @@ object HoodieSparkUtils extends SparkAdapterSupport {
*/
def createRdd(df: DataFrame, structName: String, recordNamespace: String, reconcileToLatestSchema: Boolean,
latestTableSchema: org.apache.hudi.common.util.Option[Schema] = org.apache.hudi.common.util.Option.empty()): RDD[GenericRecord] = {
- val latestTableSchemaConverted = if (latestTableSchema.isPresent && reconcileToLatestSchema) Some(latestTableSchema.get()) else None
+ var latestTableSchemaConverted : Option[Schema] = None
+
+ if (latestTableSchema.isPresent && reconcileToLatestSchema) {
+ latestTableSchemaConverted = Some(latestTableSchema.get())
+ } else {
+ // cases when users want to use latestTableSchema but have not turned on reconcileToLatestSchema explicitly
+ // for example, when using a Transformer implementation to transform source RDD to target RDD
+ latestTableSchemaConverted = if (latestTableSchema.isPresent) Some(latestTableSchema.get()) else None
+ }
createRdd(df, structName, recordNamespace, latestTableSchemaConverted)
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
index 804676f0ff..0576f6aaee 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
@@ -1690,7 +1690,6 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
testParquetDFSSource(false, null, true);
}
- @Disabled("HUDI-3707 To investigate problem with schema provider and transformer")
@Test
public void testParquetDFSSourceWithoutSchemaProviderAndTransformer() throws Exception {
testParquetDFSSource(false, Collections.singletonList(TripsWithDistanceTransformer.class.getName()));
@@ -1701,7 +1700,6 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
testParquetDFSSource(true, null);
}
- @Disabled("HUDI-3707 To investigate problem with schema provider and transformer")
@Test
public void testParquetDFSSourceWithSchemaFilesAndTransformer() throws Exception {
testParquetDFSSource(true, Collections.singletonList(TripsWithDistanceTransformer.class.getName()));
@@ -1712,7 +1710,6 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
testORCDFSSource(false, null);
}
- @Disabled("HUDI-3707 To investigate problem with schema provider and transformer")
@Test
public void testORCDFSSourceWithSchemaProviderAndWithTransformer() throws Exception {
testORCDFSSource(true, Collections.singletonList(TripsWithDistanceTransformer.class.getName()));
@@ -1807,7 +1804,6 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
testCsvDFSSource(true, '\t', false, Collections.singletonList(TripsWithDistanceTransformer.class.getName()));
}
- @Disabled("HUDI-3707 To investigate problem with schema provider and transformer")
@Test
public void testCsvDFSSourceWithHeaderAndSepWithSchemaProviderAndTransformer() throws Exception {
// The CSV files have header, the columns are separated by '\t'
@@ -1850,7 +1846,6 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
assertTrue(e.getMessage().contains("cannot resolve '`begin_lat`' given input columns:"));
}
- @Disabled("HUDI-3707 To investigate problem with schema provider and transformer")
@Test
public void testCsvDFSSourceNoHeaderWithSchemaProviderAndTransformer() throws Exception {
// The CSV files do not have header, the columns are separated by '\t'
[hudi] 03/11: [MINOR] Fix typos in log4j-surefire.properties (#5212)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 4a4407288a10feac3b8e781c03347424be84eba5
Author: 董可伦 <do...@inspur.com>
AuthorDate: Sat Apr 16 04:33:37 2022 +0800
[MINOR] Fix typos in log4j-surefire.properties (#5212)
---
.../hudi-client-common/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-flink-client/src/main/resources/log4j-surefire.properties | 4 ++--
.../hudi-flink-client/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-java-client/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-spark-client/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-common/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-examples-flink/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-examples-spark/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-flink/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-hadoop-mr/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-integ-test/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-kafka-connect/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-spark/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-spark2/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-spark3/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-datahub-sync/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-sync/hudi-dla-sync/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties | 4 ++--
.../hudi-sync-common/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-timeline-service/src/test/resources/log4j-surefire.properties | 4 ++--
hudi-utilities/src/test/resources/log4j-surefire.properties | 4 ++--
21 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties
+++ b/hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties b/hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties
+++ b/hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties
+++ b/hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties
+++ b/hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties
+++ b/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-common/src/test/resources/log4j-surefire.properties b/hudi-common/src/test/resources/log4j-surefire.properties
index c5bdf75ae2..31841a6a3a 100644
--- a/hudi-common/src/test/resources/log4j-surefire.properties
+++ b/hudi-common/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties b/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties
index 8dcd17f303..ac2807e332 100644
--- a/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties
+++ b/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties b/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties
index 8dcd17f303..ac2807e332 100644
--- a/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties
+++ b/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties b/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties
index 5806188cb5..6108218dc5 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties
+++ b/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties
@@ -19,9 +19,9 @@ log4j.rootLogger=INFO, CONSOLE
log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties b/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties
index c03e808cca..fe87e2deeb 100644
--- a/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties
+++ b/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties
@@ -19,9 +19,9 @@ log4j.rootLogger=WARN, CONSOLE
log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-integ-test/src/test/resources/log4j-surefire.properties b/hudi-integ-test/src/test/resources/log4j-surefire.properties
index c03e808cca..fe87e2deeb 100644
--- a/hudi-integ-test/src/test/resources/log4j-surefire.properties
+++ b/hudi-integ-test/src/test/resources/log4j-surefire.properties
@@ -19,9 +19,9 @@ log4j.rootLogger=WARN, CONSOLE
log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-kafka-connect/src/test/resources/log4j-surefire.properties b/hudi-kafka-connect/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-kafka-connect/src/test/resources/log4j-surefire.properties
+++ b/hudi-kafka-connect/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties b/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties
+++ b/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties b/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties
+++ b/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-spark-datasource/hudi-spark3/src/test/resources/log4j-surefire.properties b/hudi-spark-datasource/hudi-spark3/src/test/resources/log4j-surefire.properties
index 32af462093..14bbb08972 100644
--- a/hudi-spark-datasource/hudi-spark3/src/test/resources/log4j-surefire.properties
+++ b/hudi-spark-datasource/hudi-spark3/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties
index 7914f0a782..9782bf7d9c 100644
--- a/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties
+++ b/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties
@@ -18,9 +18,9 @@
log4j.rootLogger=WARN, CONSOLE
log4j.logger.org.apache.hudi=INFO
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-sync/hudi-dla-sync/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-dla-sync/src/test/resources/log4j-surefire.properties
index c03e808cca..fe87e2deeb 100644
--- a/hudi-sync/hudi-dla-sync/src/test/resources/log4j-surefire.properties
+++ b/hudi-sync/hudi-dla-sync/src/test/resources/log4j-surefire.properties
@@ -19,9 +19,9 @@ log4j.rootLogger=WARN, CONSOLE
log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties
index c03e808cca..fe87e2deeb 100644
--- a/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties
+++ b/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties
@@ -19,9 +19,9 @@ log4j.rootLogger=WARN, CONSOLE
log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties
index c03e808cca..fe87e2deeb 100644
--- a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties
+++ b/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties
@@ -19,9 +19,9 @@ log4j.rootLogger=WARN, CONSOLE
log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-timeline-service/src/test/resources/log4j-surefire.properties b/hudi-timeline-service/src/test/resources/log4j-surefire.properties
index c03e808cca..fe87e2deeb 100644
--- a/hudi-timeline-service/src/test/resources/log4j-surefire.properties
+++ b/hudi-timeline-service/src/test/resources/log4j-surefire.properties
@@ -19,9 +19,9 @@ log4j.rootLogger=WARN, CONSOLE
log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
diff --git a/hudi-utilities/src/test/resources/log4j-surefire.properties b/hudi-utilities/src/test/resources/log4j-surefire.properties
index c5bdf75ae2..31841a6a3a 100644
--- a/hudi-utilities/src/test/resources/log4j-surefire.properties
+++ b/hudi-utilities/src/test/resources/log4j-surefire.properties
@@ -20,9 +20,9 @@ log4j.logger.org.apache=INFO
log4j.logger.org.apache.hudi=DEBUG
log4j.logger.org.apache.hadoop.hbase=ERROR
-# A1 is set to be a ConsoleAppender.
+# CONSOLE is set to be a ConsoleAppender.
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
+# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
[hudi] 11/11: [HUDI-3894] Fix gcp bundle to include HBase dependencies and shading (#5349)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c0cadbca6560ace603acc056af5597befa6be6e7
Author: Raymond Xu <27...@users.noreply.github.com>
AuthorDate: Mon Apr 18 21:47:10 2022 -0700
[HUDI-3894] Fix gcp bundle to include HBase dependencies and shading (#5349)
---
packaging/hudi-gcp-bundle/pom.xml | 117 ++++++++++++++++++++++++++++++++++++++
1 file changed, 117 insertions(+)
diff --git a/packaging/hudi-gcp-bundle/pom.xml b/packaging/hudi-gcp-bundle/pom.xml
index 609b3652a6..6627b774c9 100644
--- a/packaging/hudi-gcp-bundle/pom.xml
+++ b/packaging/hudi-gcp-bundle/pom.xml
@@ -87,6 +87,7 @@
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<artifactSet>
<includes>
@@ -97,8 +98,122 @@
<include>com.google.cloud:google-cloud-bigquery</include>
<include>com.beust:jcommander</include>
+ <include>commons-io:commons-io</include>
+ <include>org.apache.hbase:hbase-common</include>
+ <include>org.apache.hbase:hbase-client</include>
+ <include>org.apache.hbase:hbase-hadoop-compat</include>
+ <include>org.apache.hbase:hbase-hadoop2-compat</include>
+ <include>org.apache.hbase:hbase-metrics</include>
+ <include>org.apache.hbase:hbase-metrics-api</include>
+ <include>org.apache.hbase:hbase-protocol-shaded</include>
+ <include>org.apache.hbase:hbase-server</include>
+ <include>org.apache.hbase.thirdparty:hbase-shaded-miscellaneous</include>
+ <include>org.apache.hbase.thirdparty:hbase-shaded-netty</include>
+ <include>org.apache.hbase.thirdparty:hbase-shaded-protobuf</include>
+ <include>org.apache.htrace:htrace-core4</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.esotericsoftware.kryo.</pattern>
+ <shadedPattern>org.apache.hudi.com.esotericsoftware.kryo.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.esotericsoftware.minlog.</pattern>
+ <shadedPattern>org.apache.hudi.com.esotericsoftware.minlog.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.io.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.commons.io.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.hbase.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.hbase.</shadedPattern>
+ <excludes>
+ <exclude>org.apache.hadoop.hbase.KeyValue$KeyComparator</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hbase.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hbase.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.htrace.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.htrace.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.objenesis.</pattern>
+ <shadedPattern>org.apache.hudi.org.objenesis.</shadedPattern>
+ </relocation>
+ <!-- The classes below in org.apache.hadoop.metrics2 package come from
+ hbase-hadoop-compat and hbase-hadoop2-compat, which have to be shaded one by one,
+ instead of shading all classes under org.apache.hadoop.metrics2 including ones
+ from hadoop. -->
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.MetricHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.MetricHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.MetricsExecutor</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.MetricsExecutor
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.impl.JmxCacheBuster</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.impl.JmxCacheBuster</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MetricsExecutorImpl</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MetricsExecutorImpl
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableFastCounter</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableFastCounter
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableRangeHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableRangeHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableSizeHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableSizeHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.lib.MutableTimeHistogram</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableTimeHistogram
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.util.MetricQuantile</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.util.MetricQuantile
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.hadoop.metrics2.util.MetricSampleQuantiles</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.hadoop.metrics2.util.MetricSampleQuantiles
+ </shadedPattern>
+ </relocation>
+ </relocations>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
@@ -108,6 +223,8 @@
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/services/javax.*</exclude>
+ <exclude>**/*.proto</exclude>
+ <exclude>hbase-webapps/**</exclude>
</excludes>
</filter>
</filters>
[hudi] 01/11: [MINOR] Removing invalid code to close parquet reader iterator (#5182)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch release-0.11.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d886e9eb8df9d74d2b8a00a98e851704af183364
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Fri Apr 15 14:50:07 2022 -0400
[MINOR] Removing invalid code to close parquet reader iterator (#5182)
---
.../src/main/scala/org/apache/hudi/HoodieBaseRelation.scala | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
index c33c6dce6d..769016ca8a 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
@@ -422,13 +422,7 @@ object HoodieBaseRelation {
partitionedFile => {
val extension = FSUtils.getFileExtension(partitionedFile.filePath)
if (HoodieFileFormat.PARQUET.getFileExtension.equals(extension)) {
- val iter = parquetReader.apply(partitionedFile)
- if (iter.isInstanceOf[Closeable]) {
- // register a callback to close parquetReader which will be executed on task completion.
- // when tasks finished, this method will be called, and release resources.
- Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => iter.asInstanceOf[Closeable].close()))
- }
- iter
+ parquetReader.apply(partitionedFile)
} else if (HoodieFileFormat.HFILE.getFileExtension.equals(extension)) {
hfileReader.apply(partitionedFile)
} else {