You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by le...@apache.org on 2020/08/14 17:27:47 UTC

[incubator-gobblin] branch master updated: [GOBBLIN-1222] Create right abstraction to assemble dataset staging dir for Hive dataset finder

This is an automated email from the ASF dual-hosted git repository.

lesun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new abd71db  [GOBBLIN-1222] Create right abstraction to assemble dataset staging dir for Hive dataset finder
abd71db is described below

commit abd71dbef280b7ba741343c8634688e7c99de394
Author: Vaibhav Arya <va...@varya-mn1.linkedin.biz>
AuthorDate: Fri Aug 14 10:27:24 2020 -0700

    [GOBBLIN-1222] Create right abstraction to assemble dataset staging dir for Hive dataset finder
    
    Closes #3070 from aryavaibhav93/master
---
 .../apache/gobblin/configuration/ConfigurationKeys.java   |  1 +
 .../apache/gobblin/data/management/copy/CopySource.java   |  7 +++++++
 .../gobblin/data/management/copy/hive/HiveDataset.java    |  8 ++++++++
 .../data/management/copy/hive/HiveDatasetFinder.java      |  2 ++
 .../copy/writer/FileAwareInputStreamDataWriter.java       | 15 ++++++++++++++-
 5 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java
index 98434e1..1300af6 100644
--- a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java
+++ b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java
@@ -209,6 +209,7 @@ public class ConfigurationKeys {
   public static final String MAXIMUM_JAR_COPY_RETRY_TIMES_KEY = JOB_JAR_FILES_KEY + ".uploading.retry.maximum";
   public static final String USER_DEFINED_STATIC_STAGING_DIR = "user.defined.static.staging.dir";
   public static final String USER_DEFINED_STAGING_DIR_FLAG = "user.defined.staging.dir.flag";
+  public static final String IS_DATASET_STAGING_DIR_USED = "dataset.staging.dir.used";
 
   public static final String QUEUED_TASK_TIME_MAX_SIZE = "taskexecutor.queued_task_time.history.max_size";
   public static final int DEFAULT_QUEUED_TASK_TIME_MAX_SIZE = 2048;
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopySource.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopySource.java
index e7d2fe9..8b72d54 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopySource.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopySource.java
@@ -53,6 +53,8 @@ import org.apache.gobblin.configuration.State;
 import org.apache.gobblin.configuration.WorkUnitState;
 import org.apache.gobblin.data.management.copy.extractor.EmptyExtractor;
 import org.apache.gobblin.data.management.copy.extractor.FileAwareInputStreamExtractor;
+import org.apache.gobblin.data.management.copy.hive.HiveDataset;
+import org.apache.gobblin.data.management.copy.hive.HiveDatasetFinder;
 import org.apache.gobblin.data.management.copy.prioritization.FileSetComparator;
 import org.apache.gobblin.data.management.copy.publisher.CopyEventSubmitterHelper;
 import org.apache.gobblin.data.management.copy.replication.ConfigBasedDataset;
@@ -130,6 +132,8 @@ public class CopySource extends AbstractSource<String, FileAwareInputStream> {
   public static final String FILESET_TOTAL_ENTITIES = "fileset.total.entities";
   public static final String FILESET_TOTAL_SIZE_IN_BYTES = "fileset.total.size";
   public static final String SCHEMA_CHECK_ENABLED = "shcema.check.enabled";
+  public static final String DATASET_STAGING_DIR_PATH = "dataset.staging.dir.path";
+  public static final String DATASET_STAGING_PATH = "dataset.staging.path";
   public final static boolean DEFAULT_SCHEMA_CHECK_ENABLED = false;
 
   private static final String WORK_UNIT_WEIGHT = CopyConfiguration.COPY_PREFIX + ".workUnitWeight";
@@ -366,6 +370,9 @@ public class CopySource extends AbstractSource<String, FileAwareInputStream> {
               workUnit.setProp(ConfigurationKeys.COPY_EXPECTED_SCHEMA, ((ConfigBasedDataset) this.copyableDataset).getExpectedSchema());
             }
           }
+          if ((this.copyableDataset instanceof HiveDataset) && (state.getPropAsBoolean(ConfigurationKeys.IS_DATASET_STAGING_DIR_USED,false))) {
+            workUnit.setProp(DATASET_STAGING_DIR_PATH, ((HiveDataset) this.copyableDataset).getProperties().getProperty(DATASET_STAGING_PATH));
+          }
           serializeCopyEntity(workUnit, copyEntity);
           serializeCopyableDataset(workUnit, metadata);
           GobblinMetrics.addCustomTagToState(workUnit,
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java
index d550ec9..059efff 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java
@@ -67,6 +67,7 @@ import org.apache.gobblin.util.ConfigUtils;
 import org.apache.gobblin.util.PathUtils;
 import org.apache.gobblin.util.request_allocation.PushDownRequestor;
 
+import static org.apache.gobblin.data.management.copy.hive.HiveTargetPathHelper.*;
 
 /**
  * Hive dataset implementing {@link CopyableDataset}.
@@ -85,6 +86,7 @@ public class HiveDataset implements PrioritizedCopyableDataset {
   public static final String DATASET_NAME_PATTERN_KEY = "hive.datasetNamePattern";
   public static final String DATABASE = "Database";
   public static final String TABLE = "Table";
+  public static final String DATASET_STAGING_PATH = "dataset.staging.path";
 
   public static final String DATABASE_TOKEN = "$DB";
   public static final String TABLE_TOKEN = "$TABLE";
@@ -93,6 +95,7 @@ public class HiveDataset implements PrioritizedCopyableDataset {
   public static final String LOGICAL_TABLE_TOKEN = "$LOGICAL_TABLE";
 
   // Will not be serialized/de-serialized
+  @Getter
   protected transient final Properties properties;
   protected transient final FileSystem fs;
   protected transient final HiveMetastoreClientPool clientPool;
@@ -125,6 +128,11 @@ public class HiveDataset implements PrioritizedCopyableDataset {
         Optional.fromNullable(this.table.getDataLocation());
 
     this.tableIdentifier = this.table.getDbName() + "." + this.table.getTableName();
+    Path tableLocation = this.table.getPath();
+    if (!(this.properties.isEmpty())) {
+      String datasetStagingDir = this.properties.getProperty(COPY_TARGET_TABLE_PREFIX_REPLACEMENT) + "/" + tableLocation.getName();
+      properties.setProperty(DATASET_STAGING_PATH,datasetStagingDir);
+    }
 
     this.datasetNamePattern = Optional.fromNullable(ConfigUtils.getString(datasetConfig, DATASET_NAME_PATTERN_KEY, null));
     this.dbAndTable = new DbAndTable(table.getDbName(), table.getTableName());
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java
index e04783f..a73ae70 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetFinder.java
@@ -29,6 +29,7 @@ import java.util.Properties;
 import javax.annotation.Nonnull;
 
 import lombok.Data;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 import org.apache.commons.lang3.StringUtils;
@@ -106,6 +107,7 @@ public class HiveDatasetFinder implements IterableDatasetFinder<HiveDataset> {
   private static final String DATASET_ERROR = "DatasetError";
   private static final String FAILURE_CONTEXT = "FailureContext";
 
+  @Getter
   protected final Properties properties;
   protected final HiveMetastoreClientPool clientPool;
   protected final FileSystem fs;
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java
index 53ca24a..852c9b4 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java
@@ -88,6 +88,8 @@ public class FileAwareInputStreamDataWriter extends InstrumentedDataWriter<FileA
   public static final boolean DEFAULT_GOBBLIN_COPY_CHECK_FILESIZE = false;
   public static final String GOBBLIN_COPY_TASK_OVERWRITE_ON_COMMIT = "gobblin.copy.task.overwrite.on.commit";
   public static final boolean DEFAULT_GOBBLIN_COPY_TASK_OVERWRITE_ON_COMMIT = false;
+  public static final String STAGING_DIR_SUFFIX = "/taskStaging";
+  public static final String DATASET_STAGING_DIR_PATH = "dataset.staging.dir.path";
 
   protected final AtomicLong bytesWritten = new AtomicLong();
   protected final AtomicLong filesWritten = new AtomicLong();
@@ -140,16 +142,27 @@ public class FileAwareInputStreamDataWriter extends InstrumentedDataWriter<FileA
     this.fs = FileSystem.get(uri, conf);
     this.fileContext = FileContext.getFileContext(uri, conf);
 
+    /**
+     * The staging directory defines the path of staging folder.
+     * USER_DEFINED_STATIC_STAGING_DIR_FLAG shall be set to true when user wants to specify the staging folder and the directory can be fetched through USER_DEFINED_STATIC_STAGING_DIR property.
+     * IS_DATASET_STAGING_DIR_USED when true creates the staging folder within a dataset location for dataset copying.
+     * Else system will calculate the staging directory automatically.
+     */
     if (state.getPropAsBoolean(ConfigurationKeys.USER_DEFINED_STAGING_DIR_FLAG,false)) {
       this.stagingDir = new Path(state.getProp(ConfigurationKeys.USER_DEFINED_STATIC_STAGING_DIR));
+    } else if ((state.getPropAsBoolean(ConfigurationKeys.IS_DATASET_STAGING_DIR_USED,false))) {
+      String stgDir = state.getProp(DATASET_STAGING_DIR_PATH) + STAGING_DIR_SUFFIX + "/" + state.getProp(ConfigurationKeys.JOB_NAME_KEY ) + "/" + state.getProp(ConfigurationKeys.JOB_ID_KEY);
+      state.setProp(ConfigurationKeys.WRITER_STAGING_DIR,stgDir);
+      this.stagingDir = this.writerAttemptIdOptional.isPresent() ? WriterUtils.getWriterStagingDir(state, numBranches, branchId, this.writerAttemptIdOptional.get())
+          : WriterUtils.getWriterStagingDir(state, numBranches, branchId);
     } else {
       this.stagingDir = this.writerAttemptIdOptional.isPresent() ? WriterUtils.getWriterStagingDir(state, numBranches, branchId, this.writerAttemptIdOptional.get())
           : WriterUtils.getWriterStagingDir(state, numBranches, branchId);
     }
 
-    this.outputDir = getOutputDir(state);
     this.copyableDatasetMetadata =
         CopyableDatasetMetadata.deserialize(state.getProp(CopySource.SERIALIZED_COPYABLE_DATASET));
+    this.outputDir = getOutputDir(state);
     this.recoveryHelper = new RecoveryHelper(this.fs, state);
     this.actualProcessedCopyableFile = Optional.absent();