You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by in...@apache.org on 2021/02/18 05:08:34 UTC

[carbondata] branch master updated: [CARBONDATA-3962] Fixed concurrent load failure with flat folder structure.

This is an automated email from the ASF dual-hosted git repository.

indhumuthumurugesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 1cab165  [CARBONDATA-3962] Fixed concurrent load failure with flat folder structure.
1cab165 is described below

commit 1cab165deb94a8ea1b2009442de92fb7b6bf05f3
Author: Nihal ojha <ni...@gmail.com>
AuthorDate: Tue Feb 16 13:01:34 2021 +0530

    [CARBONDATA-3962] Fixed concurrent load failure with flat folder structure.
    
    Why is this PR needed?
    PR #3904 has added the code to remove fact directory and because of this concurrent
    load fails with file not found exception.
    
    What changes were proposed in this PR?
    Reverted PR 3904.
    
    This closes #4905
---
 .../org/apache/carbondata/core/metadata/SegmentFileStore.java  | 10 +---------
 .../testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala  |  8 +++-----
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index 9e8be0a..0a26b5e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -443,22 +443,14 @@ public class SegmentFileStore {
   /**
    * Move the loaded data from source folder to destination folder.
    */
-  private static void moveFromTempFolder(String source, String dest) throws IOException {
+  private static void moveFromTempFolder(String source, String dest) {
 
     CarbonFile oldFolder = FileFactory.getCarbonFile(source);
     CarbonFile[] oldFiles = oldFolder.listFiles();
     for (CarbonFile file : oldFiles) {
       file.renameForce(dest + CarbonCommonConstants.FILE_SEPARATOR + file.getName());
     }
-    // delete the segment path at any cost at first, we we dont want to delete fact directory in
-    // case of multiple load scenario or update, delete scenario
     oldFolder.delete();
-    CarbonFile partDir = FileFactory.getCarbonFile(CarbonTablePath.getPartitionDir(dest));
-    // once last segment is processed(in case of update delete), delete the main fact directory
-    if (partDir.listFiles(false).size() == 0) {
-      CarbonFile oldFactDirPath = FileFactory.getCarbonFile(CarbonTablePath.getFactDir(dest));
-      oldFactDirPath.delete();
-    }
   }
 
   /**
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
index c6306c0..cde2f88 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
@@ -46,12 +46,10 @@ class FlatFolderTableLoadingTestCase extends QueryTest with BeforeAndAfterAll {
 
   }
 
-  def validateDataFiles(tableUniqueName: String): Unit = {
+  def validateDataFiles(tableUniqueName: String, segmentId: String): Unit = {
     val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
     val files = FileFactory.getCarbonFile(carbonTable.getTablePath).listFiles()
-    val factPath = FileFactory.getCarbonFile(CarbonTablePath.getFactDir(carbonTable.getTablePath))
     assert(files.exists(_.getName.endsWith(CarbonTablePath.CARBON_DATA_EXT)))
-    assert(!factPath.exists())
   }
 
   test("data loading for flat folder with global sort") {
@@ -65,7 +63,7 @@ class FlatFolderTableLoadingTestCase extends QueryTest with BeforeAndAfterAll {
       """.stripMargin)
     sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE flatfolder_gs OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
 
-    validateDataFiles("default_flatfolder_gs")
+    validateDataFiles("default_flatfolder_gs", "0")
 
     checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from flatfolder_gs order by empno"),
       sql("select  empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
@@ -83,7 +81,7 @@ class FlatFolderTableLoadingTestCase extends QueryTest with BeforeAndAfterAll {
       """.stripMargin)
     sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE flatfolder OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
 
-    validateDataFiles("default_flatfolder")
+    validateDataFiles("default_flatfolder", "0")
 
     checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from flatfolder order by empno"),
       sql("select  empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))