You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/04/30 14:44:59 UTC
carbondata git commit: [CARBONDATA-2393]TaskNo is not working for SDK
Repository: carbondata
Updated Branches:
refs/heads/master dde0873f7 -> 161347155
[CARBONDATA-2393]TaskNo is not working for SDK
Issue:- Task No is not getting reflected in the Carbon Data file and in index file .
Cause :- Task No is getting overwritten in CarbonTableOutputformate even CarbonModel have taskNo.
Solution :- if CarbomModel has taskNo then no need to overwrite .
This closes #2224
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/16134715
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/16134715
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/16134715
Branch: refs/heads/master
Commit: 161347155d9c429cf7275a617ace6dbc4c0fcdaf
Parents: dde0873
Author: BJangir <ba...@gmail.com>
Authored: Tue Apr 24 20:12:54 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Mon Apr 30 20:13:12 2018 +0530
----------------------------------------------------------------------
.../hadoop/api/CarbonTableOutputFormat.java | 8 +--
.../sdk/file/CSVCarbonWriterTest.java | 52 +++++++++++++++++++-
2 files changed, 56 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/16134715/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index 43e0221..36ba02d 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -233,9 +233,11 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Obje
public RecordWriter<NullWritable, ObjectArrayWritable> getRecordWriter(
TaskAttemptContext taskAttemptContext) throws IOException {
final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration());
- loadModel.setTaskNo(taskAttemptContext.getConfiguration().get(
- "carbon.outputformat.taskno",
- String.valueOf(System.nanoTime())));
+ //if loadModel having taskNo already(like in SDK) then no need to overwrite
+ if (null == loadModel.getTaskNo() || loadModel.getTaskNo().isEmpty()) {
+ loadModel.setTaskNo(taskAttemptContext.getConfiguration()
+ .get("carbon.outputformat.taskno", String.valueOf(System.nanoTime())));
+ }
loadModel.setDataWritePath(
taskAttemptContext.getConfiguration().get("carbon.outputformat.writepath"));
final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/16134715/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
index 76778e5..c4dcee9 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
@@ -205,7 +205,6 @@ public class CSVCarbonWriterTest {
FileUtils.deleteDirectory(new File(path));
}
-
@Test(expected = IOException.class)
public void testWhenWriterthrowsError() throws IOException{
CarbonWriter carbonWriter = null;
@@ -246,4 +245,55 @@ public class CSVCarbonWriterTest {
carbonWriter.close();
}
+
+ @Test
+ public void testTaskNo() throws IOException {
+ // TODO: write all data type and read by CarbonRecordReader to verify the content
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("intField", DataTypes.INT);
+
+
+ try {
+ CarbonWriterBuilder builder = CarbonWriter.builder()
+ .withSchema(new Schema(fields))
+ .isTransactionalTable(true).taskNo("5")
+ .outputPath(path);
+
+ CarbonWriter writer = builder.buildWriterForCSVInput();
+
+ for (int i = 0; i < 2; i++) {
+ String[] row = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i)
+ };
+ writer.write(row);
+ }
+ writer.close();
+
+ File segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null"));
+ Assert.assertTrue(segmentFolder.exists());
+
+ File[] dataFiles = segmentFolder.listFiles(new FileFilter() {
+ @Override public boolean accept(File pathname) {
+ return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
+ }
+ });
+ Assert.assertNotNull(dataFiles);
+ Assert.assertTrue(dataFiles.length > 0);
+ String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(dataFiles[0].getName());
+ long taskID = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskNo);
+ Assert.assertEquals("Task Id is not matched", taskID, 5);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ finally {
+ FileUtils.deleteDirectory(new File(path));
+ }
+ }
+
}