You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/03/27 14:57:40 UTC
[2/6] incubator-carbondata git commit: Store fileName insteads of
filePath in carbonindex.
Store fileName insteads of filePath in carbonindex.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/da4d39eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/da4d39eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/da4d39eb
Branch: refs/heads/master
Commit: da4d39eb26d2baff1acf3e41b6991b5239ccef3b
Parents: 4da2a70
Author: Yadong Qi <qi...@gmail.com>
Authored: Sat Mar 25 15:38:23 2017 +0800
Committer: Yadong Qi <qi...@gmail.com>
Committed: Sat Mar 25 15:38:23 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/util/path/CarbonTablePath.java | 7 +++++++
.../dataload/TestDataLoadWithFileName.scala | 15 +++++++++------
.../store/writer/AbstractFactDataWriter.java | 8 +++-----
.../store/writer/v1/CarbonFactDataWriterImplV1.java | 2 +-
.../store/writer/v2/CarbonFactDataWriterImplV2.java | 2 +-
.../store/writer/v3/CarbonFactDataWriterImplV3.java | 2 +-
6 files changed, 22 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 7e37689..3e81fbb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -579,6 +579,13 @@ public class CarbonTablePath extends Path {
}
/**
+ * @return prefix of carbon data
+ */
+ public static String getCarbonDataPrefix() {
+ return DATA_PART_PREFIX;
+ }
+
+ /**
*
* @return carbon data extension
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index df7e71e..be81b2d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -17,7 +17,7 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance.getProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION)
}
- test("Test the file_name in carbonindex with v1 format") {
+ test("Check the file_name in carbonindex with v1 format") {
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "1")
sql("DROP TABLE IF EXISTS test_table_v1")
sql(
@@ -39,12 +39,13 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
assert(indexReader.readIndexHeader().getVersion === 1)
while (indexReader.hasNext) {
val readBlockIndexInfo = indexReader.readBlockIndexInfo()
- assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ assert(readBlockIndexInfo.getFile_name.startsWith(CarbonTablePath.getCarbonDataPrefix))
+ assert(readBlockIndexInfo.getFile_name.endsWith(CarbonTablePath.getCarbonDataExtension))
}
}
}
- test("Test the file_name in carbonindex with v2 format") {
+ test("Check the file_name in carbonindex with v2 format") {
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "2")
sql("DROP TABLE IF EXISTS test_table_v2")
sql(
@@ -66,12 +67,13 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
assert(indexReader.readIndexHeader().getVersion === 2)
while (indexReader.hasNext) {
val readBlockIndexInfo = indexReader.readBlockIndexInfo()
- assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ assert(readBlockIndexInfo.getFile_name.startsWith(CarbonTablePath.getCarbonDataPrefix))
+ assert(readBlockIndexInfo.getFile_name.endsWith(CarbonTablePath.getCarbonDataExtension))
}
}
}
- test("Test the file_name in carbonindex with v3 format") {
+ test("Check the file_name in carbonindex with v3 format") {
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "3")
sql("DROP TABLE IF EXISTS test_table_v3")
sql(
@@ -93,7 +95,8 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
assert(indexReader.readIndexHeader().getVersion === 3)
while (indexReader.hasNext) {
val readBlockIndexInfo = indexReader.readBlockIndexInfo()
- assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ assert(readBlockIndexInfo.getFile_name.startsWith(CarbonTablePath.getCarbonDataPrefix))
+ assert(readBlockIndexInfo.getFile_name.endsWith(CarbonTablePath.getCarbonDataExtension))
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index dc431d1..f820999 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -100,9 +100,9 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
protected String fileName;
/**
- * The path of carbonData file
+ * The name of carbonData file
*/
- protected String carbonDataFilePath;
+ protected String carbonDataFileName;
/**
* Local cardinality for the segment
@@ -292,7 +292,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
// update the filename with new new sequence
// increment the file sequence counter
initFileCount();
- String carbonDataFileName = carbonTablePath
+ this.carbonDataFileName = carbonTablePath
.getCarbonDataFileName(fileCount, dataWriterVo.getCarbonDataFileAttributes().getTaskId(),
dataWriterVo.getBucketNumber(), dataWriterVo.getTaskExtension(),
dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
@@ -301,8 +301,6 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
dataWriterVo.getFileManager().add(fileData);
this.fileName = dataWriterVo.getStoreLocation() + File.separator + carbonDataFileName
+ CarbonCommonConstants.FILE_INPROGRESS_STATUS;
- this.carbonDataFilePath =
- dataWriterVo.getCarbonDataDirectoryPath() + File.separator + carbonDataFileName;
this.fileCount++;
try {
// open channel for new data file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index ea81083..cf5311c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -373,7 +373,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
FileFooter convertFileMeta = CarbonMetadataUtil
.convertFileFooter(blockletInfoList, localCardinality.length, localCardinality,
thriftColumnSchemaList, dataWriterVo.getSegmentProperties());
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
writer.writeFooter(convertFileMeta, currentPosition);
} catch (IOException e) {
throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
index 562f6d2..ec79186 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
@@ -276,7 +276,7 @@ public class CarbonFactDataWriterImplV2 extends CarbonFactDataWriterImplV1 {
.convertFilterFooter2(blockletInfoList, localCardinality, thriftColumnSchemaList,
dataChunksOffsets, dataChunksLength);
// fill the carbon index details
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
// write the footer
writer.writeFooter(convertFileMeta, currentPosition);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index b264a04..2b138b2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -294,7 +294,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
.convertFileFooterVersion3(blockletMetadata, blockletIndex, localCardinality,
thriftColumnSchemaList.size(), dataWriterVo.getSegmentProperties());
// fill the carbon index details
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
// write the footer
byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
ByteBuffer buffer =