You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/03/27 14:57:39 UTC
[1/6] incubator-carbondata git commit: Make the file_name in
carbonindex exactly.
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 50a24089c -> b615892a8
Make the file_name in carbonindex exactly.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4da2a705
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4da2a705
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4da2a705
Branch: refs/heads/master
Commit: 4da2a705ed8f050a8e89da0780d3c56751208a2e
Parents: b7c636f
Author: Yadong Qi <qi...@gmail.com>
Authored: Fri Mar 24 16:26:20 2017 +0800
Committer: Yadong Qi <qi...@gmail.com>
Committed: Fri Mar 24 16:45:55 2017 +0800
----------------------------------------------------------------------
.../dataload/TestDataLoadWithFileName.scala | 108 +++++++++++++++++++
.../store/writer/AbstractFactDataWriter.java | 11 +-
.../writer/v1/CarbonFactDataWriterImplV1.java | 2 +-
.../writer/v2/CarbonFactDataWriterImplV2.java | 2 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 5 +-
5 files changed, 121 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4da2a705/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
new file mode 100644
index 0000000..df7e71e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -0,0 +1,108 @@
+package org.apache.carbondata.spark.testsuite.dataload
+
+import java.io.{File, FilenameFilter}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.reader.CarbonIndexFileReader
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
+ var originVersion = ""
+
+ override def beforeAll() {
+ originVersion =
+ CarbonProperties.getInstance.getProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION)
+ }
+
+ test("Test the file_name in carbonindex with v1 format") {
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "1")
+ sql("DROP TABLE IF EXISTS test_table_v1")
+ sql(
+ """
+ | CREATE TABLE test_table_v1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ val testData = s"$resourcesPath/sample.csv"
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table test_table_v1")
+ val indexReader = new CarbonIndexFileReader()
+ val carbonIndexPaths = new File(s"$storeLocation/default/test_table_v1/Fact/Part0/Segment_0/")
+ .listFiles(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.endsWith(CarbonTablePath.getCarbonIndexExtension)
+ }
+ })
+ for (carbonIndexPath <- carbonIndexPaths) {
+ indexReader.openThriftReader(carbonIndexPath.getCanonicalPath)
+ assert(indexReader.readIndexHeader().getVersion === 1)
+ while (indexReader.hasNext) {
+ val readBlockIndexInfo = indexReader.readBlockIndexInfo()
+ assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ }
+ }
+ }
+
+ test("Test the file_name in carbonindex with v2 format") {
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "2")
+ sql("DROP TABLE IF EXISTS test_table_v2")
+ sql(
+ """
+ | CREATE TABLE test_table_v2(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ val testData = s"$resourcesPath/sample.csv"
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table test_table_v2")
+ val indexReader = new CarbonIndexFileReader()
+ val carbonIndexPaths = new File(s"$storeLocation/default/test_table_v2/Fact/Part0/Segment_0/")
+ .listFiles(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.endsWith(CarbonTablePath.getCarbonIndexExtension)
+ }
+ })
+ for (carbonIndexPath <- carbonIndexPaths) {
+ indexReader.openThriftReader(carbonIndexPath.getCanonicalPath)
+ assert(indexReader.readIndexHeader().getVersion === 2)
+ while (indexReader.hasNext) {
+ val readBlockIndexInfo = indexReader.readBlockIndexInfo()
+ assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ }
+ }
+ }
+
+ test("Test the file_name in carbonindex with v3 format") {
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "3")
+ sql("DROP TABLE IF EXISTS test_table_v3")
+ sql(
+ """
+ | CREATE TABLE test_table_v3(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ val testData = s"$resourcesPath/sample.csv"
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table test_table_v3")
+ val indexReader = new CarbonIndexFileReader()
+ val carbonIndexPaths = new File(s"$storeLocation/default/test_table_v3/Fact/Part0/Segment_0/")
+ .listFiles(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.endsWith(CarbonTablePath.getCarbonIndexExtension)
+ }
+ })
+ for (carbonIndexPath <- carbonIndexPaths) {
+ indexReader.openThriftReader(carbonIndexPath.getCanonicalPath)
+ assert(indexReader.readIndexHeader().getVersion === 3)
+ while (indexReader.hasNext) {
+ val readBlockIndexInfo = indexReader.readBlockIndexInfo()
+ assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ }
+ }
+ }
+
+ override protected def afterAll() {
+ sql("DROP TABLE IF EXISTS test_table_v1")
+ sql("DROP TABLE IF EXISTS test_table_v2")
+ sql("DROP TABLE IF EXISTS test_table_v3")
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
+ originVersion)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4da2a705/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 51f8960..dc431d1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -98,6 +98,12 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
* file name
*/
protected String fileName;
+
+ /**
+ * The path of carbonData file
+ */
+ protected String carbonDataFilePath;
+
/**
* Local cardinality for the segment
*/
@@ -295,6 +301,8 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
dataWriterVo.getFileManager().add(fileData);
this.fileName = dataWriterVo.getStoreLocation() + File.separator + carbonDataFileName
+ CarbonCommonConstants.FILE_INPROGRESS_STATUS;
+ this.carbonDataFilePath =
+ dataWriterVo.getCarbonDataDirectoryPath() + File.separator + carbonDataFileName;
this.fileCount++;
try {
// open channel for new data file
@@ -376,8 +384,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
minmax.setMaxValues(currentMaxValue);
BlockletIndex blockletIndex = new BlockletIndex(btree, minmax);
BlockIndexInfo blockIndexInfo =
- new BlockIndexInfo(numberOfRows, filePath.substring(0, filePath.lastIndexOf('.')),
- currentPosition, blockletIndex);
+ new BlockIndexInfo(numberOfRows, filePath, currentPosition, blockletIndex);
blockIndexInfoList.add(blockIndexInfo);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4da2a705/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index a3dd242..ea81083 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -373,7 +373,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
FileFooter convertFileMeta = CarbonMetadataUtil
.convertFileFooter(blockletInfoList, localCardinality.length, localCardinality,
thriftColumnSchemaList, dataWriterVo.getSegmentProperties());
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), filePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
writer.writeFooter(convertFileMeta, currentPosition);
} catch (IOException e) {
throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4da2a705/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
index a9c4ce0..562f6d2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
@@ -276,7 +276,7 @@ public class CarbonFactDataWriterImplV2 extends CarbonFactDataWriterImplV1 {
.convertFilterFooter2(blockletInfoList, localCardinality, thriftColumnSchemaList,
dataChunksOffsets, dataChunksLength);
// fill the carbon index details
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), filePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
// write the footer
writer.writeFooter(convertFileMeta, currentPosition);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4da2a705/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 57e03bd..b264a04 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -294,7 +294,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
.convertFileFooterVersion3(blockletMetadata, blockletIndex, localCardinality,
thriftColumnSchemaList.size(), dataWriterVo.getSegmentProperties());
// fill the carbon index details
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), filePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
// write the footer
byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
ByteBuffer buffer =
@@ -528,8 +528,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex blockletIndex =
new org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex(btree, minmax);
BlockIndexInfo blockIndexInfo =
- new BlockIndexInfo(numberOfRows, filePath.substring(0, filePath.lastIndexOf('.')),
- currentPosition, blockletIndex);
+ new BlockIndexInfo(numberOfRows, filePath, currentPosition, blockletIndex);
blockIndexInfoList.add(blockIndexInfo);
}
[3/6] incubator-carbondata git commit: Add add license header and
rename filePath to carbonDataFileName in fillBlockIndexInfoDetails.
Posted by ja...@apache.org.
Add add license header and rename filePath to carbonDataFileName in fillBlockIndexInfoDetails.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/84477cc2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/84477cc2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/84477cc2
Branch: refs/heads/master
Commit: 84477cc2590cf86ca597bd40bf4c328295e449c9
Parents: da4d39e
Author: Yadong Qi <qi...@gmail.com>
Authored: Mon Mar 27 10:20:35 2017 +0800
Committer: Yadong Qi <qi...@gmail.com>
Committed: Mon Mar 27 10:20:35 2017 +0800
----------------------------------------------------------------------
.../dataload/TestDataLoadWithFileName.scala | 17 +++++++++++++++++
.../store/writer/AbstractFactDataWriter.java | 8 ++++----
.../writer/v3/CarbonFactDataWriterImplV3.java | 6 +++---
3 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84477cc2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index be81b2d..20f2cd8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.carbondata.spark.testsuite.dataload
import java.io.{File, FilenameFilter}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84477cc2/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index f820999..a7dc534 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -347,11 +347,11 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
* Below method will be used to fill the vlock info details
*
* @param numberOfRows number of rows in file
- * @param filePath file path
+ * @param carbonDataFileName The name of carbonData file
* @param currentPosition current offset
*/
- protected void fillBlockIndexInfoDetails(long numberOfRows,
- String filePath, long currentPosition) {
+ protected void fillBlockIndexInfoDetails(long numberOfRows, String carbonDataFileName,
+ long currentPosition) {
// as min-max will change for each blocklet and second blocklet min-max can be lesser than
// the first blocklet so we need to calculate the complete block level min-max by taking
@@ -382,7 +382,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
minmax.setMaxValues(currentMaxValue);
BlockletIndex blockletIndex = new BlockletIndex(btree, minmax);
BlockIndexInfo blockIndexInfo =
- new BlockIndexInfo(numberOfRows, filePath, currentPosition, blockletIndex);
+ new BlockIndexInfo(numberOfRows, carbonDataFileName, currentPosition, blockletIndex);
blockIndexInfoList.add(blockIndexInfo);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84477cc2/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 2b138b2..668d0d8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -476,10 +476,10 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
* Below method will be used to fill the block info details
*
* @param numberOfRows number of rows in file
- * @param filePath file path
+ * @param carbonDataFileName The name of carbonData file
* @param currentPosition current offset
*/
- protected void fillBlockIndexInfoDetails(long numberOfRows, String filePath,
+ protected void fillBlockIndexInfoDetails(long numberOfRows, String carbonDataFileName,
long currentPosition) {
byte[][] currentMinValue = new byte[blockletIndex.get(0).min_max_index.max_values.size()][];
byte[][] currentMaxValue = new byte[blockletIndex.get(0).min_max_index.max_values.size()][];
@@ -528,7 +528,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex blockletIndex =
new org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex(btree, minmax);
BlockIndexInfo blockIndexInfo =
- new BlockIndexInfo(numberOfRows, filePath, currentPosition, blockletIndex);
+ new BlockIndexInfo(numberOfRows, carbonDataFileName, currentPosition, blockletIndex);
blockIndexInfoList.add(blockIndexInfo);
}
[5/6] incubator-carbondata git commit: Fix style.
Posted by ja...@apache.org.
Fix style.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9cb428c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9cb428c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9cb428c9
Branch: refs/heads/master
Commit: 9cb428c959642a399cf0bb927189424c80d5830d
Parents: d17f58f
Author: Yadong Qi <qi...@gmail.com>
Authored: Mon Mar 27 21:30:05 2017 +0800
Committer: Yadong Qi <qi...@gmail.com>
Committed: Mon Mar 27 21:30:05 2017 +0800
----------------------------------------------------------------------
.../processing/store/writer/AbstractFactDataWriter.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9cb428c9/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 90eab6f..a73b356 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -261,8 +261,8 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
protected void updateBlockletFileChannel(long blockletDataSize) throws CarbonDataWriterException {
if ((currentFileSize + blockletDataSize) >= dataBlockSize && currentFileSize != 0) {
// set the current file size to zero
- LOGGER.info("Writing data to file as max file size reached for file: " + carbonDataFileTempPath
- + " .Data block size: " + currentFileSize);
+ LOGGER.info("Writing data to file as max file size reached for file: "
+ + carbonDataFileTempPath + " .Data block size: " + currentFileSize);
// write meta data to end of the existing file
writeBlockletInfoToFile(fileChannel, carbonDataFileTempPath);
this.currentFileSize = 0;
[6/6] incubator-carbondata git commit: [CARBONDATA-818] Make the
file_name in carbonindex exactly This closes #696
Posted by ja...@apache.org.
[CARBONDATA-818] Make the file_name in carbonindex exactly This closes #696
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b615892a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b615892a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b615892a
Branch: refs/heads/master
Commit: b615892a8908e983e8b4dc0cb26618249a5d3c6f
Parents: 50a2408 9cb428c
Author: jackylk <ja...@huawei.com>
Authored: Mon Mar 27 20:27:17 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Mon Mar 27 20:27:17 2017 +0530
----------------------------------------------------------------------
.../core/util/path/CarbonTablePath.java | 7 +
.../dataload/TestDataLoadWithFileName.scala | 128 +++++++++++++++++++
.../processing/mdkeygen/file/FileData.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 46 ++++---
.../writer/v1/CarbonFactDataWriterImplV1.java | 2 +-
.../writer/v2/CarbonFactDataWriterImplV2.java | 2 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 13 +-
7 files changed, 171 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
[4/6] incubator-carbondata git commit: Rename fileName to
carbonDataFileTempPath.
Posted by ja...@apache.org.
Rename fileName to carbonDataFileTempPath.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d17f58f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d17f58f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d17f58f1
Branch: refs/heads/master
Commit: d17f58f18ede87aef53e92973bd6e319fa2e31ad
Parents: 84477cc
Author: Yadong Qi <qi...@gmail.com>
Authored: Mon Mar 27 18:48:31 2017 +0800
Committer: Yadong Qi <qi...@gmail.com>
Committed: Mon Mar 27 18:48:31 2017 +0800
----------------------------------------------------------------------
.../processing/mdkeygen/file/FileData.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 27 +++++++++++---------
.../writer/v3/CarbonFactDataWriterImplV3.java | 4 +--
3 files changed, 18 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d17f58f1/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java
index 340a2ab..024a0b9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java
@@ -37,7 +37,7 @@ public class FileData extends FileManager {
}
/**
- * @return Returns the fileName.
+ * @return Returns the carbonDataFileTempPath.
*/
public String getFileName() {
return fileName;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d17f58f1/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index a7dc534..90eab6f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -95,9 +95,9 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
protected List<BlockletInfoColumnar> blockletInfoList;
protected boolean[] isNoDictionary;
/**
- * file name
+ * The temp path of carbonData file used on executor
*/
- protected String fileName;
+ protected String carbonDataFileTempPath;
/**
* The name of carbonData file
@@ -261,10 +261,10 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
protected void updateBlockletFileChannel(long blockletDataSize) throws CarbonDataWriterException {
if ((currentFileSize + blockletDataSize) >= dataBlockSize && currentFileSize != 0) {
// set the current file size to zero
- LOGGER.info("Writing data to file as max file size reached for file: " + fileName
+ LOGGER.info("Writing data to file as max file size reached for file: " + carbonDataFileTempPath
+ " .Data block size: " + currentFileSize);
// write meta data to end of the existing file
- writeBlockletInfoToFile(fileChannel, fileName);
+ writeBlockletInfoToFile(fileChannel, carbonDataFileTempPath);
this.currentFileSize = 0;
blockletInfoList =
new ArrayList<BlockletInfoColumnar>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
@@ -276,7 +276,8 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
// rename carbon data file from in progress status to actual
renameCarbonDataFile();
executorServiceSubmitList.add(executorService
- .submit(new CopyThread(this.fileName.substring(0, this.fileName.lastIndexOf('.')))));
+ .submit(new CopyThread(this.carbonDataFileTempPath
+ .substring(0, this.carbonDataFileTempPath.lastIndexOf('.')))));
// initialize the new channel
initializeWriter();
}
@@ -299,12 +300,12 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
String actualFileNameVal = carbonDataFileName + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
FileData fileData = new FileData(actualFileNameVal, dataWriterVo.getStoreLocation());
dataWriterVo.getFileManager().add(fileData);
- this.fileName = dataWriterVo.getStoreLocation() + File.separator + carbonDataFileName
- + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
+ this.carbonDataFileTempPath = dataWriterVo.getStoreLocation() + File.separator
+ + carbonDataFileName + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
this.fileCount++;
try {
// open channel for new data file
- fileOutputStream = new FileOutputStream(this.fileName, true);
+ fileOutputStream = new FileOutputStream(this.carbonDataFileTempPath, true);
this.fileChannel = fileOutputStream.getChannel();
} catch (FileNotFoundException fileNotFoundException) {
throw new CarbonDataWriterException("Problem while getting the FileChannel for Leaf File",
@@ -420,7 +421,8 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
if (this.blockletInfoList.size() > 0) {
renameCarbonDataFile();
copyCarbonDataFileToCarbonStorePath(
- this.fileName.substring(0, this.fileName.lastIndexOf('.')));
+ this.carbonDataFileTempPath
+ .substring(0, this.carbonDataFileTempPath.lastIndexOf('.')));
try {
writeIndexFile();
} catch (IOException e) {
@@ -490,8 +492,9 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
* @throws CarbonDataWriterException
*/
protected void renameCarbonDataFile() throws CarbonDataWriterException {
- File origFile = new File(this.fileName.substring(0, this.fileName.lastIndexOf('.')));
- File curFile = new File(this.fileName);
+ File origFile = new File(this.carbonDataFileTempPath
+ .substring(0, this.carbonDataFileTempPath.lastIndexOf('.')));
+ File curFile = new File(this.carbonDataFileTempPath);
if (!curFile.renameTo(origFile)) {
throw new CarbonDataWriterException("Problem while renaming the file");
}
@@ -559,7 +562,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
*/
@Override public void writeBlockletInfoToFile() throws CarbonDataWriterException {
if (this.blockletInfoList.size() > 0) {
- writeBlockletInfoToFile(fileChannel, fileName);
+ writeBlockletInfoToFile(fileChannel, carbonDataFileTempPath);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d17f58f1/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 668d0d8..3b7c763 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -540,11 +540,11 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
public void closeWriter() throws CarbonDataWriterException {
if (dataWriterHolder.getNodeHolder().size() > 0) {
writeDataToFile(fileChannel);
- writeBlockletInfoToFile(fileChannel, fileName);
+ writeBlockletInfoToFile(fileChannel, carbonDataFileTempPath);
CarbonUtil.closeStreams(this.fileOutputStream, this.fileChannel);
renameCarbonDataFile();
copyCarbonDataFileToCarbonStorePath(
- this.fileName.substring(0, this.fileName.lastIndexOf('.')));
+ this.carbonDataFileTempPath.substring(0, this.carbonDataFileTempPath.lastIndexOf('.')));
try {
writeIndexFile();
} catch (IOException e) {
[2/6] incubator-carbondata git commit: Store fileName insteads of
filePath in carbonindex.
Posted by ja...@apache.org.
Store fileName insteads of filePath in carbonindex.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/da4d39eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/da4d39eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/da4d39eb
Branch: refs/heads/master
Commit: da4d39eb26d2baff1acf3e41b6991b5239ccef3b
Parents: 4da2a70
Author: Yadong Qi <qi...@gmail.com>
Authored: Sat Mar 25 15:38:23 2017 +0800
Committer: Yadong Qi <qi...@gmail.com>
Committed: Sat Mar 25 15:38:23 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/util/path/CarbonTablePath.java | 7 +++++++
.../dataload/TestDataLoadWithFileName.scala | 15 +++++++++------
.../store/writer/AbstractFactDataWriter.java | 8 +++-----
.../store/writer/v1/CarbonFactDataWriterImplV1.java | 2 +-
.../store/writer/v2/CarbonFactDataWriterImplV2.java | 2 +-
.../store/writer/v3/CarbonFactDataWriterImplV3.java | 2 +-
6 files changed, 22 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 7e37689..3e81fbb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -579,6 +579,13 @@ public class CarbonTablePath extends Path {
}
/**
+ * @return prefix of carbon data
+ */
+ public static String getCarbonDataPrefix() {
+ return DATA_PART_PREFIX;
+ }
+
+ /**
*
* @return carbon data extension
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index df7e71e..be81b2d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -17,7 +17,7 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance.getProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION)
}
- test("Test the file_name in carbonindex with v1 format") {
+ test("Check the file_name in carbonindex with v1 format") {
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "1")
sql("DROP TABLE IF EXISTS test_table_v1")
sql(
@@ -39,12 +39,13 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
assert(indexReader.readIndexHeader().getVersion === 1)
while (indexReader.hasNext) {
val readBlockIndexInfo = indexReader.readBlockIndexInfo()
- assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ assert(readBlockIndexInfo.getFile_name.startsWith(CarbonTablePath.getCarbonDataPrefix))
+ assert(readBlockIndexInfo.getFile_name.endsWith(CarbonTablePath.getCarbonDataExtension))
}
}
}
- test("Test the file_name in carbonindex with v2 format") {
+ test("Check the file_name in carbonindex with v2 format") {
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "2")
sql("DROP TABLE IF EXISTS test_table_v2")
sql(
@@ -66,12 +67,13 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
assert(indexReader.readIndexHeader().getVersion === 2)
while (indexReader.hasNext) {
val readBlockIndexInfo = indexReader.readBlockIndexInfo()
- assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ assert(readBlockIndexInfo.getFile_name.startsWith(CarbonTablePath.getCarbonDataPrefix))
+ assert(readBlockIndexInfo.getFile_name.endsWith(CarbonTablePath.getCarbonDataExtension))
}
}
}
- test("Test the file_name in carbonindex with v3 format") {
+ test("Check the file_name in carbonindex with v3 format") {
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "3")
sql("DROP TABLE IF EXISTS test_table_v3")
sql(
@@ -93,7 +95,8 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
assert(indexReader.readIndexHeader().getVersion === 3)
while (indexReader.hasNext) {
val readBlockIndexInfo = indexReader.readBlockIndexInfo()
- assert(readBlockIndexInfo.getFile_name.startsWith(storeLocation))
+ assert(readBlockIndexInfo.getFile_name.startsWith(CarbonTablePath.getCarbonDataPrefix))
+ assert(readBlockIndexInfo.getFile_name.endsWith(CarbonTablePath.getCarbonDataExtension))
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index dc431d1..f820999 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -100,9 +100,9 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
protected String fileName;
/**
- * The path of carbonData file
+ * The name of carbonData file
*/
- protected String carbonDataFilePath;
+ protected String carbonDataFileName;
/**
* Local cardinality for the segment
@@ -292,7 +292,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
// update the filename with new new sequence
// increment the file sequence counter
initFileCount();
- String carbonDataFileName = carbonTablePath
+ this.carbonDataFileName = carbonTablePath
.getCarbonDataFileName(fileCount, dataWriterVo.getCarbonDataFileAttributes().getTaskId(),
dataWriterVo.getBucketNumber(), dataWriterVo.getTaskExtension(),
dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
@@ -301,8 +301,6 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
dataWriterVo.getFileManager().add(fileData);
this.fileName = dataWriterVo.getStoreLocation() + File.separator + carbonDataFileName
+ CarbonCommonConstants.FILE_INPROGRESS_STATUS;
- this.carbonDataFilePath =
- dataWriterVo.getCarbonDataDirectoryPath() + File.separator + carbonDataFileName;
this.fileCount++;
try {
// open channel for new data file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index ea81083..cf5311c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -373,7 +373,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
FileFooter convertFileMeta = CarbonMetadataUtil
.convertFileFooter(blockletInfoList, localCardinality.length, localCardinality,
thriftColumnSchemaList, dataWriterVo.getSegmentProperties());
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
writer.writeFooter(convertFileMeta, currentPosition);
} catch (IOException e) {
throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
index 562f6d2..ec79186 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
@@ -276,7 +276,7 @@ public class CarbonFactDataWriterImplV2 extends CarbonFactDataWriterImplV1 {
.convertFilterFooter2(blockletInfoList, localCardinality, thriftColumnSchemaList,
dataChunksOffsets, dataChunksLength);
// fill the carbon index details
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
// write the footer
writer.writeFooter(convertFileMeta, currentPosition);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/da4d39eb/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index b264a04..2b138b2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -294,7 +294,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
.convertFileFooterVersion3(blockletMetadata, blockletIndex, localCardinality,
thriftColumnSchemaList.size(), dataWriterVo.getSegmentProperties());
// fill the carbon index details
- fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFilePath, currentPosition);
+ fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
// write the footer
byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
ByteBuffer buffer =