You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/09/14 09:20:22 UTC
[29/54] [abbrv] carbondata git commit: [CARBONDATA-1451] Removing
configuration for number_of_rows_per_blocklet_column_page
[CARBONDATA-1451] Removing configuration for number_of_rows_per_blocklet_column_page
This closes #1334
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/435ea26e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/435ea26e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/435ea26e
Branch: refs/heads/streaming_ingest
Commit: 435ea26eb8864db44b2e246d7f47a416d2dfdbd4
Parents: a5483e8
Author: dhatchayani <dh...@gmail.com>
Authored: Wed Sep 6 15:25:33 2017 +0530
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Sat Sep 9 18:45:57 2017 +0530
----------------------------------------------------------------------
.../constants/CarbonV3DataFormatConstants.java | 18 +----------
.../carbondata/core/scan/filter/FilterUtil.java | 3 +-
.../scan/scanner/AbstractBlockletScanner.java | 12 +++-----
.../carbondata/core/util/CarbonProperties.java | 32 --------------------
.../store/CarbonFactDataHandlerColumnar.java | 4 +--
5 files changed, 8 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
index edc7b9a..e888986 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonV3DataFormatConstants.java
@@ -61,24 +61,8 @@ public interface CarbonV3DataFormatConstants {
short NUMBER_OF_COLUMN_TO_READ_IN_IO_MIN = 1;
/**
- * number of rows per blocklet column page
- */
- @CarbonProperty
- String NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE = "number.of.rows.per.blocklet.column.page";
-
- /**
* number of rows per blocklet column page default value
*/
- String NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT = "32000";
-
- /**
- * number of rows per blocklet column page max value
- */
- short NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_MAX = 32000;
-
- /**
- * number of rows per blocklet column page min value
- */
- short NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_MIN = 8000;
+ short NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT = 32000;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 78c1afd..01e1cfa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1602,8 +1602,7 @@ public final class FilterUtil {
public static BitSetGroup createBitSetGroupWithDefaultValue(int pageCount, int totalRowCount,
boolean defaultValue) {
BitSetGroup bitSetGroup = new BitSetGroup(pageCount);
- int numberOfRows = Integer
- .parseInt(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
+ int numberOfRows = CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
int pagesTobeFullFilled = totalRowCount / numberOfRows;
int rowCountForLastPage = totalRowCount % numberOfRows;
for (int i = 0; i < pagesTobeFullFilled; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 0e1ede8..1e4becd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -32,17 +32,12 @@ import org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.util.CarbonProperties;
/**
* Blocklet scanner class to process the block
*/
public abstract class AbstractBlockletScanner implements BlockletScanner {
- private static final int NUMBER_OF_ROWS_PER_PAGE = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE,
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT));
-
/**
* block execution info
*/
@@ -121,9 +116,12 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
if (numberOfRows == null) {
numberOfRows = new int[blocksChunkHolder.getDataBlock().numberOfPages()];
for (int i = 0; i < numberOfRows.length; i++) {
- numberOfRows[i] = NUMBER_OF_ROWS_PER_PAGE;
+ numberOfRows[i] =
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
}
- int lastPageSize = blocksChunkHolder.getDataBlock().nodeSize() % NUMBER_OF_ROWS_PER_PAGE;
+ int lastPageSize = blocksChunkHolder.getDataBlock().nodeSize()
+ % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ ;
if (lastPageSize > 0) {
numberOfRows[numberOfRows.length - 1] = lastPageSize;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index dd416ae..4e9c21a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -100,7 +100,6 @@ public final class CarbonProperties {
validatePrefetchBufferSize();
validateBlockletGroupSizeInMB();
validateNumberOfColumnPerIORead();
- validateNumberOfRowsPerBlockletColumnPage();
validateEnableUnsafeSort();
validateCustomBlockDistribution();
validateEnableVectorReader();
@@ -313,37 +312,6 @@ public final class CarbonProperties {
}
/**
- * This method validates the number of column read in one IO
- */
- private void validateNumberOfRowsPerBlockletColumnPage() {
- String numberOfRowsPerBlockletColumnPageString = carbonProperties
- .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE,
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
- try {
- short numberOfRowsPerBlockletColumnPage =
- Short.parseShort(numberOfRowsPerBlockletColumnPageString);
- if (numberOfRowsPerBlockletColumnPage
- < CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_MIN
- || numberOfRowsPerBlockletColumnPage
- > CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_MAX) {
- LOGGER.info("The Number Of rows per blocklet column pages value \""
- + numberOfRowsPerBlockletColumnPageString + "\" is invalid. Using the default value \""
- + CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
- carbonProperties
- .setProperty(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE,
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The Number Of rows per blocklet column pages value \""
- + numberOfRowsPerBlockletColumnPageString + "\" is invalid. Using the default value \""
- + CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
- carbonProperties
- .setProperty(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE,
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
- }
- }
-
- /**
* This method validates the blocklet size
*/
private void validateBlockletSize() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/435ea26e/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 41005dd..c4a5fc5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -469,9 +469,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
.getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
if (version == ColumnarFormatVersion.V3) {
- this.pageSize = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE,
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT));
+ this.pageSize = CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
}
LOGGER.info("Number of rows per column blocklet " + pageSize);
dataRows = new ArrayList<>(this.pageSize);