You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2018/03/31 08:44:21 UTC
carbondata git commit: [CARBONDATA-2295] Added property for
UnsafeMemory configuration
Repository: carbondata
Updated Branches:
refs/heads/master f187856e3 -> fca960e37
[CARBONDATA-2295] Added property for UnsafeMemory configuration
Added property for setting up the unsafe memory limit in MB
This closes #2114
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fca960e3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fca960e3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fca960e3
Branch: refs/heads/master
Commit: fca960e372981e9dbb0cd048a4b55f3d2c9839dc
Parents: f187856
Author: Bhavya <bh...@knoldus.com>
Authored: Thu Mar 29 20:33:34 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Sat Mar 31 16:44:01 2018 +0800
----------------------------------------------------------------------
integration/presto/README.md | 4 ++++
.../apache/carbondata/presto/impl/CarbonTableConfig.java | 11 +++++++++++
.../apache/carbondata/presto/impl/CarbonTableReader.java | 7 ++++++-
.../apache/carbondata/presto/server/PrestoServer.scala | 3 ++-
4 files changed, 23 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/README.md
----------------------------------------------------------------------
diff --git a/integration/presto/README.md b/integration/presto/README.md
index d264954..e1d4631 100644
--- a/integration/presto/README.md
+++ b/integration/presto/README.md
@@ -73,10 +73,14 @@ Please follow the below steps to query carbondata in presto
```
connector.name=carbondata
carbondata-store={schema-store-path}
+ carbon.unsafe.working.memory.in.mb={value}
```
Replace the schema-store-path with the absolute path of the parent directory of the schema.
For example, if you have a schema named 'default' stored in hdfs://namenode:9000/test/carbondata/,
Then set carbondata-store=hdfs://namenode:9000/test/carbondata
+
+ carbon.unsafe.working.memory.in.mb property defines the limit for Unsafe Memory usage in Mega Bytes. Replace the value for it
+ as per your need, if your tables are big you can increase the unsafe memory. The default value is 512 MB.
If you updated the jar balls or configuration files, make sure you have dispatched them
to all the presto nodes and restarted the presto servers on the nodes. The updates will not take effect before restarting.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
index 677cefd..73f9d2c 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
@@ -30,6 +30,7 @@ public class CarbonTableConfig {
private String dbPath;
private String tablePath;
private String storePath;
+ private String unsafeMemoryInMb;
@NotNull public String getDbPath() {
return dbPath;
@@ -57,4 +58,14 @@ public class CarbonTableConfig {
this.storePath = storePath;
return this;
}
+
+ public String getUnsafeMemoryInMb() {
+ return unsafeMemoryInMb;
+ }
+
+ @Config("carbon.unsafe.working.memory.in.mb")
+ public CarbonTableConfig setUnsafeMemoryInMb(String unsafeMemoryInMb) {
+ this.unsafeMemoryInMb = unsafeMemoryInMb;
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 09389f8..78dbbec 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -42,6 +43,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
@@ -366,7 +368,10 @@ public class CarbonTableReader {
public List<CarbonLocalInputSplit> getInputSplits2(CarbonTableCacheModel tableCacheModel,
Expression filters) {
List<CarbonLocalInputSplit> result = new ArrayList<>();
-
+ if(config.getUnsafeMemoryInMb() != null) {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
+ config.getUnsafeMemoryInMb());
+ }
CarbonTable carbonTable = tableCacheModel.carbonTable;
TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo();
Configuration config = new Configuration();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
index d6d3617..127c4c9 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
@@ -69,7 +69,8 @@ object PrestoServer {
Try {
queryRunner.installPlugin(new CarbondataPlugin)
val carbonProperties = ImmutableMap.builder[String, String]
- .put("carbondata-store", carbonStorePath).build
+ .put("carbondata-store", carbonStorePath)
+ .put("carbon.unsafe.working.memory.in.mb", "512").build
// CreateCatalog will create a catalog for CarbonData in etc/catalog.
queryRunner.createCatalog(CARBONDATA_CATALOG, CARBONDATA_CONNECTOR, carbonProperties)