You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/11 05:52:14 UTC
[29/61] [abbrv] git commit: Drill configuration changes
Drill configuration changes
+ Moved bootstrap-storage-plugins.json from "conf/" to "java-exec" jar.
+ Cleared drill-override.conf and added drill-override-example.conf
+ Updated maven assembly descriptor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/41f66160
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/41f66160
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/41f66160
Branch: refs/heads/master
Commit: 41f661609ca716553e9ad1e099f75cf17678397c
Parents: fd1be92
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Fri Jun 6 16:31:31 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Jun 8 19:13:05 2014 -0700
----------------------------------------------------------------------
distribution/src/assemble/bin.xml | 2 +-
.../resources/bootstrap-storage-plugins.json | 70 ---------
.../src/resources/drill-override-example.conf | 146 +++++++++++++++++++
distribution/src/resources/drill-override.conf | 138 +++---------------
.../resources/bootstrap-storage-plugins.json | 70 +++++++++
.../src/main/resources/drill-module.conf | 8 +-
6 files changed, 241 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index ed44c84..b0a07df 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -145,7 +145,7 @@
<outputDirectory>conf</outputDirectory>
</file>
<file>
- <source>src/resources/bootstrap-storage-plugins.json</source>
+ <source>src/resources/drill-override-example.conf</source>
<outputDirectory>conf</outputDirectory>
</file>
</files>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/distribution/src/resources/bootstrap-storage-plugins.json b/distribution/src/resources/bootstrap-storage-plugins.json
deleted file mode 100644
index 3b1cbd0..0000000
--- a/distribution/src/resources/bootstrap-storage-plugins.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
- "storage":{
- dfs: {
- type: "file",
- connection: "file:///",
- workspaces: {
- "root" : {
- location: "/",
- writable: false
- },
- "tmp" : {
- location: "/tmp",
- writable: true,
- storageformat: "csv"
- }
- },
- formats: {
- "psv" : {
- type: "text",
- extensions: [ "tbl" ],
- delimiter: "|"
- },
- "csv" : {
- type: "text",
- extensions: [ "csv" ],
- delimiter: ","
- },
- "tsv" : {
- type: "text",
- extensions: [ "tsv" ],
- delimiter: "\t"
- },
- "parquet" : {
- type: "parquet"
- },
- "json" : {
- type: "json"
- }
- }
- },
- cp: {
- type: "file",
- connection: "classpath:///"
- }
-
- /*,
- hive : {
- type:"hive",
- config :
- {
- "hive.metastore.uris" : "",
- "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true",
- "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
- "fs.default.name" : "file:///",
- "hive.metastore.sasl.enabled" : "false"
- }
- }
- */
-
- /*,
- hbase : {
- type:"hbase",
- config : {
- "hbase.zookeeper.quorum" : "localhost",
- "hbase.zookeeper.property.clientPort" : 2181
- }
- }
- */
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/resources/drill-override-example.conf
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/resources/drill-override-example.conf
new file mode 100644
index 0000000..9d87f76
--- /dev/null
+++ b/distribution/src/resources/drill-override-example.conf
@@ -0,0 +1,146 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file tells Drill to consider this module when class path scanning.
+# This file can also include any supplementary configuration information.
+# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl"
+
+drill.exec: {
+ cluster-id: "drillbits1"
+ rpc: {
+ user: {
+ server: {
+ port: 31010
+ threads: 1
+ }
+ client: {
+ threads: 1
+ }
+ },
+ bit: {
+ server: {
+ port : 31011,
+ retry:{
+ count: 7200,
+ delay: 500
+ },
+ threads: 1
+ }
+ },
+ use.ip : false
+ },
+ operator: {
+ packages += "org.apache.drill.exec.physical.config"
+ },
+ optimizer: {
+ implementation: "org.apache.drill.exec.opt.IdentityOptimizer"
+ },
+ functions: ["org.apache.drill.expr.fn.impl"],
+ storage: {
+ packages += "org.apache.drill.exec.store",
+ file: {
+ text: {
+ buffer.size: 262144,
+ batch.size: 4000
+ },
+ partition.column.label: "dir"
+ }
+ },
+ metrics : {
+ context: "drillbit",
+ jmx: {
+ enabled : true
+ },
+ log: {
+ enabled : false,
+ interval : 60
+ }
+ },
+ zk: {
+ connect: "localhost:2181",
+ root: "drill",
+ refresh: 500,
+ timeout: 5000,
+ retry: {
+ count: 7200,
+ delay: 500
+ }
+ },
+ http: {
+ enabled: true,
+ port: 8047
+ },
+ functions: ["org.apache.drill.expr.fn.impl"],
+ network: {
+ start: 35000
+ },
+ work: {
+ max.width.per.endpoint: 5,
+ global.max.width: 100,
+ affinity.factor: 1.2,
+ executor.threads: 4
+ },
+ sys.store.provider: {
+ class: "org.apache.drill.exec.store.sys.local.LocalPStoreProvider",
+ local: {
+ path: "/tmp/drill",
+ write: true
+ }
+ }
+ trace: {
+ directory: "/tmp/drill-trace",
+ filesystem: "file:///"
+ },
+ tmp: {
+ directories: ["/tmp/drill"],
+ filesystem: "drill-local:///"
+ },
+ buffer:{
+ impl: "org.apache.drill.exec.work.batch.UnlimitedRawBatchBuffer",
+ size: "100",
+ spooling: {
+ delete: false,
+ size: 100000000
+ }
+ },
+ cache.hazel.subnets: ["*.*.*.*"],
+ sort: {
+ purge.threshold : 100,
+ external: {
+ batch.size : 4000,
+ spill: {
+ batch.size : 4000,
+ group.size : 100,
+ threshold : 200,
+ directories : [ "/tmp/drill/spill" ],
+ fs : "file:///"
+ }
+ }
+ },
+ memory: {
+ top.max: 1000000000000,
+ operator: {
+ max: 20000000000,
+ initial: 10000000
+ },
+ fragment: {
+ max: 20000000000,
+ initial: 20000000
+ }
+ },
+ debug.error_on_leak: true
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/resources/drill-override.conf
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-override.conf b/distribution/src/resources/drill-override.conf
index 472296b..b484ea3 100644
--- a/distribution/src/resources/drill-override.conf
+++ b/distribution/src/resources/drill-override.conf
@@ -1,123 +1,25 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
-// This file tells Drill to consider this module when class path scanning.
-// This file can also include any supplementary configuration information.
-// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+# This file tells Drill to consider this module when class path scanning.
+# This file can also include any supplementary configuration information.
+# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
-drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl"
+# See 'drill-override-example.conf' for example configurations
drill.exec: {
- cluster-id: "drillbits1"
- rpc: {
- user: {
- server: {
- port: 31010
- threads: 1
- }
- client: {
- threads: 1
- }
- },
- bit: {
- server: {
- port : 31011,
- retry:{
- count: 7200,
- delay: 500
- },
- threads: 1
- }
- },
- use.ip : false
- },
- operator: {
- packages += "org.apache.drill.exec.physical.config"
- },
- optimizer: {
- implementation: "org.apache.drill.exec.opt.IdentityOptimizer"
- },
- functions: ["org.apache.drill.expr.fn.impl"],
- storage: {
- packages += "org.apache.drill.exec.store",
- file: {
- text: {
- buffer.size: 262144,
- batch.size: 4000
- },
- partition.column.label: "dir"
- }
- },
- metrics : {
- context: "drillbit",
- jmx: {
- enabled : true
- },
- log: {
- enabled : false,
- interval : 60
- }
- },
- zk: {
- connect: "localhost:2181",
- root: "/drill",
- refresh: 500,
- timeout: 5000,
- retry: {
- count: 7200,
- delay: 500
- }
- },
- functions: ["org.apache.drill.expr.fn.impl"],
- network: {
- start: 35000
- },
- work: {
- max.width.per.endpoint: 5,
- global.max.width: 100,
- affinity.factor: 1.2,
- executor.threads: 4
- },
- trace: {
- directory: "/tmp/drill-trace",
- filesystem: "file:///"
- },
- tmp: {
- directories: ["/tmp/drill"],
- filesystem: "drill-local:///"
- },
- buffer:{
- impl: "org.apache.drill.exec.work.batch.UnlimitedRawBatchBuffer",
- size: "100",
- spooling: {
- delete: false,
- size: 100000000
- }
- },
- cache.hazel.subnets: ["*.*.*.*"],
- sort: {
- purge.threshold : 100,
- external: {
- batch.size : 4000,
- spill: {
- batch.size : 4000,
- group.size : 100,
- threshold : 200,
- directories : [ "/tmp/drill/spill" ],
- fs : "file:///"
- }
- }
- }
+ cluster-id: "drillbits1",
+ zk.connect: "localhost:2181"
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
new file mode 100644
index 0000000..3b1cbd0
--- /dev/null
+++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -0,0 +1,70 @@
+{
+ "storage":{
+ dfs: {
+ type: "file",
+ connection: "file:///",
+ workspaces: {
+ "root" : {
+ location: "/",
+ writable: false
+ },
+ "tmp" : {
+ location: "/tmp",
+ writable: true,
+ storageformat: "csv"
+ }
+ },
+ formats: {
+ "psv" : {
+ type: "text",
+ extensions: [ "tbl" ],
+ delimiter: "|"
+ },
+ "csv" : {
+ type: "text",
+ extensions: [ "csv" ],
+ delimiter: ","
+ },
+ "tsv" : {
+ type: "text",
+ extensions: [ "tsv" ],
+ delimiter: "\t"
+ },
+ "parquet" : {
+ type: "parquet"
+ },
+ "json" : {
+ type: "json"
+ }
+ }
+ },
+ cp: {
+ type: "file",
+ connection: "classpath:///"
+ }
+
+ /*,
+ hive : {
+ type:"hive",
+ config :
+ {
+ "hive.metastore.uris" : "",
+ "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true",
+ "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
+ "fs.default.name" : "file:///",
+ "hive.metastore.sasl.enabled" : "false"
+ }
+ }
+ */
+
+ /*,
+ hbase : {
+ type:"hbase",
+ config : {
+ "hbase.zookeeper.quorum" : "localhost",
+ "hbase.zookeeper.property.clientPort" : 2181
+ }
+ }
+ */
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 9ce22c7..7f399b2 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -60,7 +60,7 @@ drill.exec: {
partition.column.label: "dir"
}
},
- metrics : {
+ metrics : {
context: "drillbit",
jmx: {
enabled : true
@@ -72,13 +72,13 @@ drill.exec: {
},
zk: {
connect: "localhost:2181",
- root: "drill/happy",
+ root: "drill",
refresh: 500,
timeout: 5000,
retry: {
count: 7200,
delay: 500
- }
+ }
},
http: {
enabled: true,
@@ -111,7 +111,7 @@ drill.exec: {
},
buffer:{
impl: "org.apache.drill.exec.work.batch.UnlimitedRawBatchBuffer",
- size: "20",
+ size: "100",
spooling: {
delete: false,
size: 100000000