You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by su...@apache.org on 2017/07/03 17:09:22 UTC
[1/7] incubator-trafodion git commit: [TRAFODION-2663] Simplify HBase
config settings in installers
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 5f76a579b -> 3e5dd571c
[TRAFODION-2663] Simplify HBase config settings in installers
Remove settings that are now in per-table trafodion-site.xml config.
For Ambari, 2.2 service_advisor over-rides the 2.1 version. This allows
possibility of single plug-in to manage 2.1 & 2.2 installs, though this is not
tested.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/aaa5abf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/aaa5abf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/aaa5abf4
Branch: refs/heads/master
Commit: aaa5abf493baa04d11e482f752bf4d6b863f7279
Parents: 1555d5c
Author: Steve Varnau <st...@esgyn.com>
Authored: Thu Jun 22 19:30:53 2017 +0000
Committer: Steve Varnau <st...@esgyn.com>
Committed: Thu Jun 22 19:40:33 2017 +0000
----------------------------------------------------------------------
install/ambari-installer/Makefile | 2 +-
.../TRAFODION/2.2.0/service_advisor.py | 254 +++++++++++++++++++
install/python-installer/configs/mod_cfgs.json | 5 -
install/python-installer/scripts/hadoop_mods.py | 4 -
4 files changed, 255 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/aaa5abf4/install/ambari-installer/Makefile
----------------------------------------------------------------------
diff --git a/install/ambari-installer/Makefile b/install/ambari-installer/Makefile
index 9edb405..0c6e7c6 100644
--- a/install/ambari-installer/Makefile
+++ b/install/ambari-installer/Makefile
@@ -34,7 +34,7 @@ all: rpmbuild
# need repoinfo file per release
# traf_ambari needs to support multiple releases so ambari can
# select trafodion version for given HDP stack
-REPO_LIST= $(TRAFODION_VER) # for now, list is current release only
+REPO_LIST= 2.1.0 $(TRAFODION_VER)
$(SOURCEDIR)/ambari_rpm.tar.gz: mpack-install/LICENSE mpack-install/NOTICE mpack-install/DISCLAIMER repofiles traf-mpack/mpack.json
rm -rf $(RPMROOT)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/aaa5abf4/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py b/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py
new file mode 100644
index 0000000..b58b6c4
--- /dev/null
+++ b/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import imp
+import traceback
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class TRAFODION21ServiceAdvisor(service_advisor.DefaultStackAdvisor):
+
+ def colocateService(self, hostsComponentsMap, serviceComponents):
+ # Co-locate TRAF_NODE with HBase REGIONSERVERS, if no hosts allocated
+ traf_node = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "TRAF_NODE"][0]
+ if not self.isComponentHostsPopulated(traf_node):
+ for hostName in hostsComponentsMap.keys():
+ hostComponents = hostsComponentsMap[hostName]
+ if {"name": "HBASE_REGIONSERVER"} in hostComponents and {"name": "TRAF_NODE"} not in hostComponents:
+ hostsComponentsMap[hostName].append( { "name": "TRAF_NODE" } )
+ if {"name": "HBASE_REGIONSERVER"} not in hostComponents and {"name": "TRAF_NODE"} in hostComponents:
+ hostComponents.remove({"name": "TRAF_NODE"})
+
+ # Place TRAF_DCS_* on ZOOKEEPER_SERVER nodes by default
+ # DCS_PRIME and DCS_SECOND are exclusive
+ pplaced = False
+ for hostName in hostsComponentsMap.keys():
+ hostComponents = hostsComponentsMap[hostName]
+ if {"name": "ZOOKEEPER_SERVER"} in hostComponents:
+ if {"name": "TRAF_DCS_PRIME"} in hostComponents:
+ if pplaced:
+ hostComponents.remove({"name": "TRAF_DCS_PRIME"}) # only one allowed
+ elif {"name": "TRAF_DCS_SECOND"} in hostComponents:
+ pplaced = True
+ hostComponents.remove({"name": "TRAF_DCS_SECOND"})
+ else:
+ pplaced = True
+ else:
+ if pplaced and {"name": "TRAF_DCS_SECOND"} not in hostComponents:
+ hostsComponentsMap[hostName].append( { "name": "TRAF_DCS_SECOND" } )
+ elif not pplaced:
+ hostsComponentsMap[hostName].append( { "name": "TRAF_DCS_PRIME" } )
+ pplaced = True
+ if {"name": "TRAF_DCS_SECOND"} in hostComponents:
+ hostComponents.remove({"name": "TRAF_DCS_SECOND"})
+ else:
+ if {"name": "TRAF_DCS_PRIME"} in hostComponents:
+ hostComponents.remove({"name": "TRAF_DCS_PRIME"})
+ if {"name": "TRAF_DCS_SECOND"} in hostComponents:
+ hostComponents.remove({"name": "TRAF_DCS_SECOND"})
+
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+
+ trafNodeHosts = self.getHosts(componentsList, "TRAF_NODE")
+ regionHosts = self.getHosts(componentsList, "HBASE_REGIONSERVER")
+ hiveHosts = self.getHosts(componentsList, "HIVE_CLIENT")
+ dataHosts = self.getHosts(componentsList, "DATANODE")
+ trafMasHosts = self.getHosts(componentsList, "TRAF_MASTER")
+
+ items = []
+
+ # Generate WARNING if any TRAF_NODE is not on TRAF_MASTER
+ if not set(trafMasHosts).issubset(set(trafNodeHosts)):
+ hostsString = ', '.join(trafMasHosts)
+ message = "Trafodion Master should also be a Trafodion Node: {0}".format(hostsString)
+ items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'TRAF_NODE' } )
+
+ # Generate WARNING if any TRAF_NODE is not colocated with a DATANODE
+ mismatchHosts = sorted(set(trafNodeHosts).symmetric_difference(set(dataHosts)))
+ if len(mismatchHosts) > 0:
+ hostsString = ', '.join(mismatchHosts)
+ message = "Trafodion Nodes should be co-located with HDFS Data Nodes. " \
+ "{0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString)
+ items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'TRAF_NODE' } )
+
+ # Generate WARNING if any TRAF_NODE is not colocated with a HBASE_REGIONSERVER
+ mismatchHosts = sorted(set(trafNodeHosts).symmetric_difference(set(regionHosts)))
+ if len(mismatchHosts) > 0:
+ hostsString = ', '.join(mismatchHosts)
+ message = "Trafodion Nodes should be co-located with HBase Region Servers. " \
+ "{0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString)
+ items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'TRAF_NODE' } )
+
+ # Generate WARNING if any TRAF_NODE is not colocated with a HIVE_CLIENT
+ mismatchHosts = sorted(set(trafNodeHosts).difference(set(hiveHosts)))
+ if len(mismatchHosts) > 0:
+ hostsString = ', '.join(mismatchHosts)
+ message = "Hive Client should be installed on all Trafodion Nodes. " \
+ "{0} host(s) do not satisfy recommendation: {1}".format(len(mismatchHosts), hostsString)
+ items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'TRAF_NODE' } )
+
+ return items
+
+ def getServiceConfigurationRecommendations(self, configurations, clusterSummary, services, hosts):
+
+ # Update HBASE properties in hbase-site
+ if "hbase-site" in services["configurations"]:
+ hbase_site = services["configurations"]["hbase-site"]["properties"]
+ putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
+
+ for property, desired_value in self.getHbaseSiteDesiredValues().iteritems():
+ if property not in hbase_site:
+ putHbaseSiteProperty(property, desired_value)
+ elif hbase_site[property] != desired_value:
+ if property == "hbase.bulkload.staging.dir":
+ # don't modify unless it is empty
+ if hbase_site[property] == '':
+ putHbaseSiteProperty(property, desired_value)
+ elif property == "hbase.coprocessor.region.classes":
+ if hbase_site[property].find(desired_value) == -1:
+ # append to classpath
+ if hbase_site[property] == '':
+ putHbaseSiteProperty(property, desired_value)
+ else:
+ putHbaseSiteProperty(property, hbase_site[property] + ',' + desired_value)
+ # for any other property, modify regardless
+ else:
+ putHbaseSiteProperty(property, desired_value)
+
+ # Update HDFS properties in hdfs-site
+ if "hdfs-site" in services["configurations"]:
+ hdfs_site = services["configurations"]["hdfs-site"]["properties"]
+ putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services)
+
+ for property, desired_value in self.getHdfsSiteDesiredValues().iteritems():
+ if property not in hdfs_site or hdfs_site[property] != desired_value:
+ putHdfsSiteProperty(property, desired_value)
+
+ # Update ZOOKEEPER properties in zoo.cfg
+ if "zoo.cfg" in services["configurations"]:
+ zoo_cfg = services["configurations"]["zoo.cfg"]["properties"]
+ putZooCfgProperty = self.putProperty(configurations, "zoo.cfg", services)
+
+ for property, desired_value in self.getZooCfgDesiredValues().iteritems():
+ if property not in zoo_cfg or zoo_cfg[property] != desired_value:
+ putZooCfgProperty(property, desired_value)
+
+
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
+ items = []
+
+ if "dcs-site" in services["configurations"]:
+ val_items = []
+ cfg = configurations["dcs-site"]["properties"]
+ if cfg["dcs.master.floating.ip"] == "true" and cfg["dcs.master.floating.ip.external.ip.address"] == "":
+ message = "DCS High Availability requires a Floating IP address"
+ val_items.append({"config-name": "dcs.master.floating.ip.external.ip.address", "item": self.getErrorItem(message)})
+ items.extend(self.toConfigurationValidationProblems(val_items, "dcs-site"))
+
+ if "trafodion-env" in services["configurations"]:
+ val_items = []
+ cfg = configurations["trafodion-env"]["properties"]
+ if cfg["traf.ldap.enabled"] == "YES" and cfg["traf.ldap.hosts"] == "":
+ message = "LDAP authentication requires one or more LDAP servers"
+ val_items.append({"config-name": "traf.ldap.hosts", "item": self.getErrorItem(message)})
+ if cfg["traf.ldap.encrypt"] != "0" and cfg["traf.ldap.certpath"] == "":
+ message = "LDAP encryption requires a certificate file"
+ val_items.append({"config-name": "traf.ldap.certpath", "item": self.getErrorItem(message)})
+ items.extend(self.toConfigurationValidationProblems(val_items, "trafodion-env"))
+
+ if "hbase-site" in services["configurations"]:
+ val_items = []
+ cfg = configurations["hbase-site"]["properties"]
+ for property, desired_value in self.getHbaseSiteDesiredValues().iteritems():
+ if property not in cfg:
+ message = "Trafodion recommends value of " + desired_value
+ val_items.append({"config-name": property, "item": self.getWarnItem(message)})
+ # use any staging dir, but complain if there is none
+ elif property == "hbase.bulkload.staging.dir":
+ if cfg[property] == '':
+ message = "Trafodion recommends value of " + desired_value
+ val_items.append({"config-name": property, "item": self.getWarnItem(message)})
+ # check for substring of classpath
+ elif property == "hbase.coprocessor.region.classes":
+ if cfg[property].find(desired_value) == -1:
+ message = "Trafodion requires addition to region classpath. Go to HBase Advanced Configs, hbase-site section and select recommended value for this property."
+ val_items.append({"config-name": property, "item": self.getErrorItem(message)})
+ elif cfg[property] != desired_value:
+ message = "Trafodion recommends value of " + desired_value
+ val_items.append({"config-name": property, "item": self.getWarnItem(message)})
+ items.extend(self.toConfigurationValidationProblems(val_items, "hbase-site"))
+
+ if "hdfs-site" in services["configurations"]:
+ val_items = []
+ cfg = configurations["hdfs-site"]["properties"]
+ for property, desired_value in self.getHdfsSiteDesiredValues().iteritems():
+ if property not in cfg or cfg[property] != desired_value:
+ message = "Trafodion recommends value of " + desired_value
+ val_items.append({"config-name": property, "item": self.getWarnItem(message)})
+ items.extend(self.toConfigurationValidationProblems(val_items, "hdfs-site"))
+
+ if "zoo.cfg" in services["configurations"]:
+ val_items = []
+ cfg = configurations["zoo.cfg"]["properties"]
+ for property, desired_value in self.getZooCfgDesiredValues().iteritems():
+ if property not in cfg or cfg[property] != desired_value:
+ message = "Trafodion recommends value of " + desired_value
+ val_items.append({"config-name": property, "item": self.getWarnItem(message)})
+ items.extend(self.toConfigurationValidationProblems(val_items, "zoo.cfg"))
+
+ return items
+
+ ##### Desired values in other service configs
+ def getHbaseSiteDesiredValues(self):
+ desired = {
+ "hbase.snapshot.master.timeoutMillis": "600000",
+ "hbase.hregion.impl": "org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion",
+ "hbase.regionserver.region.split.policy": "org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy",
+ "hbase.snapshot.enabled": "true",
+ "hbase.bulkload.staging.dir": "/apps/hbase/staging",
+ "hbase.regionserver.region.transactional.tlog": "true",
+ "hbase.snapshot.region.timeout": "600000",
+ "hbase.client.keyvalue.maxsize": "0"
+ }
+ return desired
+
+ def getHdfsSiteDesiredValues(self):
+ desired = {
+ "dfs.namenode.acls.enabled": "true"
+ }
+ return desired
+
+ def getZooCfgDesiredValues(self):
+ desired = {
+ "maxClientCnxns": "0"
+ }
+ return desired
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/aaa5abf4/install/python-installer/configs/mod_cfgs.json
----------------------------------------------------------------------
diff --git a/install/python-installer/configs/mod_cfgs.json b/install/python-installer/configs/mod_cfgs.json
index df90810..9234a52 100644
--- a/install/python-installer/configs/mod_cfgs.json
+++ b/install/python-installer/configs/mod_cfgs.json
@@ -2,14 +2,12 @@
"MOD_CFGS": {
"hbase-site": {
"hbase.snapshot.master.timeoutMillis": "600000",
- "hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionObserver,org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionEndpoint,org.apache.hadoop.hbase.coprocessor.AggregateImplementation",
"hbase.hregion.impl": "org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion",
"hbase.regionserver.region.split.policy": "org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy",
"hbase.snapshot.enabled": "true",
"hbase.bulkload.staging.dir": "/hbase-staging",
"hbase.regionserver.region.transactional.tlog": "true",
"hbase.snapshot.region.timeout": "600000",
- "hbase.client.scanner.timeout.period": "600000",
"hbase.client.keyvalue.maxsize": "0"
},
"hdfs-site": { "dfs.namenode.acls.enabled": "true" },
@@ -28,9 +26,6 @@
"HBASE_RS_CONFIG": {
"items" : [ {
- "name" : "hbase_coprocessor_region_classes",
- "value" : "org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionObserver,org.apache.hadoop.hbase.coprocessor.transactional.TrxRegionEndpoint,org.apache.hadoop.hbase.coprocessor.AggregateImplementation"
- }, {
"name" : "hbase_regionserver_lease_period",
"value" : "600000"
}, {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/aaa5abf4/install/python-installer/scripts/hadoop_mods.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/hadoop_mods.py b/install/python-installer/scripts/hadoop_mods.py
index e2ca95a..51ca635 100755
--- a/install/python-installer/scripts/hadoop_mods.py
+++ b/install/python-installer/scripts/hadoop_mods.py
@@ -122,10 +122,6 @@ class HDPMod(object):
}
self.p.post('%s/config_groups' % cluster_url, hbase_config_group)
- if dbcfgs['secure_hadoop'].upper() == 'Y':
- secure_cfgs = ',org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.token.TokenProvider'
- MOD_CFGS['hbase-site']['hbase.coprocessor.region.classes'] += secure_cfgs
-
for config_type in MOD_CFGS.keys():
desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
[6/7] incubator-trafodion git commit: [TRAFODION-2663] move hbase
client setting to trafodion-site.xml
Posted by su...@apache.org.
[TRAFODION-2663] move hbase client setting to trafodion-site.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/35f92995
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/35f92995
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/35f92995
Branch: refs/heads/master
Commit: 35f929954c11ab92df230d39f84507369e799c0c
Parents: c55b2fd
Author: Steve Varnau <st...@esgyn.com>
Authored: Wed Jun 28 21:00:58 2017 +0000
Committer: Steve Varnau <st...@esgyn.com>
Committed: Wed Jun 28 21:00:58 2017 +0000
----------------------------------------------------------------------
core/sqf/conf/trafodion-site.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/35f92995/core/sqf/conf/trafodion-site.xml
----------------------------------------------------------------------
diff --git a/core/sqf/conf/trafodion-site.xml b/core/sqf/conf/trafodion-site.xml
index fc04716..7a16f2a 100644
--- a/core/sqf/conf/trafodion-site.xml
+++ b/core/sqf/conf/trafodion-site.xml
@@ -35,4 +35,8 @@
<name>hbase.client.scanner.timeout.period</name>
<value>3600000</value>
</property>
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>0</value>
+ </property>
</configuration>
[2/7] incubator-trafodion git commit: [TRAFODION-2663] Add re-check
in hbcheck
Posted by su...@apache.org.
[TRAFODION-2663] Add re-check in hbcheck
Simplifying the HBase config seems to have changed the timing of
HBase start-up on HDP nodes. We are trying to talk to HBase while
master is still initializing. I have added re-tries in hbcheck
to compensate.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/98620b76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/98620b76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/98620b76
Branch: refs/heads/master
Commit: 98620b76e7625f8ec9c73b64a0d64e6d9f5eb66b
Parents: aaa5abf
Author: Steve Varnau <st...@esgyn.com>
Authored: Tue Jun 27 18:55:17 2017 +0000
Committer: Steve Varnau <st...@esgyn.com>
Committed: Tue Jun 27 18:55:17 2017 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/hbcheck | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/98620b76/core/sqf/sql/scripts/hbcheck
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/hbcheck b/core/sqf/sql/scripts/hbcheck
index ed50d5c..79dbd97 100755
--- a/core/sqf/sql/scripts/hbcheck
+++ b/core/sqf/sql/scripts/hbcheck
@@ -32,4 +32,15 @@ fi
mkdir -p $TRAF_HOME/logs
lv_stderr_file="$TRAF_HOME/logs/hbcheck.log"
echo "Stderr being written to the file: ${lv_stderr_file}"
-$JAVA_HOME/bin/java -Xmx512m org.trafodion.dtm.hbstatus $* 2>${lv_stderr_file}
+for interval in 5 10 15 30
+do
+ $JAVA_HOME/bin/java -Xmx512m org.trafodion.dtm.hbstatus $* 2>>${lv_stderr_file}
+ if [[ $? == 0 ]]
+ then
+ exit 0
+ fi
+ echo "HBase not available. Waiting $interval seconds."
+ sleep $interval
+done
+$JAVA_HOME/bin/java -Xmx512m org.trafodion.dtm.hbstatus $* 2>>${lv_stderr_file}
+exit $?
[4/7] incubator-trafodion git commit: [TRAFODION-2263] include
ambari-installer keyvalue change
Posted by su...@apache.org.
[TRAFODION-2263] include ambari-installer keyvalue change
Missed one file in prior commit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d1e3cec9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d1e3cec9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d1e3cec9
Branch: refs/heads/master
Commit: d1e3cec9ff3a83da05176fced98423043c3aad8e
Parents: c0e3951
Author: Steve Varnau <st...@esgyn.com>
Authored: Wed Jun 28 19:45:26 2017 +0000
Committer: Steve Varnau <st...@esgyn.com>
Committed: Wed Jun 28 19:45:26 2017 +0000
----------------------------------------------------------------------
.../traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d1e3cec9/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py b/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py
index b58b6c4..17dfda3 100644
--- a/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py
+++ b/install/ambari-installer/traf-mpack/custom-services/TRAFODION/2.2.0/service_advisor.py
@@ -236,8 +236,7 @@ class TRAFODION21ServiceAdvisor(service_advisor.DefaultStackAdvisor):
"hbase.snapshot.enabled": "true",
"hbase.bulkload.staging.dir": "/apps/hbase/staging",
"hbase.regionserver.region.transactional.tlog": "true",
- "hbase.snapshot.region.timeout": "600000",
- "hbase.client.keyvalue.maxsize": "0"
+ "hbase.snapshot.region.timeout": "600000"
}
return desired
[5/7] incubator-trafodion git commit: Merge remote branch
'origin/master' into traf2663
Posted by su...@apache.org.
Merge remote branch 'origin/master' into traf2663
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c55b2fd8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c55b2fd8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c55b2fd8
Branch: refs/heads/master
Commit: c55b2fd89ff4a692ff8d3d647d45d4c66faaf35d
Parents: d1e3cec 20c459c
Author: Steve Varnau <st...@esgyn.com>
Authored: Wed Jun 28 20:55:47 2017 +0000
Committer: Steve Varnau <st...@esgyn.com>
Committed: Wed Jun 28 20:55:47 2017 +0000
----------------------------------------------------------------------
core/.gitignore | 2 +-
core/Makefile | 2 +-
.../org/trafodion/jdbc/t4/TrafT4ResultSet.java | 36 +-
core/conn/odb/build.bat | 4 +-
core/conn/odb/installer.iss | 4 +-
core/conn/odb/odb/odb.rc | Bin 5120 -> 5336 bytes
core/conn/odb/src/odb.c | 8 +
core/dbsecurity/auth/src/authEvents.cpp | 2 +-
core/sqf/commonLogger/CommonLogger.cpp | 17 +-
core/sqf/conf/log4j.sql.config | 6 +-
core/sqf/conf/trafodion-site.xml | 38 +
core/sqf/conf/trafodion.udr.config | 2 +-
core/sqf/sqenvcom.sh | 1 +
core/sqf/sql/scripts/genms | 3 +
core/sqf/sql/scripts/gensq.pl | 28 -
core/sqf/sql/scripts/install_local_hadoop | 8 -
core/sqf/sql/scripts/rmscheck | 19 +-
core/sqf/sql/scripts/sqgen | 84 +-
core/sqf/sql/scripts/sqgenrmscheck | 118 +
core/sqf/sql/scripts/trafodion-site.xml | 38 -
.../transactional/TrxRegionEndpoint.java.tmpl | 45 +-
.../transactional/TrxTransactionState.java.tmpl | 1 -
.../seatrans/tm/hbasetmlib2/TMInterface.java | 6 +-
.../java/org/trafodion/dtm/HBaseTxClient.java | 8 +-
.../main/java/org/trafodion/dtm/TrafInfo.java | 2 +-
core/sqf/sysinstall/home/trafodion/.bashrc | 2 +-
core/sql/arkcmp/CmpContext.cpp | 2 +
core/sql/arkcmp/CmpContext.h | 7 +
core/sql/cli/Context.h | 3 +
core/sql/common/Ipc.h | 2 +-
core/sql/executor/ExExeUtil.h | 5 +
core/sql/executor/ExExeUtilGet.cpp | 186 +-
core/sql/executor/HBaseClient_JNI.cpp | 38 +-
core/sql/executor/HBaseClient_JNI.h | 4 +
core/sql/executor/JavaObjectInterface.cpp | 282 ++-
core/sql/executor/JavaObjectInterface.h | 19 +-
core/sql/executor/OrcFileReader.cpp | 1 -
core/sql/executor/SequenceFileReader.cpp | 3 -
core/sql/executor/ex_frag_rt.cpp | 7 +-
core/sql/exp/ExpHbaseInterface.cpp | 19 +
core/sql/exp/ExpHbaseInterface.h | 3 +-
core/sql/export/NAStringDef.cpp | 14 +
core/sql/export/NAStringDef.h | 8 +-
core/sql/langman/LmJavaOptions.cpp | 11 +
core/sql/langman/LmJavaOptions.h | 3 +
core/sql/lib_mgmt/Makefile | 4 -
core/sql/lib_mgmt/README.rst | 36 +-
core/sql/lib_mgmt/pom.xml | 27 +-
.../java/org/trafodion/libmgmt/FileMgmt.java | 487 ++--
.../java/org/trafodion/libmgmt/SyncLibUDF.java | 171 ++
.../lib_mgmt/src/main/resources/init_libmgmt.sh | 206 --
core/sql/optimizer/BindRelExpr.cpp | 7 -
core/sql/optimizer/HDFSHook.cpp | 58 +-
core/sql/optimizer/HDFSHook.h | 78 +-
core/sql/optimizer/NAClusterInfo.cpp | 265 +-
core/sql/optimizer/NAClusterInfo.h | 12 +-
core/sql/optimizer/NodeMap.cpp | 12 +-
core/sql/optimizer/ObjectNames.cpp | 17 +
core/sql/optimizer/ObjectNames.h | 3 +
core/sql/optimizer/OptPhysRelExpr.cpp | 16 +-
core/sql/optimizer/OptimizerSimulator.cpp | 2322 +++++++++++++-----
core/sql/optimizer/OptimizerSimulator.h | 257 +-
core/sql/optimizer/RelExeUtil.cpp | 1 -
core/sql/optimizer/RelExpr.cpp | 5 -
core/sql/optimizer/RelMisc.h | 9 -
core/sql/optimizer/ScanOptimizer.cpp | 22 +-
core/sql/optimizer/ScmCostMethod.cpp | 2 +-
core/sql/regress/compGeneral/EXPECTED023 | 51 +-
core/sql/regress/compGeneral/FILTER023 | 10 +-
core/sql/regress/privs1/EXPECTED125 | 1333 ++++++++++
core/sql/regress/privs1/TEST125 | 243 ++
core/sql/regress/privs1/TestHive.java | 91 +
core/sql/regress/privs1/Utils.java | 279 +++
core/sql/regress/privs2/EXPECTED144 | Bin 59453 -> 59453 bytes
core/sql/regress/udr/EXPECTED001 | 43 +-
core/sql/regress/udr/EXPECTED002 | 6 +-
core/sql/regress/udr/EXPECTED102 | 49 +-
core/sql/regress/udr/TEST001 | 18 +-
core/sql/regress/udr/TEST002 | 6 +-
core/sql/regress/udr/TEST102 | 27 +-
core/sql/sort/scratchfile_sq.cpp | 1 -
core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 4 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 11 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.h | 77 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 25 +-
core/sql/sqlcomp/DefaultConstants.h | 3 +
core/sql/sqlcomp/PrivMgr.cpp | 6 +-
core/sql/sqlcomp/PrivMgrComponentPrivileges.cpp | 7 +
core/sql/sqlcomp/nadefaults.cpp | 23 +-
core/sql/sqludr/SqlUdrPredefLogReader.cpp | 42 +-
.../java/org/trafodion/sql/HBaseClient.java | 29 +-
.../main/java/org/trafodion/sql/HiveClient.java | 2 +-
.../java/org/trafodion/sql/OrcFileReader.java | 25 +-
.../org/trafodion/sql/SequenceFileReader.java | 23 +-
.../org/trafodion/sql/SequenceFileWriter.java | 3 +-
.../org/trafodion/sql/TrafConfiguration.java | 61 +-
.../java/org/trafodion/sql/TrafRegionStats.java | 12 +-
core/sql/udrserv/UdrCfgParser.cpp | 4 +-
core/sql/udrserv/udrglobals.cpp | 85 +-
core/sql/udrserv/udrserv.cpp | 37 +-
core/sql/ustat/hs_globals.cpp | 55 +-
core/sql/ustat/hs_globals.h | 7 +-
core/trafodion.spec | 4 +-
.../src/asciidoc/_chapters/ambari_install.adoc | 29 +-
.../src/asciidoc/_chapters/introduction.adoc | 7 -
.../src/asciidoc/_chapters/requirements.adoc | 18 -
.../_chapters/sql_language_elements.adoc | 65 +-
.../src/asciidoc/_chapters/sql_utilities.adoc | 47 +-
install/Makefile | 19 +-
install/README.md | 7 +-
install/ambari-installer/mpack.json | 3 +
.../TRAFODION/2.1/configuration/dcs-env.xml | 1 +
.../TRAFODION/2.1/package/scripts/params.py | 1 +
.../2.1/package/scripts/trafodionmaster.py | 2 +-
install/installer/acceptKeys | 61 -
install/installer/addNode_packages | 89 -
install/installer/addNode_reservePorts | 81 -
install/installer/addNode_step1 | 273 --
install/installer/addNode_step2 | 429 ----
install/installer/bashrc_default | 87 -
install/installer/checkHBase | 37 -
install/installer/checkJava.py | 66 -
install/installer/cloud_cli_setup | 89 -
install/installer/dcs_installer | 219 --
install/installer/getNodes.py | 47 -
install/installer/parseHBaseSite.py | 61 -
install/installer/rest_installer | 115 -
install/installer/setup_known_hosts.exp | 35 -
install/installer/setup_ssh_key | 27 -
install/installer/sqconfig_persist.txt | 87 -
install/installer/tools/ambari_setup | 119 -
install/installer/tools/clouderaMoveDB.sh | 191 --
install/installer/tools/cloudera_setup | 198 --
install/installer/tools/cloudera_uninstall | 155 --
install/installer/tools/hortonworks_uninstall | 154 --
install/installer/tools/mapr_uninstall | 131 -
install/installer/tools/traf_cloudera_uninstall | 148 --
.../tools/traf_cloudera_uninstall_suse | 149 --
.../installer/tools/traf_hortonworks_uninstall | 233 --
.../tools/traf_hortonworks_uninstall_suse | 232 --
install/installer/tools/traf_mapr_uninstall | 49 -
install/installer/tools/trafodion_scanner | 525 ----
install/installer/tools/trafodion_scanner.cfg | 194 --
install/installer/traf_add_kerberos | 204 --
install/installer/traf_add_ldap | 98 -
install/installer/traf_add_sudoAccess | 85 -
install/installer/traf_add_user | 217 --
install/installer/traf_apache_mods | 259 --
.../installer/traf_authentication_conf_default | 71 -
install/installer/traf_cloudera_mods | 428 ----
install/installer/traf_config | 207 --
install/installer/traf_config_check | 1158 ---------
install/installer/traf_config_setup | 816 ------
install/installer/traf_createPasswordLessSSH | 63 -
install/installer/traf_getHadoopNodes | 252 --
install/installer/traf_getMultiHadoopNodes | 145 --
install/installer/traf_hortonworks_mods | 625 -----
install/installer/traf_package_setup | 202 --
install/installer/traf_secure | 60 -
install/installer/traf_secure_setup | 345 ---
install/installer/traf_setup | 134 -
install/installer/traf_sqconfig | 84 -
install/installer/traf_sqgen | 75 -
install/installer/traf_start | 75 -
install/installer/traf_user_prompt | 28 -
install/installer/traf_user_prompt_check | 27 -
install/installer/trafodion_config_default | 191 --
install/installer/trafodion_install | 833 -------
install/installer/trafodion_license | 205 --
install/installer/trafodion_secure_install | 129 -
install/installer/trafodion_uninstaller | 154 --
.../python-installer/scripts/traf_sqconfig.py | 8 +
.../Install/win64_installer/installer.iss | 8 +-
.../odbcclient/DSNConverter/DSNConverter.def | 2 +-
win-odbc64/odbcclient/Drvr35Res/Drvr35Res.rc | 10 +-
.../TranslationDll/TranslationDll.def | 2 +-
.../odbcclient/TranslationDll/TranslationDll.rc | 10 +-
win-odbc64/odbcclient/build_os.bat | 4 +-
win-odbc64/odbcclient/drvr35/TCPIPV4/TCPIPV4.RC | 10 +-
win-odbc64/odbcclient/drvr35/TCPIPV6/TCPIPV6.RC | 10 +-
win-odbc64/odbcclient/drvr35/cdatasource.cpp | 38 +-
win-odbc64/odbcclient/drvr35/drvr35.rc | 10 +-
win-odbc64/odbcclient/drvr35adm/drvr35adm.h | 42 +-
win-odbc64/odbcclient/drvr35adm/drvr35adm.rc | 10 +-
win-odbc64/odbcclient/drvr35msg/DrvMsg35.rc | 10 +-
win-odbc64/odbcclient/update_version.pl | 1 +
186 files changed, 5879 insertions(+), 13766 deletions(-)
----------------------------------------------------------------------
[7/7] incubator-trafodion git commit: Merge [TRAFODION-2663] PR-1140
Simplify HBase config settings in installers
Posted by su...@apache.org.
Merge [TRAFODION-2663] PR-1140 Simplify HBase config settings in installers
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3e5dd571
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3e5dd571
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3e5dd571
Branch: refs/heads/master
Commit: 3e5dd571cbfb7a486a0e4e86f6e52845ef2a0579
Parents: 5f76a57 35f9299
Author: Suresh Subbiah <su...@apache.org>
Authored: Mon Jul 3 17:09:05 2017 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Mon Jul 3 17:09:05 2017 +0000
----------------------------------------------------------------------
core/sqf/conf/trafodion-site.xml | 4 +
core/sqf/sql/scripts/hbcheck | 13 +-
install/ambari-installer/Makefile | 2 +-
.../TRAFODION/2.2.0/service_advisor.py | 253 +++++++++++++++++++
install/python-installer/configs/mod_cfgs.json | 11 +-
install/python-installer/scripts/hadoop_mods.py | 4 -
install/python-installer/scripts/hdfs_cmds.py | 4 +-
7 files changed, 274 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[3/7] incubator-trafodion git commit: [TRAFODION-2663] Use hbase
bulkload staging dir default value
Posted by su...@apache.org.
[TRAFODION-2663] Use hbase bulkload staging dir default value
This change-set exposed a race condition in installation.
Hbase regionserver was dying because pyinstaller is starting
regionserver before the new staging dir exists in hdfs. Prior to
simplifying the settings, installer won the race of creating hdfs
dir before regionserver looked for it.
To address this, we should just use HBase default value for.
Also remove hbase setting for hbase.client.keyvalue.maxsize from
installer. That can be added to trafodion-site.xml if needed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c0e39511
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c0e39511
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c0e39511
Branch: refs/heads/master
Commit: c0e395117a66a81904ba1650f7996ad35d740f94
Parents: 98620b7
Author: Steve Varnau <st...@esgyn.com>
Authored: Wed Jun 28 19:27:03 2017 +0000
Committer: Steve Varnau <st...@esgyn.com>
Committed: Wed Jun 28 19:27:03 2017 +0000
----------------------------------------------------------------------
install/python-installer/configs/mod_cfgs.json | 6 ++----
install/python-installer/scripts/hdfs_cmds.py | 4 ++--
2 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c0e39511/install/python-installer/configs/mod_cfgs.json
----------------------------------------------------------------------
diff --git a/install/python-installer/configs/mod_cfgs.json b/install/python-installer/configs/mod_cfgs.json
index 9234a52..f1223a1 100644
--- a/install/python-installer/configs/mod_cfgs.json
+++ b/install/python-installer/configs/mod_cfgs.json
@@ -5,10 +5,8 @@
"hbase.hregion.impl": "org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion",
"hbase.regionserver.region.split.policy": "org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy",
"hbase.snapshot.enabled": "true",
- "hbase.bulkload.staging.dir": "/hbase-staging",
"hbase.regionserver.region.transactional.tlog": "true",
- "hbase.snapshot.region.timeout": "600000",
- "hbase.client.keyvalue.maxsize": "0"
+ "hbase.snapshot.region.timeout": "600000"
},
"hdfs-site": { "dfs.namenode.acls.enabled": "true" },
"zoo.cfg": { "maxClientCnxns": "0" }
@@ -30,7 +28,7 @@
"value" : "600000"
}, {
"name" : "hbase_regionserver_config_safety_valve",
- "value" : "<property>\r\n <name>hbase.hregion.impl</name>\r\n <value>org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.split.policy</name>\r\n <value>org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.enabled</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.bulkload.staging.dir</name>\r\n <value>/hbase-staging</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.transactional.tlog</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.region.timeout</name>\r\n <value>600000</value>\r\n</property>\r\n <property>\r\n <name>hbase.client.keyvalue.maxsize</name>\r\n <value>0</value>\r\n</property>\r\n "
+ "value" : "<property>\r\n <name>hbase.hregion.impl</name>\r\n <value>org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.split.policy</name>\r\n <value>org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.enabled</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.regionserver.region.transactional.tlog</name>\r\n <value>true</value>\r\n</property>\r\n <property>\r\n <name>hbase.snapshot.region.timeout</name>\r\n <value>600000</value>\r\n</property>\r\n "
} ]
},
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c0e39511/install/python-installer/scripts/hdfs_cmds.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/hdfs_cmds.py b/install/python-installer/scripts/hdfs_cmds.py
index e363860..1826f3b 100755
--- a/install/python-installer/scripts/hdfs_cmds.py
+++ b/install/python-installer/scripts/hdfs_cmds.py
@@ -47,8 +47,8 @@ def run():
hbase_user = dbcfgs['hbase_user']
run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin)
- run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive /hbase-staging' % (hdfs_bin, traf_loc))
- run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive /hbase-staging' % (hdfs_bin, hbase_user, hbase_user))
+ run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive' % (hdfs_bin, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive' % (hdfs_bin, hbase_user, hbase_user))
run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs' % (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc))
run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))