You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/10/13 15:10:49 UTC
[30/50] [abbrv] ambari git commit: AMBARI-22188. Make hive server
create directories related to replication (echekanskiy)
AMBARI-22188. Make hive server create directories related to replication (echekanskiy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a47e792
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a47e792
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a47e792
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 4a47e792ae6e4fe4fb67ca80c96c5e2054f7a9cd
Parents: 7341567
Author: Eugene Chekanskiy <ec...@hortonworks.com>
Authored: Wed Oct 11 19:38:41 2017 +0300
Committer: Eugene Chekanskiy <ec...@hortonworks.com>
Committed: Wed Oct 11 19:38:41 2017 +0300
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/configuration/hive-site.xml | 47 +++++++++++++++++++
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 34 +++++++++++++-
.../0.12.0.2.0/package/scripts/params_linux.py | 4 ++
.../HIVE/2.1.0.3.0/configuration/hive-site.xml | 48 ++++++++++++++++++++
.../HIVE/2.1.0.3.0/package/scripts/hive.py | 32 +++++++++++++
.../2.1.0.3.0/package/scripts/params_linux.py | 4 ++
6 files changed, 168 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index 69d1c69..762530b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -471,4 +471,51 @@ limitations under the License.
</depends-on>
<on-ambari-upgrade add="false"/>
</property>
+
+ <!-- Hive replication properties -->
+ <property>
+ <name>hive.metastore.dml.events</name>
+ <value/>
+ <description>If true, the metastore will be asked to fire events for DML operations</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.repl.cm.enabled</name>
+ <value/>
+ <description>Turn on ChangeManager, so delete files will go to cmrootdir.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.metastore.transactional.event.listeners</name>
+ <value/>
+ <description>A comma separated list of Java classes that implement the org.apache.hadoop.hive.metastore.MetaStoreEventListener interface. Both the metastore event and corresponding listener method will be invoked in the same JDO transaction.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.repl.cmrootdir</name>
+ <value/>
+ <description>Root dir for ChangeManager, used for deleted files.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.repl.rootdir</name>
+ <value/>
+ <description>HDFS root dir for all replication dumps.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 8e176b6..c4b34a5 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -245,7 +245,22 @@ def setup_hiveserver2():
owner=params.hive_user,
group=params.hdfs_user,
mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
-
+
+ if params.hive_repl_cmrootdir is not None:
+ params.HdfsResource(params.hive_repl_cmrootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 01777)
+ if params.hive_repl_rootdir is not None:
+ params.HdfsResource(params.hive_repl_rootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 0700)
+
params.HdfsResource(None, action="execute")
def setup_non_client():
@@ -310,6 +325,23 @@ def setup_metastore():
create_parents = True,
mode=0777)
+ if params.hive_repl_cmrootdir is not None:
+ params.HdfsResource(params.hive_repl_cmrootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 01777)
+ if params.hive_repl_rootdir is not None:
+ params.HdfsResource(params.hive_repl_rootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 0700)
+ if params.hive_repl_cmrootdir is not None or params.hive_repl_rootdir is not None:
+ params.HdfsResource(None, action="execute")
+
def create_metastore_schema():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 1d68ef4..16e1a71 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -871,3 +871,7 @@ if 'druid-common' in config['configurations'] \
manage_hive_fsroot = default('/configurations/cluster-env/manage_hive_fsroot', True)
+
+# replication directories
+hive_repl_cmrootdir = default('/configurations/hive-site/hive.repl.cmrootdir', None)
+hive_repl_rootdir = default('/configurations/hive-site/hive.repl.rootdir', None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
index edbbc9d..3c153b6 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
@@ -2001,4 +2001,52 @@ limitations under the License.
</description>
<on-ambari-upgrade add="false"/>
</property>
+
+ <!-- Hive replication properties -->
+ <property>
+ <name>hive.metastore.dml.events</name>
+ <value/>
+ <description>If true, the metastore will be asked to fire events for DML operations</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.repl.cm.enabled</name>
+ <value/>
+ <description>Turn on ChangeManager, so delete files will go to cmrootdir.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.metastore.transactional.event.listeners</name>
+ <value/>
+ <description>A comma separated list of Java classes that implement the org.apache.hadoop.hive.metastore.MetaStoreEventListener interface. Both the metastore event and corresponding listener method will be invoked in the same JDO transaction.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.repl.cmrootdir</name>
+ <value/>
+ <description>Root dir for ChangeManager, used for deleted files.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.repl.rootdir</name>
+ <value/>
+ <description>HDFS root dir for all replication dumps.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
index e7e92ec..22ff9fd 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
@@ -245,6 +245,21 @@ def setup_hiveserver2():
group=params.hdfs_user,
mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
+ if params.hive_repl_cmrootdir is not None:
+ params.HdfsResource(params.hive_repl_cmrootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 01777)
+ if params.hive_repl_rootdir is not None:
+ params.HdfsResource(params.hive_repl_rootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 0700)
+
params.HdfsResource(None, action="execute")
def setup_non_client():
@@ -300,6 +315,23 @@ def setup_metastore():
content=StaticFile('startMetastore.sh')
)
+ if params.hive_repl_cmrootdir is not None:
+ params.HdfsResource(params.hive_repl_cmrootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 01777)
+ if params.hive_repl_rootdir is not None:
+ params.HdfsResource(params.hive_repl_rootdir,
+ type = "directory",
+ action = "create_on_execute",
+ owner = params.hive_user,
+ group=params.user_group,
+ mode = 0700)
+ if params.hive_repl_cmrootdir is not None or params.hive_repl_rootdir is not None:
+ params.HdfsResource(None, action="execute")
+
def create_metastore_schema():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
index f6b676b..fb2c84a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
@@ -848,3 +848,7 @@ ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-prope
if security_enabled:
hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower())
+
+# replication directories
+hive_repl_cmrootdir = default('/configurations/hive-site/hive.repl.cmrootdir', None)
+hive_repl_rootdir = default('/configurations/hive-site/hive.repl.rootdir', None)