You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/05/27 14:33:42 UTC
[ambari] branch trunk updated (91009d3 -> 4773eb0)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git.
from 91009d3 AMBARI-23943. Hiveserver2 fails to start on viewFS enabled cluster: {hive_server2_zookeeper_namespace} is not ready yet (aonishuk)
new b953a65 AMBARI-23945. Add new --delete-znode command for solrCloudClient.sh
new 5ef331a AMBARI-23945. Infra Solr: Add ranger managed-schema (new) for migration
new 4773eb0 AMBARI-23945. Infra Solr Migration: create parent folders during backup folder creation
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ambari-infra/ambari-infra-solr-client/build.xml | 1 +
.../ambari/infra/solr/AmbariSolrCloudCLI.java | 19 ++++-
.../ambari/infra/solr/AmbariSolrCloudClient.java | 8 ++
...odeZkCommand.java => DeleteZnodeZkCommand.java} | 14 +---
.../src/main/resources/managed-schema | 93 ++++++++++++++++++++++
.../0.1.0/package/scripts/collection.py | 1 +
6 files changed, 123 insertions(+), 13 deletions(-)
copy ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/{TransferZnodeZkCommand.java => DeleteZnodeZkCommand.java} (71%)
create mode 100644 ambari-infra/ambari-infra-solr-client/src/main/resources/managed-schema
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.
[ambari] 01/03: AMBARI-23945. Add new --delete-znode command for
solrCloudClient.sh
Posted by ol...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit b953a6518d0f3e7ddc27e1c01970dd3d0bdae91c
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Sun May 27 15:52:40 2018 +0200
AMBARI-23945. Add new --delete-znode command for solrCloudClient.sh
---
.../ambari/infra/solr/AmbariSolrCloudCLI.java | 19 +++++++++--
.../ambari/infra/solr/AmbariSolrCloudClient.java | 8 +++++
.../infra/solr/commands/DeleteZnodeZkCommand.java | 37 ++++++++++++++++++++++
3 files changed, 62 insertions(+), 2 deletions(-)
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
index ca8569f..c70c507 100644
--- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
+++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
@@ -54,6 +54,7 @@ public class AmbariSolrCloudCLI {
private static final String SECURITY_JSON_LOCATION = "security-json-location";
private static final String REMOVE_ADMIN_HANDLERS = "remove-admin-handlers";
private static final String TRANSFER_ZNODE_COMMAND = "transfer-znode";
+ private static final String DELETE_ZNODE_COMMAND = "delete-znode";
private static final String CMD_LINE_SYNTAX =
"\n./solrCloudCli.sh --create-collection -z host1:2181,host2:2181/ambari-solr -c collection -cs conf_set"
+ "\n./solrCloudCli.sh --upload-config -z host1:2181,host2:2181/ambari-solr -d /tmp/myconfig_dir -cs config_set"
@@ -63,6 +64,7 @@ public class AmbariSolrCloudCLI {
+ "\n./solrCloudCli.sh --remove-admin-handlers -z host1:2181,host2:2181/ambari-solr -c collection"
+ "\n./solrCloudCli.sh --create-znode -z host1:2181,host2:2181 -zn /ambari-solr"
+ "\n./solrCloudCli.sh --check-znode -z host1:2181,host2:2181 -zn /ambari-solr"
+ + "\n./solrCloudCli.sh --delete-znode -z host1:2181,host2:2181 -zn /ambari-solr"
+ "\n./solrCloudCli.sh --transfer-znode -z host1:2181,host2:2181 -cps /ambari-solr -cpd /ambari-solr-backup"
+ "\n./solrCloudCli.sh --cluster-prop -z host1:2181,host2:2181/ambari-solr -cpn urlScheme -cpn http"
+ "\n./solrCloudCli.sh --secure-znode -z host1:2181,host2:2181 -zn /ambari-solr -su logsearch,atlas,ranger --jaas-file /etc/myconf/jaas_file"
@@ -151,6 +153,11 @@ public class AmbariSolrCloudCLI {
.desc("Transfer znode (copy from/to local or to another znode)")
.build();
+ final Option deleteZnodeOption = Option.builder("dz")
+ .longOpt(DELETE_ZNODE_COMMAND)
+ .desc("Delete znode")
+ .build();
+
final Option shardNameOption = Option.builder("sn")
.longOpt("shard-name")
.desc("Name of the shard for create-shard command")
@@ -396,6 +403,7 @@ public class AmbariSolrCloudCLI {
options.addOption(copyDestOption);
options.addOption(saslUsersOption);
options.addOption(checkZnodeOption);
+ options.addOption(deleteZnodeOption);
options.addOption(setupKerberosPluginOption);
options.addOption(securityJsonLocationOption);
@@ -452,10 +460,13 @@ public class AmbariSolrCloudCLI {
} else if (cli.hasOption("tz")) {
command = TRANSFER_ZNODE_COMMAND;
validateRequiredOptions(cli, command, zkConnectStringOption, copyScrOption, copyDestOption);
- }else {
+ } else if (cli.hasOption("dz")) {
+ command = DELETE_ZNODE_COMMAND;
+ validateRequiredOptions(cli, command, zkConnectStringOption, znodeOption);
+ } else {
List<String> commands = Arrays.asList(CREATE_COLLECTION_COMMAND, CREATE_SHARD_COMMAND, UPLOAD_CONFIG_COMMAND,
DOWNLOAD_CONFIG_COMMAND, CONFIG_CHECK_COMMAND, SET_CLUSTER_PROP, CREATE_ZNODE, SECURE_ZNODE_COMMAND, UNSECURE_ZNODE_COMMAND,
- SECURE_SOLR_ZNODE_COMMAND, CHECK_ZNODE, SETUP_KERBEROS_PLUGIN, REMOVE_ADMIN_HANDLERS, TRANSFER_ZNODE_COMMAND);
+ SECURE_SOLR_ZNODE_COMMAND, CHECK_ZNODE, SETUP_KERBEROS_PLUGIN, REMOVE_ADMIN_HANDLERS, TRANSFER_ZNODE_COMMAND, DELETE_ZNODE_COMMAND);
helpFormatter.printHelp(CMD_LINE_SYNTAX, options);
exit(1, String.format("One of the supported commands is required (%s)", StringUtils.join(commands, "|")));
}
@@ -591,6 +602,10 @@ public class AmbariSolrCloudCLI {
solrCloudClient = clientBuilder.build();
solrCloudClient.transferZnode();
break;
+ case DELETE_ZNODE_COMMAND:
+ solrCloudClient = clientBuilder.build();
+ solrCloudClient.deleteZnode();
+ break;
default:
throw new AmbariSolrCloudClientException(String.format("Not found command: '%s'", command));
}
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
index 7d3401c..2632fcc 100644
--- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
+++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
@@ -22,6 +22,7 @@ import org.apache.ambari.infra.solr.commands.CheckConfigZkCommand;
import org.apache.ambari.infra.solr.commands.CreateCollectionCommand;
import org.apache.ambari.infra.solr.commands.CreateShardCommand;
import org.apache.ambari.infra.solr.commands.CreateSolrZnodeZkCommand;
+import org.apache.ambari.infra.solr.commands.DeleteZnodeZkCommand;
import org.apache.ambari.infra.solr.commands.DownloadConfigZkCommand;
import org.apache.ambari.infra.solr.commands.EnableKerberosPluginSolrZkCommand;
import org.apache.ambari.infra.solr.commands.GetShardsCommand;
@@ -279,6 +280,13 @@ public class AmbariSolrCloudClient {
return new TransferZnodeZkCommand(getRetryTimes(), getInterval()).run(this);
}
+ /**
+ * Delete znode path (and all sub nodes)
+ */
+ public boolean deleteZnode() throws Exception {
+ return new DeleteZnodeZkCommand(getRetryTimes(), getInterval()).run(this);
+ }
+
public String getZkConnectString() {
return zkConnectString;
}
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/DeleteZnodeZkCommand.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/DeleteZnodeZkCommand.java
new file mode 100644
index 0000000..1c9d4fb
--- /dev/null
+++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/DeleteZnodeZkCommand.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.solr.commands;
+
+import org.apache.ambari.infra.solr.AmbariSolrCloudClient;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.SolrZooKeeper;
+
+public class DeleteZnodeZkCommand extends AbstractZookeeperRetryCommand<Boolean> {
+
+ public DeleteZnodeZkCommand(int maxRetries, int interval) {
+ super(maxRetries, interval);
+ }
+
+ @Override
+ protected Boolean executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception {
+ zkClient.clean(client.getZnode());
+ return true;
+ }
+
+}
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.
[ambari] 02/03: AMBARI-23945. Infra Solr: Add ranger managed-schema
(new) for migration
Posted by ol...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 5ef331ae8764b10888dc2db27d85151ac601d477
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Sun May 27 16:22:07 2018 +0200
AMBARI-23945. Infra Solr: Add ranger managed-schema (new) for migration
---
ambari-infra/ambari-infra-solr-client/build.xml | 1 +
.../src/main/resources/managed-schema | 93 ++++++++++++++++++++++
2 files changed, 94 insertions(+)
diff --git a/ambari-infra/ambari-infra-solr-client/build.xml b/ambari-infra/ambari-infra-solr-client/build.xml
index 3ddbb51..866e323 100644
--- a/ambari-infra/ambari-infra-solr-client/build.xml
+++ b/ambari-infra/ambari-infra-solr-client/build.xml
@@ -41,6 +41,7 @@
<fileset file="target/migrate/*.jar"/>
<fileset file="target/package/libs/lucene-core-${solr.version}.jar"/>
<fileset file="target/package/libs/lucene-backward-codecs-${solr.version}.jar"/>
+ <fileset file="src/main/resources/managed-schema"/>
</copy>
<copy todir="target/package" includeEmptyDirs="no">
<fileset file="src/main/resources/solrCloudCli.sh"/>
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/managed-schema b/ambari-infra/ambari-infra-solr-client/src/main/resources/managed-schema
new file mode 100644
index 0000000..6c87af7
--- /dev/null
+++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/managed-schema
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="ranger-audit-schema" version="1.6">
+ <uniqueKey>id</uniqueKey>
+ <fieldType name="binary" class="solr.BinaryField"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+ <fieldType name="booleans" class="solr.BoolField" multiValued="true" sortMissingLast="true"/>
+ <fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="ignored" class="solr.StrField" multiValued="true" indexed="false" stored="false"/>
+ <fieldType name="int" class="solr.TrieIntField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="key_lower_case" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.LengthFilterFactory" min="0" max="2500"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+ <fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdates" class="solr.TrieDateField" docValues="true" precisionStep="6" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="text_std_token_lower_case" class="solr.TextField" multiValued="true" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tints" class="solr.TrieIntField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <field name="_expire_at_" type="tdate" multiValued="false" stored="true" docValues="true"/>
+ <field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="false" stored="true"/>
+ <field name="access" type="key_lower_case" multiValued="false"/>
+ <field name="action" type="key_lower_case" multiValued="false"/>
+ <field name="agent" type="key_lower_case" multiValued="false"/>
+ <field name="agentHost" type="key_lower_case" multiValued="false"/>
+ <field name="cliIP" type="key_lower_case" multiValued="false"/>
+ <field name="cliType" type="key_lower_case" multiValued="false"/>
+ <field name="cluster" type="key_lower_case" multiValued="false"/>
+ <field name="reqContext" type="key_lower_case" multiValued="true"/>
+ <field name="enforcer" type="key_lower_case" multiValued="false"/>
+ <field name="event_count" type="tlong" multiValued="false" docValues="true" default="1"/>
+ <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/>
+ <field name="evtTime" type="tdate" docValues="true"/>
+ <field name="id" type="string" multiValued="false" indexed="true" required="true" stored="true"/>
+ <field name="logType" type="key_lower_case" multiValued="false"/>
+ <field name="policy" type="tlong" docValues="true"/>
+ <field name="proxyUsers" type="key_lower_case" multiValued="true"/>
+ <field name="reason" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/>
+ <field name="repo" type="key_lower_case" multiValued="false"/>
+ <field name="repoType" type="tint" multiValued="false" docValues="true"/>
+ <field name="req_caller_id" type="key_lower_case" multiValued="false"/>
+ <field name="req_self_id" type="key_lower_case" multiValued="false"/>
+ <field name="reqData" type="text_std_token_lower_case" multiValued="false"/>
+ <field name="reqUser" type="key_lower_case" multiValued="false"/>
+ <field name="resType" type="key_lower_case" multiValued="false"/>
+ <field name="resource" type="key_lower_case" multiValued="false"/>
+ <field name="result" type="tint" multiValued="false"/>
+ <field name="seq_num" type="tlong" multiValued="false" docValues="true" default="0"/>
+ <field name="sess" type="key_lower_case" multiValued="false"/>
+ <field name="tags" type="key_lower_case" multiValued="true"/>
+ <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/>
+ <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/>
+</schema>
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.
[ambari] 03/03: AMBARI-23945. Infra Solr Migration: create parent
folders during backup folder creation
Posted by ol...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 4773eb0696468da5f15908580224a13c01f2d799
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Sun May 27 16:28:09 2018 +0200
AMBARI-23945. Infra Solr Migration: create parent folders during backup folder creation
---
.../AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
index 094f782..46c4e0e 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
@@ -34,6 +34,7 @@ def backup_collection(env):
Directory(command_commons.index_location,
mode=0755,
cd_access='a',
+ create_parents=True,
owner=params.infra_solr_user,
group=params.user_group
)
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.