You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by kr...@apache.org on 2019/02/27 10:59:52 UTC
[ambari] branch branch-2.7 updated: AMBARI-25169 - Upgrade Apache
Solr version to 7.7.0 or later in Ambari (#2834)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 6eaf34b AMBARI-25169 - Upgrade Apache Solr version to 7.7.0 or later in Ambari (#2834)
6eaf34b is described below
commit 6eaf34b99e3c4e097e0cd237630066e273cf5499
Author: kasakrisz <33...@users.noreply.github.com>
AuthorDate: Wed Feb 27 11:59:47 2019 +0100
AMBARI-25169 - Upgrade Apache Solr version to 7.7.0 or later in Ambari (#2834)
---
.../ambari-infra-manager/docker/docker-compose.yml | 2 +-
.../docker/infra-manager-docker-compose.sh | 2 +-
ambari-infra/ambari-infra-solr-client/README.md | 4 ++--
.../src/main/python/migrationHelper.py | 19 ++++++++++---------
.../src/main/resources/solrIndexHelper.sh | 8 ++++----
.../docker/infra-solr-docker-compose.sh | 2 +-
ambari-infra/pom.xml | 2 +-
.../main/configsets/audit_logs/conf/solrconfig.xml | 2 +-
.../main/configsets/hadoop_logs/conf/solrconfig.xml | 2 +-
.../src/main/configsets/history/conf/solrconfig.xml | 2 +-
ambari-logsearch/docker/Dockerfile | 4 ++--
ambari-logsearch/docker/docker-compose.yml | 2 +-
ambari-logsearch/docker/logsearch-docker.sh | 2 +-
ambari-logsearch/docker/solr.yml | 2 +-
ambari-logsearch/docker/sso.yml | 2 +-
ambari-logsearch/pom.xml | 2 +-
.../0.1.0/package/scripts/command_commons.py | 8 ++++----
.../0.5.0/properties/service_logs-solrconfig.xml.j2 | 2 +-
18 files changed, 35 insertions(+), 34 deletions(-)
diff --git a/ambari-infra/ambari-infra-manager/docker/docker-compose.yml b/ambari-infra/ambari-infra-manager/docker/docker-compose.yml
index c031cd7..602835e 100644
--- a/ambari-infra/ambari-infra-manager/docker/docker-compose.yml
+++ b/ambari-infra/ambari-infra-manager/docker/docker-compose.yml
@@ -27,7 +27,7 @@ services:
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
# TODO: use infra-solr
- image: solr:${SOLR_VERSION:-7.5.0}
+ image: solr:${SOLR_VERSION:-7.7.0}
restart: always
hostname: solr
ports:
diff --git a/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh b/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh
index 531440d..d32a64d 100755
--- a/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh
+++ b/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh
@@ -72,7 +72,7 @@ AMBARI_LOCATION=$AMBARI_LOCATION
ZOOKEEPER_VERSION=3.4.10
ZOOKEEPER_CONNECTION_STRING=zookeeper:2181
-SOLR_VERSION=7.5.0
+SOLR_VERSION=7.7.0
HADOOP_VERSION=3.0.0
EOF
diff --git a/ambari-infra/ambari-infra-solr-client/README.md b/ambari-infra/ambari-infra-solr-client/README.md
index 24218eb..a3eb6c2 100644
--- a/ambari-infra/ambari-infra-solr-client/README.md
+++ b/ambari-infra/ambari-infra-solr-client/README.md
@@ -472,7 +472,7 @@ If the script finished successfully and everything looks green on Ambari UI as w
Migration for `ranger_audits` collection (cores):
```bash
-# by default, you will mirate to Lucene 6.6.2, if you want to migrate again to Solr 7 (not requred), you can use --version 7.5.0 flag
+# by default, you will migrate to Lucene 6.6.2, if you want to migrate again to Solr 7 (not requred), you can use --version 7.7.0 flag
/usr/lib/ambari-infra-solr-client/migrationHelper.py --ini-file $CONFIG_INI_LOCATION --action migrate -s RANGER
```
@@ -852,7 +852,7 @@ Options:
location of the index backups (for ranger). required
only if no backup path in the ini file
--version=INDEX_VERSION
- lucene index version for migration (6.6.2 or 7.5.0)
+ lucene index version for migration (6.6.2 or 7.7.0)
--solr-async-request-tries=SOLR_ASYNC_REQUEST_TRIES
number of max tries for async Solr requests (e.g.:
delete operation)
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
index 911bc52..2aba85c 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
+++ b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
@@ -18,25 +18,26 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
+import ConfigParser
+import base64
import copy
import glob
-import logging
-import os
-import sys
-import urllib2, ssl
import json
-import base64
+import logging
import optparse
+import os
import socket
+import ssl
+import sys
import time
import traceback
-import ConfigParser
-import solrDataManager as solr_data_manager
-
+import urllib2
from datetime import datetime, timedelta
from random import randrange, randint
from subprocess import Popen, PIPE
+import solrDataManager as solr_data_manager
+
HTTP_PROTOCOL = 'http'
HTTPS_PROTOCOL = 'https'
@@ -1913,7 +1914,7 @@ if __name__=="__main__":
parser.add_option("--atlas-index-location", dest="atlas_index_location", type="string", help="location of the index backups (for atlas). required only if no backup path in the ini file")
parser.add_option("--ranger-index-location", dest="ranger_index_location", type="string", help="location of the index backups (for ranger). required only if no backup path in the ini file")
- parser.add_option("--version", dest="index_version", type="string", default="6.6.2", help="lucene index version for migration (6.6.2 or 7.5.0)")
+ parser.add_option("--version", dest="index_version", type="string", default="6.6.2", help="lucene index version for migration (6.6.2 or 7.7.0)")
parser.add_option("--solr-async-request-tries", dest="solr_async_request_tries", type="int", default=400, help="number of max tries for async Solr requests (e.g.: delete operation)")
parser.add_option("--request-tries", dest="request_tries", type="int", help="number of tries for BACKUP/RESTORE status api calls in the request")
parser.add_option("--request-time-interval", dest="request_time_interval", type="int", help="time interval between BACKUP/RESTORE status api calls in the request")
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
index ab84546..67682af 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
+++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
@@ -43,7 +43,7 @@ function print_help() {
-b, --backup-enabled Use indexer tool with backup snapshots. (core filter won't be used)
-g, --debug Enable debug mode, IndexUpgrader output will be verbose.
-f, --force Force to start index upgrade, even is the version is at least 6.
- -v, --version Lucene version to upgrade (default: 6.6.2, available: 6.6.2, 7.5.0)
+ -v, --version Lucene version to upgrade (default: 6.6.2, available: 6.6.2, 7.7.0)
EOF
}
@@ -51,7 +51,7 @@ function upgrade_core() {
local INDEX_DIR=${1:?"usage: <index_base_dir> e.g.: /opt/ambari_infra_solr/data"}
local FORCE_UPDATE=${2:?"usage <force_update_flag> e.g.: true"}
local SOLR_CORE_FILTERS=${3:?"usage: <comma separated core filters> e.g.: hadoop_logs,audit_logs,history"}
- local LUCENE_VERSION=${4:?"usage <lucene_index_version> e.g.: 7.5.0"}
+ local LUCENE_VERSION=${4:?"usage <lucene_index_version> e.g.: 7.7.0"}
local BACKUP_MODE=${5:?"usage <backup_mode_enabled> e.g.: true"}
local DEBUG_MODE=${6:?"usage <debug_mode> e.g.: true"}
SOLR_CORE_FILTER_ARR=$(echo $SOLR_CORE_FILTERS | sed "s/,/ /g")
@@ -204,12 +204,12 @@ function upgrade_index() {
function upgrade_index_tool() {
# see: https://cwiki.apache.org/confluence/display/solr/IndexUpgrader+Tool
- : ${INDEX_VERSION:?"Please set the INDEX_VERSION variable! (6.6.2 or 7.5.0)"}
+ : ${INDEX_VERSION:?"Please set the INDEX_VERSION variable! (6.6.2 or 7.7.0)"}
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$INDEX_VERSION.jar:$DIR/migrate/lucene-backward-codecs-$INDEX_VERSION.jar" org.apache.lucene.index.IndexUpgrader ${@}
}
function check_index_tool() {
- : ${INDEX_VERSION:?"Please set the INDEX_VERSION variable! (6.6.2 or 7.5.0)"}
+ : ${INDEX_VERSION:?"Please set the INDEX_VERSION variable! (6.6.2 or 7.7.0)"}
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$INDEX_VERSION.jar:$DIR/migrate/lucene-backward-codecs-$INDEX_VERSION.jar" org.apache.lucene.index.CheckIndex ${@}
}
diff --git a/ambari-infra/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh b/ambari-infra/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
index 7ddb757..ec07f9a 100755
--- a/ambari-infra/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
+++ b/ambari-infra/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
@@ -72,7 +72,7 @@ AMBARI_LOCATION=$AMBARI_LOCATION
ZOOKEEPER_VERSION=3.4.10
ZOOKEEPER_CONNECTION_STRING=zookeeper:2181
-SOLR_VERSION=7.5.0
+SOLR_VERSION=7.7.0
EOF
}
diff --git a/ambari-infra/pom.xml b/ambari-infra/pom.xml
index c2c2653..5f3ac03 100644
--- a/ambari-infra/pom.xml
+++ b/ambari-infra/pom.xml
@@ -25,7 +25,7 @@
<properties>
<jdk.version>1.8</jdk.version>
- <solr.version>7.5.0</solr.version>
+ <solr.version>7.7.0</solr.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<python.ver>python >= 2.6</python.ver>
<deb.python.ver>python (>= 2.6)</deb.python.ver>
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
index ea77a09..dcc9390 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
@@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>7.5.0</luceneMatchVersion>
+ <luceneMatchVersion>7.7.0</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml
index 946f215..6152b6c 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml
@@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>7.5.0</luceneMatchVersion>
+ <luceneMatchVersion>7.7.0</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml
index 4c8cc1c..5502229 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml
@@ -16,7 +16,7 @@
limitations under the License.
-->
<config>
- <luceneMatchVersion>7.5.0</luceneMatchVersion>
+ <luceneMatchVersion>7.7.0</luceneMatchVersion>
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-dataimporthandler-.*\.jar" />
diff --git a/ambari-logsearch/docker/Dockerfile b/ambari-logsearch/docker/Dockerfile
index 9087bc3..ed89a89 100644
--- a/ambari-logsearch/docker/Dockerfile
+++ b/ambari-logsearch/docker/Dockerfile
@@ -54,7 +54,7 @@ RUN echo 'X11DisplayOffset 10\n' /etc/ssh/sshd_config
RUN git config --global url."https://".insteadOf git://
# Install Solr
-ENV SOLR_VERSION 7.5.0
+ENV SOLR_VERSION 7.7.0
RUN wget --no-check-certificate -O /root/solr-$SOLR_VERSION.tgz http://public-repo-1.hortonworks.com/ARTIFACTS/dist/lucene/solr/$SOLR_VERSION/solr-$SOLR_VERSION.tgz
RUN cd /root && tar -zxvf /root/solr-$SOLR_VERSION.tgz
@@ -62,7 +62,7 @@ RUN cd /root && tar -zxvf /root/solr-$SOLR_VERSION.tgz
WORKDIR /
RUN adduser knox
ENV KNOX_VERSION 1.0.0
-RUN wget -q -O /knox-${KNOX_VERSION}.zip http://download.nextag.com/apache/knox/${KNOX_VERSION}/knox-${KNOX_VERSION}.zip && unzip /knox-${KNOX_VERSION}.zip && rm knox-${KNOX_VERSION}.zip && ln -nsf knox-${KNOX_VERSION} knox && chmod +x /knox/bin/*.sh && chown -R knox /knox/
+RUN wget -q -O /knox-${KNOX_VERSION}.zip http://archive.apache.org/dist/knox/${KNOX_VERSION}/knox-${KNOX_VERSION}.zip && unzip /knox-${KNOX_VERSION}.zip && rm knox-${KNOX_VERSION}.zip && ln -nsf knox-${KNOX_VERSION} knox && chmod +x /knox/bin/*.sh && chown -R knox /knox/
ADD knox/keystores /knox-secrets
RUN cd /knox-secrets && unzip test-secrets.zip
diff --git a/ambari-logsearch/docker/docker-compose.yml b/ambari-logsearch/docker/docker-compose.yml
index dc136bf..2729e4a 100644
--- a/ambari-logsearch/docker/docker-compose.yml
+++ b/ambari-logsearch/docker/docker-compose.yml
@@ -26,7 +26,7 @@ services:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
- image: solr:${SOLR_VERSION:-7.5.0}
+ image: solr:${SOLR_VERSION:-7.7.0}
restart: always
hostname: solr
ports:
diff --git a/ambari-logsearch/docker/logsearch-docker.sh b/ambari-logsearch/docker/logsearch-docker.sh
index 8966361..39ce470 100755
--- a/ambari-logsearch/docker/logsearch-docker.sh
+++ b/ambari-logsearch/docker/logsearch-docker.sh
@@ -109,7 +109,7 @@ AMBARI_LOCATION=$AMBARI_LOCATION
ZOOKEEPER_VERSION=3.4.10
ZOOKEEPER_CONNECTION_STRING=zookeeper:2181
-SOLR_VERSION=7.5.0
+SOLR_VERSION=7.7.0
EOF
echo ".env file has been created. Check it out before starting Log Search. ($sdir/.env)"
exit
diff --git a/ambari-logsearch/docker/solr.yml b/ambari-logsearch/docker/solr.yml
index 5bef837..ce02eaa 100644
--- a/ambari-logsearch/docker/solr.yml
+++ b/ambari-logsearch/docker/solr.yml
@@ -15,7 +15,7 @@
version: '3.3'
services:
solr:
- image: solr:${SOLR_VERSION:-7.5.0}
+ image: solr:${SOLR_VERSION:-7.7.0}
restart: always
networks:
- logsearch-network
diff --git a/ambari-logsearch/docker/sso.yml b/ambari-logsearch/docker/sso.yml
index 9d27d4f..66fac8f 100644
--- a/ambari-logsearch/docker/sso.yml
+++ b/ambari-logsearch/docker/sso.yml
@@ -26,7 +26,7 @@ services:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
- image: solr:${SOLR_VERSION:-7.5.0}
+ image: solr:${SOLR_VERSION:-7.7.0}
restart: always
hostname: solr
ports:
diff --git a/ambari-logsearch/pom.xml b/ambari-logsearch/pom.xml
index 923d483..1e9e59a 100644
--- a/ambari-logsearch/pom.xml
+++ b/ambari-logsearch/pom.xml
@@ -45,7 +45,7 @@
<deb.python.ver>python (>= 2.6)</deb.python.ver>
<deb.architecture>amd64</deb.architecture>
<deb.dependency.list>${deb.python.ver}</deb.dependency.list>
- <solr.version>7.5.0</solr.version>
+ <solr.version>7.7.0</solr.version>
<hadoop.version>3.0.0</hadoop.version>
<common.io.version>2.5</common.io.version>
<zookeeper.version>3.4.6.2.3.0.0-2557</zookeeper.version>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
index 29f1857..fcb3439 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
@@ -19,24 +19,24 @@ limitations under the License.
import fnmatch
import json
import os
-import params
import socket
import time
import traceback
-
-from resource_management.core.shell import call
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
+from resource_management.core.shell import call
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.libraries.resources.hdfs_resource import HdfsResource
+import params
+
index_helper_script = '/usr/lib/ambari-infra-solr-client/solrIndexHelper.sh'
# folder location which contains the snapshot/core folder
index_location = default("/commandParams/solr_index_location", None)
-# index version (available index versions: 6.6.2 and 7.3.1, second one is used by default)
+# index version (available index versions: 6.6.2 and 7.7.0, first one is used by default)
index_version = default("/commandParams/solr_index_version", '6.6.2')
# if this flag is false, skip upgrade if the version is proper, you can force to re-run the tool with setting the flag to true
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2
index f42a074..b631860 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2
@@ -35,7 +35,7 @@ this file, see http://wiki.apache.org/solr/SolrConfigXml.
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>7.5.0</luceneMatchVersion>
+ <luceneMatchVersion>7.7.0</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in