You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/02/10 00:16:44 UTC
[ambari] branch trunk updated: AMBARI-22899. Upgrade Infra Solr
version to 7.2.1 (#309)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2421d98 AMBARI-22899. Upgrade Infra Solr version to 7.2.1 (#309)
2421d98 is described below
commit 2421d981baab8552b15319550d9fe46ca569e27d
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Sat Feb 10 01:16:38 2018 +0100
AMBARI-22899. Upgrade Infra Solr version to 7.2.1 (#309)
* AMBARI-22899. Upgrade Infra Solr version to 7.2.1
* AMBARI-22899. Set missing Lucene version for service logs.
* AMBARI-22899. Updated Solr tarball location.
* AMBARI-22899. Delete migrate dir on start of package.
---
.../libraries/functions/solr_cloud_util.py | 6 +-
ambari-infra/ambari-infra-assembly/pom.xml | 2 +-
.../ambari-infra-assembly/src/main/resources/solr | 356 +++--
ambari-infra/ambari-infra-solr-client/build.xml | 9 +
ambari-infra/ambari-infra-solr-client/pom.xml | 8 +
.../ambari/infra/solr/AmbariSolrCloudCLI.java | 12 +-
.../ambari/infra/solr/AmbariSolrCloudClient.java | 10 +-
.../infra/solr/AmbariSolrCloudClientBuilder.java | 17 +-
.../solr/commands/CreateCollectionCommand.java | 6 +-
.../src/main/resources/solrIndexHelper.sh | 60 +-
.../InfraRuleBasedAuthorizationPlugin.java | 542 --------
.../security/InfraKerberosHostValidator.java | 2 +-
.../InfraRuleBasedAuthorizationPlugin.java | 303 +++++
.../security/InfraUserRolesLookupStrategy.java | 2 +-
.../security/InfraKerberosHostValidatorTest.java | 2 +-
.../InfraRuleBasedAuthorizationPluginTest.java | 39 +-
.../security/InfraUserRolesLookupStrategyTest.java | 2 +-
ambari-infra/pom.xml | 2 +-
.../main/configsets/audit_logs/conf/solrconfig.xml | 10 +-
.../configsets/hadoop_logs/conf/solrconfig.xml | 10 +-
.../main/configsets/history/conf/solrconfig.xml | 1414 +-------------------
ambari-logsearch/docker/docker-compose.yml | 2 +-
ambari-logsearch/docker/solr.yml | 2 +-
.../package/templates/infra-solr-security.json.j2 | 2 +-
.../0.1.0/properties/infra-solr-env.sh.j2 | 5 +-
.../0.7.0.2.5/configuration/atlas-solrconfig.xml | 2 +-
.../0.5.0/properties/audit_logs-solrconfig.xml.j2 | 7 +-
.../properties/service_logs-solrconfig.xml.j2 | 7 +-
.../0.7.0/properties/ranger-solrconfig.xml.j2 | 7 +-
29 files changed, 738 insertions(+), 2110 deletions(-)
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
index b92c77e..5a43aac 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
@@ -87,7 +87,7 @@ def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_
)
def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java64_home,
- shards = 1, replication_factor = 1, max_shards = 1, retry = 5, interval = 10,
+ shards = 1, replication_factor = 1, max_shards = 1, retry = 5, interval = 10, implicitRouting = False,
router_name = None, router_field = None, jaas_file = None, key_store_location = None,
key_store_password = None, key_store_type = None, trust_store_location = None,
trust_store_password = None, trust_store_type = None, java_opts=None):
@@ -107,7 +107,9 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
create_collection_cmd = format('{solr_cli_prefix} --create-collection --collection {collection} --config-set {config_set} '\
'--shards {shards} --replication {replication_factor} --max-shards {max_shards} --retry {retry} '\
- '--interval {interval} --no-sharding')
+ '--interval {interval}')
+
+ create_collection_cmd = create_collection_cmd + ' --implicit-routing' if implicitRouting else create_collection_cmd
appendableDict = {}
appendableDict["--router-name"] = router_name
appendableDict["--router-field"] = router_field
diff --git a/ambari-infra/ambari-infra-assembly/pom.xml b/ambari-infra/ambari-infra-assembly/pom.xml
index 41fa573..294e787 100644
--- a/ambari-infra/ambari-infra-assembly/pom.xml
+++ b/ambari-infra/ambari-infra-assembly/pom.xml
@@ -30,7 +30,7 @@
<properties>
<mapping.base.path>/usr/lib</mapping.base.path>
- <!-- original apache repo location: http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz -->
+ <!-- original url: <solr.tar>http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar> -->
<solr.tar>http://public-repo-1.hortonworks.com/ARTIFACTS/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar>
<solr.mapping.path>${mapping.base.path}/ambari-infra-solr</solr.mapping.path>
<solr.package.name>ambari-infra-solr</solr.package.name>
diff --git a/ambari-infra/ambari-infra-assembly/src/main/resources/solr b/ambari-infra/ambari-infra-assembly/src/main/resources/solr
index 6f2de8f..a42695e 100755
--- a/ambari-infra/ambari-infra-assembly/src/main/resources/solr
+++ b/ambari-infra/ambari-infra-assembly/src/main/resources/solr
@@ -19,7 +19,7 @@
#
# Use solr -help to see available command-line options. In addition
# to passing command-line options, this script looks for an include
-# file named solr.in.sh to set environment variables. Specifically,
+# file named solr.in.sh to set environment variables. Specifically,
# the following locations are searched in this order:
#
# ./
@@ -34,15 +34,15 @@
#
# $ SOLR_INCLUDE=/path/to/solr.in.sh solr start
#
-# Note: This is particularly handy for running multiple instances on a
+# Note: This is particularly handy for running multiple instances on a
# single installation, or for quick tests.
#
-# Finally, developers and enthusiasts who frequently run from an SVN
+# Finally, developers and enthusiasts who frequently run from an SVN
# checkout, and do not want to locally modify bin/solr.in.sh, can put
# a customized include file at ~/.solr.in.sh.
#
# If you would rather configure startup entirely from the environment, you
-# can disable the include by exporting an empty SOLR_INCLUDE, or by
+# can disable the include by exporting an empty SOLR_INCLUDE, or by
# ensuring that no include files exist in the aforementioned search list.
SOLR_SCRIPT="$0"
@@ -163,12 +163,21 @@ fi
SOLR_URL_SCHEME=http
SOLR_JETTY_CONFIG=()
SOLR_SSL_OPTS=""
-if [ -n "$SOLR_SSL_KEY_STORE" ]; then
+if [ -z "$SOLR_SSL_ENABLED" ]; then
+ if [ -n "$SOLR_SSL_KEY_STORE" ]; then
+ SOLR_SSL_ENABLED="true" # implicitly from earlier behaviour
+ else
+ SOLR_SSL_ENABLED="false"
+ fi
+fi
+if [ "$SOLR_SSL_ENABLED" == "true" ]; then
SOLR_JETTY_CONFIG+=("--module=https")
SOLR_URL_SCHEME=https
- SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE"
+ if [ -n "$SOLR_SSL_KEY_STORE" ]; then
+ SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE"
+ fi
if [ -n "$SOLR_SSL_KEY_STORE_PASSWORD" ]; then
- SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore.password=$SOLR_SSL_KEY_STORE_PASSWORD"
+ export SOLR_SSL_KEY_STORE_PASSWORD=$SOLR_SSL_KEY_STORE_PASSWORD
fi
if [ -n "$SOLR_SSL_KEY_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore.type=$SOLR_SSL_KEY_STORE_TYPE"
@@ -178,7 +187,7 @@ if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore=$SOLR_SSL_TRUST_STORE"
fi
if [ -n "$SOLR_SSL_TRUST_STORE_PASSWORD" ]; then
- SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore.password=$SOLR_SSL_TRUST_STORE_PASSWORD"
+ export SOLR_SSL_TRUST_STORE_PASSWORD=$SOLR_SSL_TRUST_STORE_PASSWORD
fi
if [ -n "$SOLR_SSL_TRUST_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore.type=$SOLR_SSL_TRUST_STORE_TYPE"
@@ -195,7 +204,7 @@ if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_CLIENT_KEY_STORE"
if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD" ]; then
- SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD"
+ export SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD
fi
if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreType=$SOLR_SSL_CLIENT_KEY_STORE_TYPE"
@@ -204,9 +213,6 @@ if [ -n "$SOLR_SSL_KEY_STORE" ]; then
if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_KEY_STORE"
fi
- if [ -n "$SOLR_SSL_KEY_STORE_PASSWORD" ]; then
- SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_KEY_STORE_PASSWORD"
- fi
if [ -n "$SOLR_SSL_KEY_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreType=$SOLR_SSL_KEYSTORE_TYPE"
fi
@@ -216,9 +222,8 @@ if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStore=$SOLR_SSL_CLIENT_TRUST_STORE"
if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" ]; then
- SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD"
+ export SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD
fi
-
if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStoreType=$SOLR_SSL_CLIENT_TRUST_STORE_TYPE"
fi
@@ -227,10 +232,6 @@ if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStore=$SOLR_SSL_TRUST_STORE"
fi
- if [ -n "$SOLR_SSL_TRUST_STORE_PASSWORD" ]; then
- SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_TRUST_STORE_PASSWORD"
- fi
-
if [ -n "$SOLR_SSL_TRUST_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStoreType=$SOLR_SSL_TRUST_STORE_TYPE"
fi
@@ -246,20 +247,20 @@ if [ -z "$SOLR_AUTH_TYPE" ] && [ -n "$SOLR_AUTHENTICATION_OPTS" ]; then
echo " Currently supported authentication types are [kerberos, basic]"
fi
-if [ -n "$SOLR_AUTH_TYPE" ] && [ -n "$SOLR_AUTHENTICATION_CLIENT_CONFIGURER" ]; then
- echo "WARNING: SOLR_AUTHENTICATION_CLIENT_CONFIGURER and SOLR_AUTH_TYPE environment variables are configured together."
+if [ -n "$SOLR_AUTH_TYPE" ] && [ -n "$SOLR_AUTHENTICATION_CLIENT_BUILDER" ]; then
+ echo "WARNING: SOLR_AUTHENTICATION_CLIENT_BUILDER and SOLR_AUTH_TYPE environment variables are configured together."
echo " Use SOLR_AUTH_TYPE environment variable to configure authentication type to be used. "
echo " Currently supported authentication types are [kerberos, basic]"
- echo " The value of SOLR_AUTHENTICATION_CLIENT_CONFIGURER environment variable will be ignored"
+ echo " The value of SOLR_AUTHENTICATION_CLIENT_BUILDER environment variable will be ignored"
fi
if [ -n "$SOLR_AUTH_TYPE" ]; then
case "$(echo $SOLR_AUTH_TYPE | awk '{print tolower($0)}')" in
basic)
- SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthConfigurer"
+ SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
;;
kerberos)
- SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer"
+ SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder"
;;
*)
echo "ERROR: Value specified for SOLR_AUTH_TYPE environment variable is invalid."
@@ -268,9 +269,13 @@ if [ -n "$SOLR_AUTH_TYPE" ]; then
fi
if [ "$SOLR_AUTHENTICATION_CLIENT_CONFIGURER" != "" ]; then
- AUTHC_CLIENT_CONFIGURER_ARG="-Dsolr.httpclient.builder.factory=$SOLR_AUTHENTICATION_CLIENT_CONFIGURER"
+ echo "WARNING: Found unsupported configuration variable SOLR_AUTHENTICATION_CLIENT_CONFIGURER"
+ echo " Please start using SOLR_AUTH_TYPE instead"
+fi
+if [ "$SOLR_AUTHENTICATION_CLIENT_BUILDER" != "" ]; then
+ AUTHC_CLIENT_BUILDER_ARG="-Dsolr.httpclient.builder.factory=$SOLR_AUTHENTICATION_CLIENT_BUILDER"
fi
-AUTHC_OPTS="$AUTHC_CLIENT_CONFIGURER_ARG $SOLR_AUTHENTICATION_OPTS"
+AUTHC_OPTS="$AUTHC_CLIENT_BUILDER_ARG $SOLR_AUTHENTICATION_OPTS"
# Set the SOLR_TOOL_HOST variable for use when connecting to a running Solr instance
if [ "$SOLR_HOST" != "" ]; then
@@ -282,15 +287,15 @@ fi
function print_usage() {
CMD="$1"
ERROR_MSG="$2"
-
+
if [ "$ERROR_MSG" != "" ]; then
echo -e "\nERROR: $ERROR_MSG\n"
fi
-
+
if [ -z "$CMD" ]; then
echo ""
echo "Usage: solr COMMAND OPTIONS"
- echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth"
+ echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert"
echo ""
echo " Standalone server example (start Solr running in the background on port 8984):"
echo ""
@@ -305,7 +310,7 @@ function print_usage() {
echo ""
elif [[ "$CMD" == "start" || "$CMD" == "restart" ]]; then
echo ""
- echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-a \"additional-options\"] [-V]"
+ echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a \"additional-options\"] [-V]"
echo ""
echo " -f Start Solr in foreground; default starts Solr in the background"
echo " and sends stdout / stderr to solr-PORT-console.log"
@@ -333,7 +338,11 @@ function print_usage() {
echo " while reusing the same server directory set using the -d parameter. If set, the"
echo " specified directory should contain a solr.xml file, unless solr.xml exists in Zookeeper."
echo " This parameter is ignored when running examples (-e), as the solr.solr.home depends"
- echo " on which example is run. The default value is server/solr."
+ echo " on which example is run. The default value is server/solr. If passed relative dir,"
+ echo " validation with current dir will be done, before trying default server/<dir>"
+ echo ""
+ echo " -t <dir> Sets the solr.data.home system property, where Solr will store data (index)."
+ echo " If not set, Solr uses solr.solr.home for config and data."
echo ""
echo " -e <example> Name of the example to run; available examples:"
echo " cloud: SolrCloud example"
@@ -346,6 +355,11 @@ function print_usage() {
echo " you could pass: -a \"-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18983\""
echo " In most cases, you should wrap the additional parameters in double quotes."
echo ""
+ echo " -j Additional parameters to pass to Jetty when starting Solr."
+ echo " For example, to add configuration folder that jetty should read"
+ echo " you could pass: -j \"--include-jetty-dir=/etc/jetty/custom/server/\""
+ echo " In most cases, you should wrap the additional parameters in double quotes."
+ echo ""
echo " -noprompt Don't prompt for input; accept all defaults when running examples that accept user input"
echo ""
echo " -v and -q Verbose (-v) or quiet (-q) logging. Sets default log level to DEBUG or WARN instead of INFO"
@@ -366,12 +380,14 @@ function print_usage() {
echo ""
elif [ "$CMD" == "healthcheck" ]; then
echo ""
- echo "Usage: solr healthcheck [-c collection] [-z zkHost]"
+ echo "Usage: solr healthcheck [-c collection] [-z zkHost] [-V]"
echo ""
echo " -c <collection> Collection to run healthcheck against."
echo ""
echo " -z <zkHost> Zookeeper connection string; default is localhost:9983"
echo ""
+ echo " -V Enable more verbose output"
+ echo ""
elif [ "$CMD" == "status" ]; then
echo ""
echo "Usage: solr status"
@@ -380,7 +396,7 @@ function print_usage() {
echo ""
elif [ "$CMD" == "create" ]; then
echo ""
- echo "Usage: solr create [-c name] [-d confdir] [-n configName] [-shards #] [-replicationFactor #] [-p port]"
+ echo "Usage: solr create [-c name] [-d confdir] [-n configName] [-shards #] [-replicationFactor #] [-p port] [-V]"
echo ""
echo " Create a core or collection depending on whether Solr is running in standalone (core) or SolrCloud"
echo " mode (collection). In other words, this action detects which mode Solr is running in, and then takes"
@@ -394,7 +410,7 @@ function print_usage() {
echo ""
elif [ "$CMD" == "delete" ]; then
echo ""
- echo "Usage: solr delete [-c name] [-deleteConfig true|false] [-p port]"
+ echo "Usage: solr delete [-c name] [-deleteConfig true|false] [-p port] [-V]"
echo ""
echo " Deletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud"
echo " mode (collection). If you're deleting a collection in SolrCloud mode, the default behavior is to also"
@@ -409,20 +425,21 @@ function print_usage() {
echo " If not specified, the script will search the local system for a running"
echo " Solr instance and will use the port of the first server it finds."
echo ""
+ echo " -V Enables more verbose output."
+ echo ""
elif [ "$CMD" == "create_core" ]; then
echo ""
- echo "Usage: solr create_core [-c core] [-d confdir] [-p port]"
+ echo "Usage: solr create_core [-c core] [-d confdir] [-p port] [-V]"
echo ""
echo " -c <core> Name of core to create"
echo ""
echo " -d <confdir> Configuration directory to copy when creating the new core, built-in options are:"
echo ""
- echo " basic_configs: Minimal Solr configuration"
- echo " data_driven_schema_configs: Managed schema with field-guessing support enabled"
+ echo " _default: Minimal configuration, which supports enabling/disabling field-guessing support"
echo " sample_techproducts_configs: Example configuration with many optional features enabled to"
echo " demonstrate the full power of Solr"
echo ""
- echo " If not specified, default is: data_driven_schema_configs"
+ echo " If not specified, default is: _default"
echo ""
echo " Alternatively, you can pass the path to your own configuration directory instead of using"
echo " one of the built-in configurations, such as: bin/solr create_core -c mycore -d /tmp/myconfig"
@@ -431,20 +448,21 @@ function print_usage() {
echo " If not specified, the script will search the local system for a running"
echo " Solr instance and will use the port of the first server it finds."
echo ""
+ echo " -V Enable more verbose output."
+ echo ""
elif [ "$CMD" == "create_collection" ]; then
echo ""
- echo "Usage: solr create_collection [-c collection] [-d confdir] [-n configName] [-shards #] [-replicationFactor #] [-p port]"
+ echo "Usage: solr create_collection [-c collection] [-d confdir] [-n configName] [-shards #] [-replicationFactor #] [-p port] [-V]"
echo ""
echo " -c <collection> Name of collection to create"
echo ""
echo " -d <confdir> Configuration directory to copy when creating the new collection, built-in options are:"
echo ""
- echo " basic_configs: Minimal Solr configuration"
- echo " data_driven_schema_configs: Managed schema with field-guessing support enabled"
+ echo " _default: Minimal configuration, which supports enabling/disabling field-guessing support"
echo " sample_techproducts_configs: Example configuration with many optional features enabled to"
echo " demonstrate the full power of Solr"
echo ""
- echo " If not specified, default is: data_driven_schema_configs"
+ echo " If not specified, default is: _default"
echo ""
echo " Alternatively, you can pass the path to your own configuration directory instead of using"
echo " one of the built-in configurations, such as: bin/solr create_collection -c mycoll -d /tmp/myconfig"
@@ -466,27 +484,31 @@ function print_usage() {
echo " If not specified, the script will search the local system for a running"
echo " Solr instance and will use the port of the first server it finds."
echo ""
+ echo " -V Enable more verbose output."
+ echo ""
elif [ "$CMD" == "zk" ]; then
print_short_zk_usage ""
echo " Be sure to check the Solr logs in case of errors."
echo ""
- echo " -z zkHost Optional Zookeeper connection string for all commands. If specified it"
+ echo " -z zkHost Optional Zookeeper connection string for all commands. If specified it"
echo " overrides the 'ZK_HOST=...'' defined in solr.in.sh."
echo ""
+ echo " -V Enable more verbose output."
+ echo ""
echo " upconfig uploads a configset from the local machine to Zookeeper. (Backcompat: -upconfig)"
echo ""
echo " downconfig downloads a configset from Zookeeper to the local machine. (Backcompat: -downconfig)"
echo ""
- echo " -n configName Name of the configset in Zookeeper that will be the destination of"
+ echo " -n configName Name of the configset in Zookeeper that will be the destination of"
echo " 'upconfig' and the source for 'downconfig'."
echo ""
- echo " -d confdir The local directory the configuration will be uploaded from for"
+ echo " -d confdir The local directory the configuration will be uploaded from for"
echo " 'upconfig' or downloaded to for 'downconfig'. If 'confdir' is a child of"
echo " ...solr/server/solr/configsets' then the configs will be copied from/to"
echo " that directory. Otherwise it is interpreted as a simple local path."
echo ""
echo " cp copies files or folders to/from Zookeeper or Zokeeper -> Zookeeper"
- echo " -r Recursively copy <src> to <dst>. Command will fail if <src> has children and "
+ echo " -r Recursively copy <src> to <dst>. Command will fail if <src> has children and "
echo " -r is not specified. Optional"
echo ""
echo " <src>, <dest> : [file:][/]path/to/local/file or zk:/path/to/zk/node"
@@ -516,9 +538,9 @@ function print_usage() {
echo " Wildcards are supported when copying from local, trailing only and must be quoted."
echo ""
echo " rm deletes files or folders on Zookeeper"
- echo " -r Recursively delete if <path> is a directory. Command will fail if <path>"
+ echo " -r Recursively delete if <path> is a directory. Command will fail if <path>"
echo " has children and -r is not specified. Optional"
- echo " <path> : [zk:]/path/to/zk/node. <path> may not be the root ('/')"
+ echo " <path> : [zk:]/path/to/zk/node. <path> may not be the root ('/')"
echo ""
echo " mv moves (renames) znodes on Zookeeper"
echo " <src>, <dest> : Zookeeper nodes, the 'zk:' prefix is optional."
@@ -542,22 +564,25 @@ function print_usage() {
echo ""
elif [ "$CMD" == "auth" ]; then
echo ""
- echo "Usage: solr auth enable [-type basicAuth] -credentials user:pass [-blockUnknown <true|false>] [-updateIncludeFileOnly <true|false>]"
- echo " solr auth enable [-type basicAuth] -prompt <true|false> [-blockUnknown <true|false>] [-updateIncludeFileOnly <true|false>]"
- echo " solr auth disable [-updateIncludeFileOnly <true|false>]"
+ echo "Usage: solr auth enable [-type basicAuth] -credentials user:pass [-blockUnknown <true|false>] [-updateIncludeFileOnly <true|false>] [-V]"
+ echo " solr auth enable [-type basicAuth] -prompt <true|false> [-blockUnknown <true|false>] [-updateIncludeFileOnly <true|false>] [-V]"
+ echo " solr auth enable -type kerberos -config \"<kerberos configs>\" [-updateIncludeFileOnly <true|false>] [-V]"
+ echo " solr auth disable [-updateIncludeFileOnly <true|false>] [-V]"
echo ""
- echo " -type <type> The authentication mechanism to enable. Defaults to 'basicAuth'."
+ echo " -type <type> The authentication mechanism (basicAuth or kerberos) to enable. Defaults to 'basicAuth'."
echo ""
- echo " -credentials <user:pass> The username and password of the initial user"
+ echo " -credentials <user:pass> The username and password of the initial user. Applicable for basicAuth only."
echo " Note: only one of -prompt or -credentials must be provided"
echo ""
- echo " -prompt <true|false> Prompts the user to provide the credentials"
+ echo " -config \"<configs>\" Configuration parameters (Solr startup parameters). Required and applicable only for Kerberos"
+ echo ""
+ echo " -prompt <true|false> Prompts the user to provide the credentials. Applicable for basicAuth only."
echo " Note: only one of -prompt or -credentials must be provided"
echo ""
echo " -blockUnknown <true|false> When true, this blocks out access to unauthenticated users. When not provided,"
echo " this defaults to false (i.e. unauthenticated users can access all endpoints, except the"
echo " operations like collection-edit, security-edit, core-admin-edit etc.). Check the reference"
- echo " guide for Basic Authentication for more details."
+ echo " guide for Basic Authentication for more details. Applicable for basicAuth only."
echo ""
echo " -updateIncludeFileOnly <true|false> Only update the solr.in.sh or solr.in.cmd file, and skip actual enabling/disabling"
echo " authentication (i.e. don't update security.json)"
@@ -569,6 +594,8 @@ function print_usage() {
echo " -s <dir> Specify the Solr home directory. This is where any credentials or authentication"
echo " configuration files (e.g. basicAuth.conf) would be placed."
echo ""
+ echo " -V Enable more verbose output."
+ echo ""
fi
} # end print_usage
@@ -621,7 +648,7 @@ function solr_pid_by_port() {
echo "$solrPID"
}
-# extract the value of the -Djetty.port parameter from a running Solr process
+# extract the value of the -Djetty.port parameter from a running Solr process
function jetty_port() {
SOLR_PID="$1"
SOLR_PROC=`ps auxww | grep -w $SOLR_PID | grep start\.jar | grep jetty\.port`
@@ -633,11 +660,11 @@ function jetty_port() {
local jetty_port="${pair[1]}"
break
fi
- done
+ done
echo "$jetty_port"
} # end jetty_port func
-# run a Solr command-line tool using the SolrCLI class;
+# run a Solr command-line tool using the SolrCLI class;
# useful for doing cross-platform work from the command-line using Java
function run_tool() {
@@ -696,19 +723,19 @@ function get_info() {
return $CODE
} # end get_info
-# tries to gracefully stop Solr using the Jetty
+# tries to gracefully stop Solr using the Jetty
# stop command and if that fails, then uses kill -9
function stop_solr() {
DIR="$1"
SOLR_PORT="$2"
- STOP_PORT=`expr $SOLR_PORT - 1000`
+ THIS_STOP_PORT="${STOP_PORT:-$(expr $SOLR_PORT - 1000)}"
STOP_KEY="$3"
SOLR_PID="$4"
if [ "$SOLR_PID" != "" ]; then
echo -e "Sending stop command to Solr running on port $SOLR_PORT ... waiting up to $SOLR_STOP_WAIT seconds to allow Jetty process $SOLR_PID to stop gracefully."
- "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS -jar "$DIR/start.jar" "STOP.PORT=$STOP_PORT" "STOP.KEY=$STOP_KEY" --stop || true
+ "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS -jar "$DIR/start.jar" "STOP.PORT=$THIS_STOP_PORT" "STOP.KEY=$STOP_KEY" --stop || true
(loops=0
while true
do
@@ -766,7 +793,7 @@ if [ $# -eq 1 ]; then
fi
if [ $# -gt 0 ]; then
- # if first arg starts with a dash (and it's not -help or -info),
+ # if first arg starts with a dash (and it's not -help or -info),
# then assume they are starting Solr, such as: solr -f
if [[ $1 == -* ]]; then
SCRIPT_CMD="start"
@@ -777,7 +804,7 @@ if [ $# -gt 0 ]; then
else
# no args - just show usage and exit
print_usage ""
- exit
+ exit
fi
if [ "$SCRIPT_CMD" == "status" ]; then
@@ -796,8 +823,10 @@ fi
# run a healthcheck and exit if requested
if [ "$SCRIPT_CMD" == "healthcheck" ]; then
+ VERBOSE=""
+
if [ $# -gt 0 ]; then
- while true; do
+ while true; do
case "$1" in
-c|-collection)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
@@ -807,7 +836,7 @@ if [ "$SCRIPT_CMD" == "healthcheck" ]; then
HEALTHCHECK_COLLECTION="$2"
shift 2
;;
- -z|-zkhost)
+ -z|-zkhost)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
print_usage "$SCRIPT_CMD" "ZooKeeper connection string is required when using the $1 option!"
exit 1
@@ -816,37 +845,41 @@ if [ "$SCRIPT_CMD" == "healthcheck" ]; then
shift 2
;;
-help|-usage)
- print_usage "$SCRIPT_CMD"
+ print_usage "$SCRIPT_CMD"
exit 0
;;
+ -V|--verbose)
+ VERBOSE="-verbose"
+ shift
+ ;;
--)
shift
break
;;
*)
- if [ "$1" != "" ]; then
+ if [ "$1" != "" ]; then
print_usage "$SCRIPT_CMD" "Unrecognized or misplaced argument: $1!"
exit 1
else
break # out-of-args, stop looping
- fi
+ fi
;;
esac
done
fi
-
+
if [ -z "$ZK_HOST" ]; then
ZK_HOST=localhost:9983
fi
-
+
if [ -z "$HEALTHCHECK_COLLECTION" ]; then
echo "collection parameter is required!"
print_usage "healthcheck"
- exit 1
+ exit 1
fi
-
- run_tool healthcheck -zkHost "$ZK_HOST" -collection "$HEALTHCHECK_COLLECTION"
-
+
+ run_tool healthcheck -zkHost "$ZK_HOST" -collection "$HEALTHCHECK_COLLECTION" $VERBOSE
+
exit $?
fi
@@ -856,6 +889,7 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM
CREATE_NUM_SHARDS=1
CREATE_REPFACT=1
FORCE=false
+ VERBOSE=""
if [ $# -gt 0 ]; then
while true; do
@@ -908,6 +942,10 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM
CREATE_PORT="$2"
shift 2
;;
+ -V|--verbose)
+ VERBOSE="-verbose"
+ shift
+ ;;
-force)
FORCE=true
shift
@@ -933,10 +971,10 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM
fi
if [ -z "$CREATE_CONFDIR" ]; then
- CREATE_CONFDIR='data_driven_schema_configs'
+ CREATE_CONFDIR='_default'
fi
- # validate the confdir arg
+ # validate the confdir arg (if provided)
if [[ ! -d "$SOLR_TIP/server/solr/configsets/$CREATE_CONFDIR" && ! -d "$CREATE_CONFDIR" ]]; then
echo -e "\nSpecified configuration directory $CREATE_CONFDIR not found!\n"
exit 1
@@ -948,11 +986,6 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM
exit 1
fi
- # If not defined, use the collection name for the name of the configuration in Zookeeper
- if [ -z "$CREATE_CONFNAME" ]; then
- CREATE_CONFNAME="$CREATE_NAME"
- fi
-
if [ -z "$CREATE_PORT" ]; then
for ID in `ps auxww | grep java | grep start\.jar | awk '{print $2}' | sort -r`
do
@@ -969,6 +1002,14 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM
exit 1
fi
+ if [[ "$CREATE_CONFDIR" == "_default" ]] && ([[ "$CREATE_CONFNAME" == "" ]] || [[ "$CREATE_CONFNAME" == "_default" ]]); then
+ echo "WARNING: Using _default configset. Data driven schema functionality is enabled by default, which is"
+ echo " NOT RECOMMENDED for production use."
+ echo
+ echo " To turn it off:"
+ echo " curl http://$SOLR_TOOL_HOST:$CREATE_PORT/solr/$CREATE_NAME/config -d '{\"set-user-property\": {\"update.autoCreateFields\":\"false\"}}'"
+ fi
+
if [[ "$(whoami)" == "root" ]] && [[ "$FORCE" == "false" ]] ; then
echo "WARNING: Creating cores as the root user can cause Solr to fail and is not advisable. Exiting."
echo " If you started Solr as root (not advisable either), force core creation by adding argument -force"
@@ -976,13 +1017,15 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM
fi
if [ "$SCRIPT_CMD" == "create_core" ]; then
run_tool create_core -name "$CREATE_NAME" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$CREATE_PORT/solr" \
- -confdir "$CREATE_CONFDIR" -configsetsDir "$SOLR_TIP/server/solr/configsets"
+ -confdir "$CREATE_CONFDIR" -configsetsDir "$SOLR_TIP/server/solr/configsets" \
+ $VERBOSE
exit $?
else
run_tool "$SCRIPT_CMD" -name "$CREATE_NAME" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$CREATE_PORT/solr" \
-shards "$CREATE_NUM_SHARDS" -replicationFactor "$CREATE_REPFACT" \
-confname "$CREATE_CONFNAME" -confdir "$CREATE_CONFDIR" \
- -configsetsDir "$SOLR_TIP/server/solr/configsets"
+ -configsetsDir "$SOLR_TIP/server/solr/configsets" \
+ $VERBOSE
exit $?
fi
fi
@@ -990,6 +1033,8 @@ fi
# delete a core or collection
if [[ "$SCRIPT_CMD" == "delete" ]]; then
+ VERBOSE=""
+
if [ $# -gt 0 ]; then
while true; do
case "$1" in
@@ -1017,6 +1062,10 @@ if [[ "$SCRIPT_CMD" == "delete" ]]; then
DELETE_CONFIG="$2"
shift 2
;;
+ -V|--verbose)
+ VERBOSE="-verbose"
+ shift
+ ;;
-help|-usage)
print_usage "$SCRIPT_CMD"
exit 0
@@ -1065,7 +1114,8 @@ if [[ "$SCRIPT_CMD" == "delete" ]]; then
fi
run_tool delete -name "$DELETE_NAME" -deleteConfig "$DELETE_CONFIG" \
- -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$DELETE_PORT/solr"
+ -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$DELETE_PORT/solr" \
+ $VERBOSE
exit $?
fi
@@ -1075,6 +1125,8 @@ ZK_RECURSE=false
# necessary for back-compat
if [[ "$SCRIPT_CMD" == "zk" ]]; then
+ VERBOSE=""
+
if [ $# -gt 0 ]; then
while true; do
case "$1" in
@@ -1111,6 +1163,10 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
ZK_RECURSE="true"
shift
;;
+ -V|--verbose)
+ VERBOSE="-verbose"
+ shift
+ ;;
-help|-usage|-h)
print_usage "$SCRIPT_CMD"
exit 0
@@ -1174,34 +1230,34 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
case "$ZK_OP" in
upconfig)
- run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" -configsetsDir "$SOLR_TIP/server/solr/configsets"
+ run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" -configsetsDir "$SOLR_TIP/server/solr/configsets" $VERBOSE
;;
downconfig)
- run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST"
+ run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" $VERBOSE
;;
rm)
if [ -z "$ZK_SRC" ]; then
print_short_zk_usage "Zookeeper path to remove must be specified when using the 'rm' command"
fi
- run_tool "$ZK_OP" -path "$ZK_SRC" -zkHost "$ZK_HOST" -recurse "$ZK_RECURSE"
+ run_tool "$ZK_OP" -path "$ZK_SRC" -zkHost "$ZK_HOST" -recurse "$ZK_RECURSE" $VERBOSE
;;
mv)
- run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" -zkHost "$ZK_HOST"
+ run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" -zkHost "$ZK_HOST" $VERBOSE
;;
cp)
- run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" -zkHost "$ZK_HOST" -recurse "$ZK_RECURSE"
+ run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" -zkHost "$ZK_HOST" -recurse "$ZK_RECURSE" $VERBOSE
;;
ls)
if [ -z "$ZK_SRC" ]; then
print_short_zk_usage "Zookeeper path to list must be specified when using the 'ls' command"
fi
- run_tool "$ZK_OP" -path "$ZK_SRC" -recurse "$ZK_RECURSE" -zkHost "$ZK_HOST"
+ run_tool "$ZK_OP" -path "$ZK_SRC" -recurse "$ZK_RECURSE" -zkHost "$ZK_HOST" $VERBOSE
;;
mkroot)
if [ -z "$ZK_SRC" ]; then
print_short_zk_usage "Zookeeper path to list must be specified when using the 'mkroot' command"
fi
- run_tool "$ZK_OP" -path "$ZK_SRC" -zkHost "$ZK_HOST"
+ run_tool "$ZK_OP" -path "$ZK_SRC" -zkHost "$ZK_HOST" $VERBOSE
;;
*)
print_short_zk_usage "Unrecognized Zookeeper operation $ZK_OP"
@@ -1212,6 +1268,9 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
fi
if [[ "$SCRIPT_CMD" == "auth" ]]; then
+
+ VERBOSE=""
+
declare -a AUTH_PARAMS
if [ $# -gt 0 ]; then
while true; do
@@ -1236,6 +1295,11 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-credentials" "$AUTH_CREDENTIALS")
shift 2
;;
+ -config)
+ AUTH_CONFIG="`echo $2| base64`"
+ AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-config" "$AUTH_CONFIG")
+ shift 2
+ ;;
-solrIncludeFile)
SOLR_INCLUDE="$2"
shift 2
@@ -1254,6 +1318,10 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
shift
break
;;
+ -V|--verbose)
+ VERBOSE="-verbose"
+ shift
+ ;;
-d|-dir)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
print_usage "$SCRIPT_CMD" "Server directory is required when using the $1 option!"
@@ -1306,14 +1374,15 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
echo -e "\nSolr server directory $SOLR_SERVER_DIR not found!\n"
exit 1
fi
+
if [ -z "$SOLR_HOME" ]; then
SOLR_HOME="$SOLR_SERVER_DIR/solr"
- else
- if [[ $SOLR_HOME != /* ]] && [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
+ elif [[ $SOLR_HOME != /* ]]; then
+ if [[ -d "`pwd`/$SOLR_HOME" ]]; then
+ SOLR_HOME="$(pwd)/$SOLR_HOME"
+ elif [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME"
SOLR_PID_DIR="$SOLR_HOME"
- elif [[ $SOLR_HOME != /* ]] && [[ -d "`pwd`/$SOLR_HOME" ]]; then
- SOLR_HOME="$(pwd)/$SOLR_HOME"
fi
fi
@@ -1334,7 +1403,7 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
fi
done
fi
- run_tool auth ${AUTH_PARAMS[@]} -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$AUTH_PORT/solr" -authConfDir "$SOLR_HOME"
+ run_tool auth ${AUTH_PARAMS[@]} -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$AUTH_PORT/solr" -authConfDir "$SOLR_HOME" $VERBOSE
exit $?
fi
@@ -1345,6 +1414,40 @@ if [ "$SCRIPT_CMD" != "stop" ] && [ "$SCRIPT_CMD" != "start" ] && [ "$SCRIPT_CMD
exit 1
fi
+#Check current Ulimits for Open Files and Max Processes. Warn if they are below the recommended values.
+
+if [ -z "$SOLR_RECOMMENDED_MAX_PROCESSES" ]; then
+ SOLR_RECOMMENDED_MAX_PROCESSES=65000
+fi
+
+if [ -z "$SOLR_RECOMMENDED_OPEN_FILES" ]; then
+ SOLR_RECOMMENDED_OPEN_FILES=65000
+fi
+
+if [ -z "$SOLR_ULIMIT_CHECKS" ] || [ "$SOLR_ULIMIT_CHECKS" != "false" ]; then
+ if [ "$SCRIPT_CMD" == "start" ] || [ "$SCRIPT_CMD" == "restart" ] || [ "$SCRIPT_CMD" == "status" ]; then
+ if hash ulimit 2>/dev/null; then
+ openFiles=$(ulimit -n)
+ maxProcs=$(ulimit -u)
+ if [ $openFiles -lt "$SOLR_RECOMMENDED_OPEN_FILES" ]; then
+ echo "*** [WARN] *** Your open file limit is currently $openFiles. "
+ echo " It should be set to $SOLR_RECOMMENDED_OPEN_FILES to avoid operational disruption. "
+ echo " If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh"
+ fi
+
+ if [ $maxProcs -lt "$SOLR_RECOMMENDED_MAX_PROCESSES" ]; then
+ echo "*** [WARN] *** Your Max Processes Limit is currently $maxProcs. "
+ echo " It should be set to $SOLR_RECOMMENDED_MAX_PROCESSES to avoid operational disruption. "
+ echo " If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh"
+ fi
+ else
+ echo "Could not check ulimits for processes and open files, recommended values are"
+ echo " max processes: $SOLR_RECOMMENDED_MAX_PROCESSES "
+ echo " open files: $SOLR_RECOMMENDED_OPEN_FILES"
+ fi
+ fi
+fi
+
# Run in foreground (default is to run in the background)
FG="false"
FORCE=false
@@ -1353,7 +1456,7 @@ SOLR_OPTS=($SOLR_OPTS)
PASS_TO_RUN_EXAMPLE=
if [ $# -gt 0 ]; then
- while true; do
+ while true; do
case "$1" in
-c|-cloud)
SOLR_MODE="solrcloud"
@@ -1389,6 +1492,10 @@ if [ $# -gt 0 ]; then
SOLR_HOME="$2"
shift 2
;;
+ -t|-data.home)
+ SOLR_DATA_HOME="$2"
+ shift 2
+ ;;
-e|-example)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
print_usage "$SCRIPT_CMD" "Example name is required when using the $1 option!"
@@ -1443,6 +1550,11 @@ if [ $# -gt 0 ]; then
PASS_TO_RUN_EXAMPLE+=" -a \"$ADDITIONAL_CMD_OPTS\""
shift 2
;;
+ -j|-jettyconfig)
+ ADDITIONAL_JETTY_CONFIG="$2"
+ PASS_TO_RUN_EXAMPLE+=" -j \"$ADDITIONAL_JETTY_CONFIG\""
+ shift 2
+ ;;
-k|-key)
STOP_KEY="$2"
shift 2
@@ -1475,6 +1587,7 @@ if [ $# -gt 0 ]; then
;;
-force)
FORCE=true
+ PASS_TO_RUN_EXAMPLE+=" -force"
shift
;;
--)
@@ -1642,15 +1755,20 @@ fi
if [ -z "$SOLR_HOME" ]; then
SOLR_HOME="$SOLR_SERVER_DIR/solr"
-else
- if [[ $SOLR_HOME != /* ]] && [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
+elif [[ $SOLR_HOME != /* ]]; then
+ if [[ -d "`pwd`/$SOLR_HOME" ]]; then
+ SOLR_HOME="$(pwd)/$SOLR_HOME"
+ elif [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME"
SOLR_PID_DIR="$SOLR_HOME"
- elif [[ $SOLR_HOME != /* ]] && [[ -d "`pwd`/$SOLR_HOME" ]]; then
- SOLR_HOME="$(pwd)/$SOLR_HOME"
fi
fi
+# Set the default configset dir to be bootstrapped as _default
+if [ -z "$DEFAULT_CONFDIR" ]; then
+ DEFAULT_CONFDIR="$SOLR_SERVER_DIR/solr/configsets/_default/conf"
+fi
+
# This is quite hacky, but examples rely on a different log4j.properties
# so that we can write logs for examples to $SOLR_HOME/../logs
if [ -z "$SOLR_LOGS_DIR" ]; then
@@ -1678,6 +1796,10 @@ if [ ! -e "$SOLR_HOME" ]; then
echo -e "\nSolr home directory $SOLR_HOME not found!\n"
exit 1
fi
+if [[ $SOLR_DATA_HOME ]] && [ ! -e "$SOLR_DATA_HOME" ]; then
+ echo -e "\nSolr data home directory $SOLR_DATA_HOME not found!\n"
+ exit 1
+fi
if $verbose ; then
q=""
else
@@ -1732,9 +1854,9 @@ if [ "$SOLR_MODE" == 'solrcloud' ]; then
if [ -z "$ZK_CLIENT_TIMEOUT" ]; then
ZK_CLIENT_TIMEOUT="15000"
fi
-
+
CLOUD_MODE_OPTS=("-DzkClientTimeout=$ZK_CLIENT_TIMEOUT")
-
+
if [ "$ZK_HOST" != "" ]; then
CLOUD_MODE_OPTS+=("-DzkHost=$ZK_HOST")
else
@@ -1749,7 +1871,7 @@ if [ "$SOLR_MODE" == 'solrcloud' ]; then
if [ -e "$SOLR_HOME/collection1/core.properties" ]; then
CLOUD_MODE_OPTS+=('-Dbootstrap_confdir=./solr/collection1/conf' '-Dcollection.configName=myconf' '-DnumShards=1')
fi
-
+
else
if [ ! -e "$SOLR_HOME/solr.xml" ]; then
echo -e "\nSolr home directory $SOLR_HOME must contain a solr.xml file!\n"
@@ -1806,8 +1928,9 @@ function launch_solr() {
run_in_foreground="$1"
stop_port="$STOP_PORT"
-
+
SOLR_ADDL_ARGS="$2"
+ SOLR_JETTY_ADDL_CONFIG="$3"
# define default GC_TUNE
if [ -z ${GC_TUNE+x} ]; then
@@ -1830,9 +1953,13 @@ function launch_solr() {
GC_TUNE=($GC_TUNE)
fi
+ if [ -n "$SOLR_WAIT_FOR_ZK" ]; then
+ WAIT_FOR_ZK_PROP="-DwaitForZk=$SOLR_WAIT_FOR_ZK"
+ SOLR_OPTS+=($WAIT_FOR_ZK_PROP)
+ fi
# If SSL-related system props are set, add them to SOLR_OPTS
- if [ -n "$SOLR_SSL_OPTS" ]; then
+ if [ "$SOLR_SSL_ENABLED" ]; then
# If using SSL and solr.jetty.https.port not set explicitly, use the jetty.port
SSL_PORT_PROP="-Dsolr.jetty.https.port=$SOLR_PORT"
SOLR_OPTS+=($SOLR_SSL_OPTS "$SSL_PORT_PROP")
@@ -1877,12 +2004,15 @@ function launch_solr() {
echo -e " SOLR_LOG_LEVEL = $SOLR_LOG_LEVEL"
fi
+ if [ "$SOLR_DATA_HOME" != "" ]; then
+ echo -e " SOLR_DATA_HOME = $SOLR_DATA_HOME"
+ fi
echo -e "\n"
fi
-
+
# need to launch solr from the server dir
cd "$SOLR_SERVER_DIR"
-
+
if [ ! -e "$SOLR_SERVER_DIR/start.jar" ]; then
echo -e "\nERROR: start.jar file not found in $SOLR_SERVER_DIR!\nPlease check your -d parameter to set the correct Solr server directory.\n"
exit 1
@@ -1892,8 +2022,8 @@ function launch_solr() {
"${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" $SOLR_LOG_LEVEL_OPT -Dsolr.log.dir="$SOLR_LOGS_DIR" \
"-Djetty.port=$SOLR_PORT" "-DSTOP.PORT=$stop_port" "-DSTOP.KEY=$STOP_KEY" \
"${SOLR_HOST_ARG[@]}" "-Duser.timezone=$SOLR_TIMEZONE" \
- "-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
- "${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}")
+ "-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.data.home=$SOLR_DATA_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
+ "-Dsolr.default.confdir=$DEFAULT_CONFDIR" "${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}")
if [ "$SOLR_MODE" == "solrcloud" ]; then
IN_CLOUD_MODE=" in SolrCloud mode"
@@ -1916,12 +2046,12 @@ function launch_solr() {
esac
if [ "$run_in_foreground" == "true" ]; then
- exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -Dsolr.kerberos.name.rules="$SOLR_KERB_NAME_RULES" -jar start.jar "${SOLR_JETTY_CONFIG[@]}"
+ exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -Dsolr.kerberos.name.rules="$SOLR_KERB_NAME_RULES" -jar start.jar "${SOLR_JETTY_CONFIG[@]}" $SOLR_JETTY_ADDL_CONFIG
else
# run Solr in the background
nohup "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -Dsolr.kerberos.name.rules="$SOLR_KERB_NAME_RULES" -Dsolr.log.muteconsole \
"-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" \
- -jar start.jar "${SOLR_JETTY_CONFIG[@]}" \
+ -jar start.jar "${SOLR_JETTY_CONFIG[@]}" $SOLR_JETTY_ADDL_CONFIG \
1>"$SOLR_LOGS_DIR/solr-$SOLR_PORT-console.log" 2>&1 & echo $! > "$SOLR_PID_DIR/solr-$SOLR_PORT.pid"
# check if /proc/sys/kernel/random/entropy_avail exists then check output of cat /proc/sys/kernel/random/entropy_avail to see if less than 300
@@ -1965,6 +2095,6 @@ function launch_solr() {
fi
}
-launch_solr "$FG" "$ADDITIONAL_CMD_OPTS"
+launch_solr "$FG" "$ADDITIONAL_CMD_OPTS" "$ADDITIONAL_JETTY_CONFIG"
-exit $?
+exit $?
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-solr-client/build.xml b/ambari-infra/ambari-infra-solr-client/build.xml
index d9d3a6c..725cb62 100644
--- a/ambari-infra/ambari-infra-solr-client/build.xml
+++ b/ambari-infra/ambari-infra-solr-client/build.xml
@@ -27,12 +27,21 @@
<target name="package">
<delete dir="target/package"/>
+ <delete dir="target/migrate"/>
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset dir="target/libs"/>
</copy>
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset file="target/*.jar"/>
</copy>
+ <mkdir dir="target/migrate"/>
+ <get src="${lucene6-core.url}" dest="target/migrate/${lucene6-core-jar.name}" usetimestamp="true"/>
+ <get src="${lucene6-backward-codecs.url}" dest="target/migrate/${lucene6-backward-codecs-jar.name}" usetimestamp="true"/>
+ <copy todir="target/package/migrate" includeEmptyDirs="no">
+ <fileset file="target/migrate/*.jar"/>
+ <fileset file="target/package/libs/lucene-core-${solr.version}.jar"/>
+ <fileset file="target/package/libs/lucene-backward-codecs-${solr.version}.jar"/>
+ </copy>
<copy todir="target/package" includeEmptyDirs="no">
<fileset file="src/main/resources/solrCloudCli.sh"/>
<fileset file="src/main/resources/solrIndexHelper.sh"/>
diff --git a/ambari-infra/ambari-infra-solr-client/pom.xml b/ambari-infra/ambari-infra-solr-client/pom.xml
index 8782a97..a7dbbbc 100644
--- a/ambari-infra/ambari-infra-solr-client/pom.xml
+++ b/ambari-infra/ambari-infra-solr-client/pom.xml
@@ -29,6 +29,14 @@
<artifactId>ambari-infra-solr-client</artifactId>
+ <properties>
+ <lucene6.version>6.6.0</lucene6.version>
+ <lucene6-core-jar.name>lucene-core-${lucene6.version}.jar</lucene6-core-jar.name>
+ <lucene6-core.url>http://central.maven.org/maven2/org/apache/lucene/lucene-core/${lucene6.version}/${lucene6-core-jar.name}</lucene6-core.url>
+ <lucene6-backward-codecs-jar.name>lucene-backward-codecs-${lucene6.version}.jar</lucene6-backward-codecs-jar.name>
+ <lucene6-backward-codecs.url>http://central.maven.org/maven2/org/apache/lucene/lucene-backward-codecs/${lucene6.version}/${lucene6-backward-codecs-jar.name}</lucene6-backward-codecs.url>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.solr</groupId>
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
index 95708d7..d4567e8 100644
--- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
+++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java
@@ -151,9 +151,9 @@ public class AmbariSolrCloudCLI {
.argName("my_new_shard")
.build();
- final Option disableShardingOption = Option.builder("ns")
- .longOpt("no-sharding")
- .desc("Sharding not used when creating collection")
+ final Option implicitRoutingOption = Option.builder("ir")
+ .longOpt("implicit-routing")
+ .desc("Use implicit routing when creating a collection")
.build();
final Option zkConnectStringOption = Option.builder("z")
@@ -350,7 +350,7 @@ public class AmbariSolrCloudCLI {
options.addOption(routerNameOption);
options.addOption(routerFieldOption);
options.addOption(shardNameOption);
- options.addOption(disableShardingOption);
+ options.addOption(implicitRoutingOption);
options.addOption(createCollectionOption);
options.addOption(downloadConfigOption);
options.addOption(uploadConfigurationOption);
@@ -445,7 +445,7 @@ public class AmbariSolrCloudCLI {
String routerName = cli.hasOption("rn") ? cli.getOptionValue("rn") : null;
String routerField = cli.hasOption("rf") ? cli.getOptionValue("rf") : null;
String shardName = cli.hasOption("sn") ? cli.getOptionValue("sn") : null;
- boolean isSplitting = !cli.hasOption("ns");
+ boolean implicitRouting = cli.hasOption("ir");
String jaasFile = cli.hasOption("jf") ? cli.getOptionValue("jf") : null;
String keyStoreLocation = cli.hasOption("ksl") ? cli.getOptionValue("ksl") : null;
String keyStorePassword = cli.hasOption("ksp") ? cli.getOptionValue("ksp") : null;
@@ -472,7 +472,7 @@ public class AmbariSolrCloudCLI {
.withRouterName(routerName)
.withRouterField(routerField)
.withJaasFile(jaasFile) // call before creating SolrClient
- .withSplitting(isSplitting)
+ .isImplicitRouting(implicitRouting)
.withSolrZkClient(ZK_CLIENT_TIMEOUT, ZK_CLIENT_CONNECT_TIMEOUT)
.withKeyStoreLocation(keyStoreLocation)
.withKeyStorePassword(keyStorePassword)
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
index 96c07a3..70c590f 100644
--- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
+++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java
@@ -64,7 +64,7 @@ public class AmbariSolrCloudClient {
private final int maxShardsPerNode;
private final String routerName;
private final String routerField;
- private final boolean splitting;
+ private final boolean implicitRouting;
private final String jaasFile;
private final String znode;
private final String saslUsers;
@@ -88,7 +88,7 @@ public class AmbariSolrCloudClient {
this.maxShardsPerNode = builder.maxShardsPerNode;
this.routerName = builder.routerName;
this.routerField = builder.routerField;
- this.splitting = builder.splitting;
+ this.implicitRouting = builder.implicitRouting;
this.znode = builder.znode;
this.saslUsers = builder.saslUsers;
this.propName = builder.propName;
@@ -114,7 +114,7 @@ public class AmbariSolrCloudClient {
LOG.info("Collection '{}' creation request sent.", collection);
} else {
LOG.info("Collection '{}' already exits.", getCollection());
- if (this.isSplitting()) {
+ if (this.isImplicitRouting()) {
createShard(null);
}
}
@@ -317,8 +317,8 @@ public class AmbariSolrCloudClient {
return routerField;
}
- public boolean isSplitting() {
- return splitting;
+ public boolean isImplicitRouting() {
+ return implicitRouting;
}
public String getJaasFile() {
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java
index a187cdb..bf482c7 100644
--- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java
+++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClientBuilder.java
@@ -20,8 +20,7 @@
package org.apache.ambari.infra.solr;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.solr.common.cloud.SolrZkClient;
public class AmbariSolrCloudClientBuilder {
@@ -31,7 +30,9 @@ public class AmbariSolrCloudClientBuilder {
private static final String TRUSTSTORE_LOCATION_ARG = "javax.net.ssl.trustStore";
private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword";
private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType";
-
+ private static final String JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
+ private static final String SOLR_HTTPCLIENT_BUILDER_FACTORY = "solr.httpclient.builder.factory";
+
String zkConnectString;
String collection;
String configSet;
@@ -45,7 +46,7 @@ public class AmbariSolrCloudClientBuilder {
String routerField = "_router_field_";
CloudSolrClient solrCloudClient;
SolrZkClient solrZkClient;
- boolean splitting;
+ boolean implicitRouting;
String jaasFile;
String znode;
String saslUsers;
@@ -113,8 +114,8 @@ public class AmbariSolrCloudClientBuilder {
return this;
}
- public AmbariSolrCloudClientBuilder withSplitting(boolean splitting) {
- this.splitting = splitting;
+ public AmbariSolrCloudClientBuilder isImplicitRouting(boolean implicitRouting) {
+ this.implicitRouting = implicitRouting;
return this;
}
@@ -208,8 +209,8 @@ public class AmbariSolrCloudClientBuilder {
private void setupSecurity(String jaasFile) {
if (jaasFile != null) {
- System.setProperty("java.security.auth.login.config", jaasFile);
- HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer());
+ System.setProperty(JAVA_SECURITY_AUTH_LOGIN_CONFIG, jaasFile);
+ System.setProperty(SOLR_HTTPCLIENT_BUILDER_FACTORY, Krb5HttpClientBuilder.class.getCanonicalName());
}
}
}
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/CreateCollectionCommand.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/CreateCollectionCommand.java
index 317d0b6..5d296ae 100644
--- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/CreateCollectionCommand.java
+++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/CreateCollectionCommand.java
@@ -38,12 +38,10 @@ public class CreateCollectionCommand extends AbstractSolrRetryCommand<Collection
public CollectionAdminRequest.Create createRequest(AmbariSolrCloudClient client) {
CollectionAdminRequest.Create request =
CollectionAdminRequest.createCollection(client.getCollection(), client.getConfigSet(), client.getShards(), client.getReplication());
- request.setMaxShardsPerNode(client.getMaxShardsPerNode());;
- if (client.getRouterField() != null && client.getRouterName()!= null) {
+ request.setMaxShardsPerNode(client.getMaxShardsPerNode());
+ if (client.isImplicitRouting()) {
request.setRouterName(client.getRouterName());
request.setRouterField(client.getRouterField());
- }
- if (client.isSplitting()) {
request.setShards(ShardUtils.generateShardListStr(client.getMaxShardsPerNode()));
}
return request;
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
index 6e817ec..8959740 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
+++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
@@ -41,7 +41,13 @@ function print_help() {
-d, --index-data-dir <DIRECTORY> Location of the solr cores (e.g.: /opt/ambari_infra_solr/data)
-c, --core-filter <FILTER1,FILTER2> Comma separated name filters of core directoies (default: hadoop_logs,audit_logs,history)
-f, --force Force to start index upgrade, even is the version is at least 6.
+ -v, --version Lucene version to upgrade (default: 6.6.0, available: 6.6.0, 7.2.1)
+ run-check-index-tool arguments:
+ -v, --version Lucene version to upgrade (default: 6.6.0, available: 6.6.0, 7.2.1)
+
+ run-upgrade-index-tool arguments:
+ -v, --version Lucene version to upgrade (default: 6.6.0, available: 6.6.0, 7.2.1)
EOF
}
@@ -49,23 +55,25 @@ function upgrade_core() {
local INDEX_DIR=${1:?"usage: <index_base_dir> e.g.: /opt/ambari_infra_solr/data"}
local FORCE_UPDATE=${2:?"usage <force_update_flag> e.g.: true"}
local SOLR_CORE_FILTERS=${3:?"usage: <comma separated core filters> e.g.: hadoop_logs,audit_logs,history"}
-
+ local LUCENE_VERSION=${4:?"usage <lucene_index_version> e.g.: 7.2.1"}
SOLR_CORE_FILTER_ARR=$(echo $SOLR_CORE_FILTERS | sed "s/,/ /g")
+ local version_prefix="$(echo $LUCENE_VERSION | head -c 1)"
+
for coll in $SOLR_CORE_FILTER_ARR; do
if [[ "$1" == *"$coll"* ]]; then
echo "Core '$1' dir name contains $coll (core filter)'";
- version=$(PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex -fast $1|grep " version="|sed -e 's/.*=//g'|head -1)
+ version=$(PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$LUCENE_VERSION.jar:$DIR/migrate/lucene-backward-codecs-$LUCENE_VERSION.jar" org.apache.lucene.index.CheckIndex -fast $1|grep " version="|sed -e 's/.*=//g'|head -1)
if [ -z $version ] ; then
echo "Core '$1' - Empty index?"
return
fi
majorVersion=$(echo $version|cut -c 1)
- if [ $majorVersion -ge 6 ] && [ $FORCE_UPDATE == "false" ] ; then
+ if [ $majorVersion -ge $version_prefix ] && [ $FORCE_UPDATE == "false" ] ; then
echo "Core '$1' - Already on version $version, not upgrading. Use -f or --force option to run upgrade anyway."
else
echo "Core '$1' - Index version is $version, upgrading ..."
- PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader -delete-prior-commits $1
+ PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$LUCENE_VERSION.jar:$DIR/migrate/lucene-backward-codecs-$LUCENE_VERSION.jar" org.apache.lucene.index.IndexUpgrader -delete-prior-commits $1
echo "Upgrading core '$1' has finished"
fi
fi
@@ -89,6 +97,10 @@ function upgrade_index() {
local INDEX_DIR="$2"
shift 2
;;
+ -v|--version)
+ local LUCENE_VERSION="$2"
+ shift 2
+ ;;
*)
echo "Unknown option: $1"
exit 1
@@ -104,6 +116,10 @@ function upgrade_index() {
SOLR_CORE_FILTERS="hadoop_logs,audit_logs,history"
fi
+ if [[ -z "$LUCENE_VERSION" ]] ; then
+ LUCENE_VERSION="6.6.0"
+ fi
+
if [[ -z "$FORCE_UPDATE" ]] ; then
FORCE_UPDATE="false"
else
@@ -120,7 +136,7 @@ function upgrade_index() {
abspath=$(cd "$(dirname "$c")"; pwd)/$(basename "$c")
find $c/data -maxdepth 1 -type d -name 'index*' | while read indexDir; do
echo "Checking core $name - $abspath"
- upgrade_core "$indexDir" "$FORCE_UPDATE" "$SOLR_CORE_FILTERS"
+ upgrade_core "$indexDir" "$FORCE_UPDATE" "$SOLR_CORE_FILTERS" "$LUCENE_VERSION"
done
else
echo "No index folder found for $name"
@@ -132,11 +148,41 @@ function upgrade_index() {
function upgrade_index_tool() {
# see: https://cwiki.apache.org/confluence/display/solr/IndexUpgrader+Tool
- PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader ${@}
+ luc_version = "6.6.0"
+ while [[ $# -gt 0 ]]
+ do
+ key="$1"
+ case $key in
+ -v|--version)
+ luc_version="$2"
+ shift 2
+ ;;
+ *)
+ echo "Unknown option: $1"
+ exit 1
+ ;;
+ esac
+ done
+ PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$luc_version.jar:$DIR/migrate/lucene-backward-codecs-$luc_version.jar" org.apache.lucene.index.IndexUpgrader ${@}
}
function check_index_tool() {
- PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex ${@}
+ luc_version = "6.6.0"
+ while [[ $# -gt 0 ]]
+ do
+ key="$1"
+ case $key in
+ -v|--version)
+ luc_version="$2"
+ shift 2
+ ;;
+ *)
+ echo "Unknown option: $1"
+ exit 1
+ ;;
+ esac
+ done
+ PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$luc_version.jar:$DIR/migrate/lucene-backward-codecs-$luc_version.jar" org.apache.lucene.index.CheckIndex ${@}
}
function main() {
diff --git a/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraRuleBasedAuthorizationPlugin.java b/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraRuleBasedAuthorizationPlugin.java
deleted file mode 100644
index 2f1a558..0000000
--- a/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraRuleBasedAuthorizationPlugin.java
+++ /dev/null
@@ -1,542 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.infra.security;
-
-import com.google.common.collect.ImmutableSet;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.security.Principal;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.security.AuthorizationContext;
-import org.apache.solr.security.AuthorizationPlugin;
-import org.apache.solr.security.AuthorizationResponse;
-import org.apache.solr.security.ConfigEditablePlugin;
-import org.apache.solr.util.CommandOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static java.util.Collections.singleton;
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.common.util.Utils.getDeepCopy;
-import static org.apache.solr.handler.admin.SecurityConfHandler.getListValue;
-import static org.apache.solr.handler.admin.SecurityConfHandler.getMapValue;
-
-/**
- * Modified copy of solr.RuleBasedAuthorizationPlugin to handle role - permission mappings with KereberosPlugin
- * Added 2 new JSON map: (precedence: user-host-regex > user-host)
- * 1. "user-host": user host mappings (array) for hostname validation
- * 2. "user-host-regex": user host regex mapping (string) for hostname validation
- */
-public class InfraRuleBasedAuthorizationPlugin implements AuthorizationPlugin, ConfigEditablePlugin {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private final Map<String, Set<String>> usersVsRoles = new HashMap<>();
- private final Map<String, WildCardSupportMap> mapping = new HashMap<>();
- private final List<Permission> permissions = new ArrayList<>();
- private final Map<String, Set<String>> userVsHosts = new HashMap<>();
- private final Map<String, String> userVsHostRegex = new HashMap<>();
-
- private final InfraUserRolesLookupStrategy infraUserRolesLookupStrategy = new InfraUserRolesLookupStrategy();
- private final InfraKerberosHostValidator infraKerberosDomainValidator = new InfraKerberosHostValidator();
-
- private static class WildCardSupportMap extends HashMap<String, List<Permission>> {
- final Set<String> wildcardPrefixes = new HashSet<>();
-
- @Override
- public List<Permission> put(String key, List<Permission> value) {
- if (key != null && key.endsWith("/*")) {
- key = key.substring(0, key.length() - 2);
- wildcardPrefixes.add(key);
- }
- return super.put(key, value);
- }
-
- @Override
- public List<Permission> get(Object key) {
- List<Permission> result = super.get(key);
- if (key == null || result != null) return result;
- if (!wildcardPrefixes.isEmpty()) {
- for (String s : wildcardPrefixes) {
- if (key.toString().startsWith(s)) {
- List<Permission> l = super.get(s);
- if (l != null) {
- result = result == null ? new ArrayList<Permission>() : new ArrayList<Permission>(result);
- result.addAll(l);
- }
- }
- }
- }
- return result;
- }
- }
-
- @Override
- public AuthorizationResponse authorize(AuthorizationContext context) {
- List<AuthorizationContext.CollectionRequest> collectionRequests = context.getCollectionRequests();
- if (context.getRequestType() == AuthorizationContext.RequestType.ADMIN) {
- MatchStatus flag = checkCollPerm(mapping.get(null), context);
- return flag.rsp;
- }
-
- for (AuthorizationContext.CollectionRequest collreq : collectionRequests) {
- //check permissions for each collection
- MatchStatus flag = checkCollPerm(mapping.get(collreq.collectionName), context);
- if (flag != MatchStatus.NO_PERMISSIONS_FOUND) return flag.rsp;
- }
- //check wildcard (all=*) permissions.
- MatchStatus flag = checkCollPerm(mapping.get("*"), context);
- return flag.rsp;
- }
-
- private MatchStatus checkCollPerm(Map<String, List<Permission>> pathVsPerms,
- AuthorizationContext context) {
- if (pathVsPerms == null) return MatchStatus.NO_PERMISSIONS_FOUND;
-
- String path = context.getResource();
- MatchStatus flag = checkPathPerm(pathVsPerms.get(path), context);
- if (flag != MatchStatus.NO_PERMISSIONS_FOUND) return flag;
- return checkPathPerm(pathVsPerms.get(null), context);
- }
-
- private MatchStatus checkPathPerm(List<Permission> permissions, AuthorizationContext context) {
- if (permissions == null || permissions.isEmpty()) return MatchStatus.NO_PERMISSIONS_FOUND;
- Principal principal = context.getUserPrincipal();
- loopPermissions:
- for (int i = 0; i < permissions.size(); i++) {
- Permission permission = permissions.get(i);
- if (permission.method != null && !permission.method.contains(context.getHttpMethod())) {
- //this permissions HTTP method does not match this rule. try other rules
- continue;
- }
- if(permission.predicate != null){
- if(!permission.predicate.test(context)) continue ;
- }
-
- if (permission.params != null) {
- for (Map.Entry<String, Object> e : permission.params.entrySet()) {
- String paramVal = context.getParams().get(e.getKey());
- Object val = e.getValue();
- if (val instanceof List) {
- if (!((List) val).contains(paramVal)) continue loopPermissions;
- } else if (!Objects.equals(val, paramVal)) continue loopPermissions;
- }
- }
-
- if (permission.role == null) {
- //no role is assigned permission.That means everybody is allowed to access
- return MatchStatus.PERMITTED;
- }
- if (principal == null) {
- log.info("request has come without principal. failed permission {} ",permission);
- //this resource needs a principal but the request has come without
- //any credential.
- return MatchStatus.USER_REQUIRED;
- } else if (permission.role.contains("*")) {
- return MatchStatus.PERMITTED;
- }
-
- for (String role : permission.role) {
- Set<String> userRoles = infraUserRolesLookupStrategy.getUserRolesFromPrincipal(usersVsRoles, principal);
- boolean validHostname = infraKerberosDomainValidator.validate(principal, userVsHosts, userVsHostRegex);
- if (!validHostname) {
- log.warn("Hostname is not valid for principal {}", principal);
- return MatchStatus.FORBIDDEN;
- }
- if (userRoles != null && userRoles.contains(role)) return MatchStatus.PERMITTED;
- }
- log.info("This resource is configured to have a permission {}, The principal {} does not have the right role ", permission, principal);
- return MatchStatus.FORBIDDEN;
- }
- log.debug("No permissions configured for the resource {} . So allowed to access", context.getResource());
- return MatchStatus.NO_PERMISSIONS_FOUND;
- }
-
- @Override
- public void init(Map<String, Object> initInfo) {
- mapping.put(null, new WildCardSupportMap());
- Map<String, Object> map = getMapValue(initInfo, "user-role");
- for (Object o : map.entrySet()) {
- Map.Entry e = (Map.Entry) o;
- String roleName = (String) e.getKey();
- usersVsRoles.put(roleName, readValueAsSet(map, roleName));
- }
- List<Map> perms = getListValue(initInfo, "permissions");
- for (Map o : perms) {
- Permission p;
- try {
- p = Permission.load(o);
- } catch (Exception exp) {
- log.error("Invalid permission ", exp);
- continue;
- }
- permissions.add(p);
- add2Mapping(p);
- }
- // adding user-host
- Map<String, Object> userHostsMap = getMapValue(initInfo, "user-host");
- for (Object userHost : userHostsMap.entrySet()) {
- Map.Entry e = (Map.Entry) userHost;
- String roleName = (String) e.getKey();
- userVsHosts.put(roleName, readValueAsSet(userHostsMap, roleName));
- }
- // adding user-host-regex
- Map<String, Object> userHostRegexMap = getMapValue(initInfo, "user-host-regex");
- for (Map.Entry<String, Object> entry : userHostRegexMap.entrySet()) {
- userVsHostRegex.put(entry.getKey(), entry.getValue().toString());
- }
-
- }
-
- //this is to do optimized lookup of permissions for a given collection/path
- private void add2Mapping(Permission permission) {
- for (String c : permission.collections) {
- WildCardSupportMap m = mapping.get(c);
- if (m == null) mapping.put(c, m = new WildCardSupportMap());
- for (String path : permission.path) {
- List<Permission> perms = m.get(path);
- if (perms == null) m.put(path, perms = new ArrayList<>());
- perms.add(permission);
- }
- }
- }
-
- /**
- * read a key value as a set. if the value is a single string ,
- * return a singleton set
- *
- * @param m the map from which to lookup
- * @param key the key with which to do lookup
- */
- static Set<String> readValueAsSet(Map m, String key) {
- Set<String> result = new HashSet<>();
- Object val = m.get(key);
- if (val == null) {
- if("collection".equals(key)){
- //for collection collection: null means a core admin/ collection admin request
- // otherwise it means a request where collection name is ignored
- return m.containsKey(key) ? singleton((String) null) : singleton("*");
- }
- return null;
- }
- if (val instanceof Collection) {
- Collection list = (Collection) val;
- for (Object o : list) result.add(String.valueOf(o));
- } else if (val instanceof String) {
- result.add((String) val);
- } else {
- throw new RuntimeException("Bad value for : " + key);
- }
- return result.isEmpty() ? null : Collections.unmodifiableSet(result);
- }
-
- @Override
- public void close() throws IOException { }
-
- static class Permission {
- String name;
- Set<String> path, role, collections, method;
- Map<String, Object> params;
- Predicate<AuthorizationContext> predicate;
- Map originalConfig;
-
- private Permission() {
- }
-
- static Permission load(Map m) {
- Permission p = new Permission();
- p.originalConfig = new LinkedHashMap<>(m);
- String name = (String) m.get(NAME);
- if (!m.containsKey("role")) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "role not specified");
- p.role = readValueAsSet(m, "role");
- if (well_known_permissions.containsKey(name)) {
- HashSet<String> disAllowed = new HashSet<>(knownKeys);
- disAllowed.remove("role");//these are the only
- disAllowed.remove(NAME);//allowed keys for well-known permissions
- for (String s : disAllowed) {
- if (m.containsKey(s))
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, s + " is not a valid key for the permission : " + name);
- }
- p.predicate = (Predicate<AuthorizationContext>) ((Map) well_known_permissions.get(name)).get(Predicate.class.getName());
- m = well_known_permissions.get(name);
- }
- p.name = name;
- p.path = readSetSmart(name, m, "path");
- p.collections = readSetSmart(name, m, "collection");
- p.method = readSetSmart(name, m, "method");
- p.params = (Map<String, Object>) m.get("params");
- return p;
- }
-
- @Override
- public String toString() {
- return Utils.toJSONString(originalConfig);
- }
-
- static final Set<String> knownKeys = ImmutableSet.of("collection", "role", "params", "path", "method", NAME);
- }
-
- enum MatchStatus {
- USER_REQUIRED(AuthorizationResponse.PROMPT),
- NO_PERMISSIONS_FOUND(AuthorizationResponse.OK),
- PERMITTED(AuthorizationResponse.OK),
- FORBIDDEN(AuthorizationResponse.FORBIDDEN);
-
- final AuthorizationResponse rsp;
-
- MatchStatus(AuthorizationResponse rsp) {
- this.rsp = rsp;
- }
- }
-
- /**
- * This checks for the defaults available other rules for the keys
- */
- private static Set<String> readSetSmart(String permissionName, Map m, String key) {
- Set<String> set = readValueAsSet(m, key);
- if (set == null && well_known_permissions.containsKey(permissionName)) {
- set = readValueAsSet((Map) well_known_permissions.get(permissionName), key);
- }
- if ("method".equals(key)) {
- if (set != null) {
- for (String s : set) if (!HTTP_METHODS.contains(s)) return null;
- }
- return set;
- }
- return set == null ? singleton((String)null) : set;
- }
-
- @Override
- public Map<String, Object> edit(Map<String, Object> latestConf, List<CommandOperation> commands) {
- for (CommandOperation op : commands) {
- OPERATION operation = null;
- for (OPERATION o : OPERATION.values()) {
- if (o.name.equals(op.name)) {
- operation = o;
- break;
- }
- }
- if (operation == null) {
- op.unknownOperation();
- return null;
- }
- latestConf = operation.edit(latestConf, op);
- if (latestConf == null) return null;
-
- }
- return latestConf;
- }
-
- enum OPERATION {
- SET_USER_ROLE("set-user-role") {
- @Override
- public Map<String, Object> edit(Map<String, Object> latestConf, CommandOperation op) {
- Map<String, Object> roleMap = getMapValue(latestConf, "user-role");
- Map<String, Object> map = op.getDataMap();
- if (op.hasError()) return null;
- for (Map.Entry<String, Object> e : map.entrySet()) {
- if (e.getValue() == null) {
- roleMap.remove(e.getKey());
- continue;
- }
- if (e.getValue() instanceof String || e.getValue() instanceof List) {
- roleMap.put(e.getKey(), e.getValue());
- } else {
- op.addError("Unexpected value ");
- return null;
- }
- }
- return latestConf;
- }
- },
- SET_PERMISSION("set-permission") {
- @Override
- public Map<String, Object> edit(Map<String, Object> latestConf, CommandOperation op) {
- String name = op.getStr(NAME);
- Map<String, Object> dataMap = op.getDataMap();
- if (op.hasError()) return null;
- dataMap = getDeepCopy(dataMap, 3);
- String before = (String) dataMap.remove("before");
- for (String key : dataMap.keySet()) {
- if (!Permission.knownKeys.contains(key)) op.addError("Unknown key, " + key);
- }
- try {
- Permission.load(dataMap);
- } catch (Exception e) {
- op.addError(e.getMessage());
- return null;
- }
- List<Map> permissions = getListValue(latestConf, "permissions");
- List<Map> permissionsCopy = new ArrayList<>();
- boolean added = false;
- for (Map e : permissions) {
- Object n = e.get(NAME);
- if (n.equals(before) || n.equals(name)) {
- added = true;
- permissionsCopy.add(dataMap);
- }
- if (!n.equals(name)) permissionsCopy.add(e);
- }
- if (!added && before != null) {
- op.addError("Invalid 'before' :" + before);
- return null;
- }
- if (!added) permissionsCopy.add(dataMap);
- latestConf.put("permissions", permissionsCopy);
- return latestConf;
- }
- },
- UPDATE_PERMISSION("update-permission") {
- @Override
- public Map<String, Object> edit(Map<String, Object> latestConf, CommandOperation op) {
- String name = op.getStr(NAME);
- if (op.hasError()) return null;
- for (Map permission : (List<Map>) getListValue(latestConf, "permissions")) {
- if (name.equals(permission.get(NAME))) {
- LinkedHashMap copy = new LinkedHashMap<>(permission);
- copy.putAll(op.getDataMap());
- op.setCommandData(copy);
- return SET_PERMISSION.edit(latestConf, op);
- }
- }
- op.addError("No such permission " + name);
- return null;
- }
- },
- DELETE_PERMISSION("delete-permission") {
- @Override
- public Map<String, Object> edit(Map<String, Object> latestConf, CommandOperation op) {
- List<String> names = op.getStrs("");
- if (names == null || names.isEmpty()) {
- op.addError("Invalid command");
- return null;
- }
- names = new ArrayList<>(names);
- List<Map> copy = new ArrayList<>();
- List<Map> p = getListValue(latestConf, "permissions");
- for (Map map : p) {
- Object n = map.get(NAME);
- if (names.contains(n)) {
- names.remove(n);
- continue;
- } else {
- copy.add(map);
- }
- }
- if (!names.isEmpty()) {
- op.addError("Unknown permission name(s) " + names);
- return null;
- }
- latestConf.put("permissions", copy);
- return latestConf;
- }
- };
-
- public abstract Map<String, Object> edit(Map<String, Object> latestConf, CommandOperation op);
-
- public final String name;
-
- OPERATION(String s) {
- this.name = s;
- }
-
- public static OPERATION get(String name) {
- for (OPERATION o : values()) if (o.name.equals(name)) return o;
- return null;
- }
- }
-
- public static final Set<String> HTTP_METHODS = ImmutableSet.of("GET", "POST", "DELETE", "PUT", "HEAD");
-
- private static final Map<String, Map<String,Object>> well_known_permissions = (Map) Utils.fromJSONString(
- " { " +
- " security-edit :{" +
- " path:['/admin/authentication','/admin/authorization']," +
- " collection:null," +
- " method:POST }," +
- " security-read :{" +
- " path:['/admin/authentication','/admin/authorization']," +
- " collection:null," +
- " method:GET}," +
- " schema-edit :{" +
- " method:POST," +
- " path:'/schema/*'}," +
- " collection-admin-edit :{" +
- " collection:null," +
- " path:'/admin/collections'}," +
- " collection-admin-read :{" +
- " collection:null," +
- " path:'/admin/collections'}," +
- " schema-read :{" +
- " method:GET," +
- " path:'/schema/*'}," +
- " config-read :{" +
- " method:GET," +
- " path:'/config/*'}," +
- " update :{" +
- " path:'/update/*'}," +
- " read :{" +
- " path:['/select', '/get','/browse','/tvrh','/terms','/clustering','/elevate', '/export','/spell','/clustering']}," +
- " config-edit:{" +
- " method:POST," +
- " path:'/config/*'}," +
- " all:{collection:['*', null]}" +
- "}");
-
- static {
- ((Map) well_known_permissions.get("collection-admin-edit")).put(Predicate.class.getName(), getCollectionActionPredicate(true));
- ((Map) well_known_permissions.get("collection-admin-read")).put(Predicate.class.getName(), getCollectionActionPredicate(false));
- }
-
- private static Predicate<AuthorizationContext> getCollectionActionPredicate(final boolean isEdit) {
- return new Predicate<AuthorizationContext>() {
- @Override
- public boolean test(AuthorizationContext context) {
- String action = context.getParams().get("action");
- if (action == null) return false;
- CollectionParams.CollectionAction collectionAction = CollectionParams.CollectionAction.get(action);
- if (collectionAction == null) return false;
- return isEdit ? collectionAction.isWrite : !collectionAction.isWrite;
- }
- };
- }
-
-
- public static void main(String[] args) {
- System.out.println(Utils.toJSONString(well_known_permissions));
-
- }
-
- public interface Predicate<T> {
-
- boolean test(T t);
- }
-}
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraKerberosHostValidator.java b/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraKerberosHostValidator.java
similarity index 98%
rename from ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraKerberosHostValidator.java
rename to ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraKerberosHostValidator.java
index 4a47a89..606adca 100644
--- a/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraKerberosHostValidator.java
+++ b/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraKerberosHostValidator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.security;
+package org.apache.solr.security;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
diff --git a/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraRuleBasedAuthorizationPlugin.java b/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraRuleBasedAuthorizationPlugin.java
new file mode 100644
index 0000000..cc55d7d
--- /dev/null
+++ b/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraRuleBasedAuthorizationPlugin.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.solr.security;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.security.Principal;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+import org.apache.solr.common.util.CommandOperation;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.common.util.ValidatingJsonMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singleton;
+import static java.util.Collections.unmodifiableMap;
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.solr.handler.admin.SecurityConfHandler.getListValue;
+import static org.apache.solr.handler.admin.SecurityConfHandler.getMapValue;
+
+/**
+ * Modified copy of solr.RuleBasedAuthorizationPlugin to handle role - permission mappings with KereberosPlugin
+ * Added 2 new JSON map: (precedence: user-host-regex > user-host)
+ * 1. "user-host": user host mappings (array) for hostname validation
+ * 2. "user-host-regex": user host regex mapping (string) for hostname validation
+ */
+public class InfraRuleBasedAuthorizationPlugin extends RuleBasedAuthorizationPlugin {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final Map<String, Set<String>> usersVsRoles = new HashMap<>();
+ private final Map<String, WildCardSupportMap> mapping = new HashMap<>();
+ private final List<Permission> permissions = new ArrayList<>();
+ private final Map<String, Set<String>> userVsHosts = new HashMap<>();
+ private final Map<String, String> userVsHostRegex = new HashMap<>();
+
+ private InfraKerberosHostValidator infraKerberosHostValidator = new InfraKerberosHostValidator();
+ private InfraUserRolesLookupStrategy infraUserRolesLookupStrategy = new InfraUserRolesLookupStrategy();
+
+
+ private static class WildCardSupportMap extends HashMap<String, List<Permission>> {
+ final Set<String> wildcardPrefixes = new HashSet<>();
+
+ @Override
+ public List<Permission> put(String key, List<Permission> value) {
+ if (key != null && key.endsWith("/*")) {
+ key = key.substring(0, key.length() - 2);
+ wildcardPrefixes.add(key);
+ }
+ return super.put(key, value);
+ }
+
+ @Override
+ public List<Permission> get(Object key) {
+ List<Permission> result = super.get(key);
+ if (key == null || result != null) return result;
+ if (!wildcardPrefixes.isEmpty()) {
+ for (String s : wildcardPrefixes) {
+ if (key.toString().startsWith(s)) {
+ List<Permission> l = super.get(s);
+ if (l != null) {
+ result = result == null ? new ArrayList<>() : new ArrayList<>(result);
+ result.addAll(l);
+ }
+ }
+ }
+ }
+ return result;
+ }
+ }
+
+ @Override
+ public AuthorizationResponse authorize(AuthorizationContext context) {
+ List<AuthorizationContext.CollectionRequest> collectionRequests = context.getCollectionRequests();
+ if (context.getRequestType() == AuthorizationContext.RequestType.ADMIN) {
+ InfraRuleBasedAuthorizationPlugin.MatchStatus flag = checkCollPerm(mapping.get(null), context);
+ return flag.rsp;
+ }
+
+ for (AuthorizationContext.CollectionRequest collreq : collectionRequests) {
+ //check permissions for each collection
+ InfraRuleBasedAuthorizationPlugin.MatchStatus flag = checkCollPerm(mapping.get(collreq.collectionName), context);
+ if (flag != InfraRuleBasedAuthorizationPlugin.MatchStatus.NO_PERMISSIONS_FOUND) return flag.rsp;
+ }
+ //check wildcard (all=*) permissions.
+ InfraRuleBasedAuthorizationPlugin.MatchStatus flag = checkCollPerm(mapping.get("*"), context);
+ return flag.rsp;
+ }
+
+ private InfraRuleBasedAuthorizationPlugin.MatchStatus checkCollPerm(Map<String, List<Permission>> pathVsPerms,
+ AuthorizationContext context) {
+ if (pathVsPerms == null) return InfraRuleBasedAuthorizationPlugin.MatchStatus.NO_PERMISSIONS_FOUND;
+
+ String path = context.getResource();
+ InfraRuleBasedAuthorizationPlugin.MatchStatus flag = checkPathPerm(pathVsPerms.get(path), context);
+ if (flag != InfraRuleBasedAuthorizationPlugin.MatchStatus.NO_PERMISSIONS_FOUND) return flag;
+ return checkPathPerm(pathVsPerms.get(null), context);
+ }
+
+ private InfraRuleBasedAuthorizationPlugin.MatchStatus checkPathPerm(List<Permission> permissions, AuthorizationContext context) {
+ if (permissions == null || permissions.isEmpty()) return InfraRuleBasedAuthorizationPlugin.MatchStatus.NO_PERMISSIONS_FOUND;
+ Principal principal = context.getUserPrincipal();
+ loopPermissions:
+ for (int i = 0; i < permissions.size(); i++) {
+ Permission permission = permissions.get(i);
+ if (PermissionNameProvider.values.containsKey(permission.name)) {
+ if (context.getHandler() instanceof PermissionNameProvider) {
+ PermissionNameProvider handler = (PermissionNameProvider) context.getHandler();
+ PermissionNameProvider.Name permissionName = handler.getPermissionName(context);
+ if (permissionName == null || !permission.name.equals(permissionName.name)) {
+ continue;
+ }
+ } else {
+ //all is special. it can match any
+ if(permission.wellknownName != PermissionNameProvider.Name.ALL) continue;
+ }
+ } else {
+ if (permission.method != null && !permission.method.contains(context.getHttpMethod())) {
+ //this permissions HTTP method does not match this rule. try other rules
+ continue;
+ }
+ if (permission.params != null) {
+ for (Map.Entry<String, Function<String[], Boolean>> e : permission.params.entrySet()) {
+ String[] paramVal = context.getParams().getParams(e.getKey());
+ if(!e.getValue().apply(paramVal)) continue loopPermissions;
+ }
+ }
+ }
+
+ if (permission.role == null) {
+ //no role is assigned permission.That means everybody is allowed to access
+ return InfraRuleBasedAuthorizationPlugin.MatchStatus.PERMITTED;
+ }
+ if (principal == null) {
+ log.info("request has come without principal. failed permission {} ",permission);
+ //this resource needs a principal but the request has come without
+ //any credential.
+ return InfraRuleBasedAuthorizationPlugin.MatchStatus.USER_REQUIRED;
+ } else if (permission.role.contains("*")) {
+ return InfraRuleBasedAuthorizationPlugin.MatchStatus.PERMITTED;
+ }
+
+ for (String role : permission.role) {
+ Set<String> userRoles = infraUserRolesLookupStrategy.getUserRolesFromPrincipal(usersVsRoles, principal);
+ boolean validHostname = infraKerberosHostValidator.validate(principal, userVsHosts, userVsHostRegex);
+ if (!validHostname) {
+ log.warn("Hostname is not valid for principal {}", principal);
+ return MatchStatus.FORBIDDEN;
+ }
+ if (userRoles != null && userRoles.contains(role)) return MatchStatus.PERMITTED;
+ }
+ log.info("This resource is configured to have a permission {}, The principal {} does not have the right role ", permission, principal);
+ return InfraRuleBasedAuthorizationPlugin.MatchStatus.FORBIDDEN;
+ }
+ log.debug("No permissions configured for the resource {} . So allowed to access", context.getResource());
+ return InfraRuleBasedAuthorizationPlugin.MatchStatus.NO_PERMISSIONS_FOUND;
+ }
+
+ @Override
+ public void init(Map<String, Object> initInfo) {
+ mapping.put(null, new InfraRuleBasedAuthorizationPlugin.WildCardSupportMap());
+ Map<String, Object> map = getMapValue(initInfo, "user-role");
+ for (Object o : map.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ String roleName = (String) e.getKey();
+ usersVsRoles.put(roleName, Permission.readValueAsSet(map, roleName));
+ }
+ List<Map> perms = getListValue(initInfo, "permissions");
+ for (Map o : perms) {
+ Permission p;
+ try {
+ p = Permission.load(o);
+ } catch (Exception exp) {
+ log.error("Invalid permission ", exp);
+ continue;
+ }
+ permissions.add(p);
+ add2Mapping(p);
+ // adding user-host
+ Map<String, Object> userHostsMap = getMapValue(initInfo, "user-host");
+ for (Object userHost : userHostsMap.entrySet()) {
+ Map.Entry e = (Map.Entry) userHost;
+ String roleName = (String) e.getKey();
+ userVsHosts.put(roleName, readValueAsSet(userHostsMap, roleName));
+ }
+ // adding user-host-regex
+ Map<String, Object> userHostRegexMap = getMapValue(initInfo, "user-host-regex");
+ for (Map.Entry<String, Object> entry : userHostRegexMap.entrySet()) {
+ userVsHostRegex.put(entry.getKey(), entry.getValue().toString());
+ }
+ }
+ }
+
+ /**
+ * read a key value as a set. if the value is a single string ,
+ * return a singleton set
+ *
+ * @param m the map from which to lookup
+ * @param key the key with which to do lookup
+ */
+ static Set<String> readValueAsSet(Map m, String key) {
+ Set<String> result = new HashSet<>();
+ Object val = m.get(key);
+ if (val == null) {
+ if("collection".equals(key)){
+ //for collection collection: null means a core admin/ collection admin request
+ // otherwise it means a request where collection name is ignored
+ return m.containsKey(key) ? singleton((String) null) : singleton("*");
+ }
+ return null;
+ }
+ if (val instanceof Collection) {
+ Collection list = (Collection) val;
+ for (Object o : list) result.add(String.valueOf(o));
+ } else if (val instanceof String) {
+ result.add((String) val);
+ } else {
+ throw new RuntimeException("Bad value for : " + key);
+ }
+ return result.isEmpty() ? null : Collections.unmodifiableSet(result);
+ }
+
+ //this is to do optimized lookup of permissions for a given collection/path
+ private void add2Mapping(Permission permission) {
+ for (String c : permission.collections) {
+ InfraRuleBasedAuthorizationPlugin.WildCardSupportMap m = mapping.get(c);
+ if (m == null) mapping.put(c, m = new InfraRuleBasedAuthorizationPlugin.WildCardSupportMap());
+ for (String path : permission.path) {
+ List<Permission> perms = m.get(path);
+ if (perms == null) m.put(path, perms = new ArrayList<>());
+ perms.add(permission);
+ }
+ }
+ }
+
+
+ @Override
+ public void close() throws IOException { }
+
+ enum MatchStatus {
+ USER_REQUIRED(AuthorizationResponse.PROMPT),
+ NO_PERMISSIONS_FOUND(AuthorizationResponse.OK),
+ PERMITTED(AuthorizationResponse.OK),
+ FORBIDDEN(AuthorizationResponse.FORBIDDEN);
+
+ final AuthorizationResponse rsp;
+
+ MatchStatus(AuthorizationResponse rsp) {
+ this.rsp = rsp;
+ }
+ }
+
+
+
+ @Override
+ public Map<String, Object> edit(Map<String, Object> latestConf, List<CommandOperation> commands) {
+ for (CommandOperation op : commands) {
+ AutorizationEditOperation operation = ops.get(op.name);
+ if (operation == null) {
+ op.unknownOperation();
+ return null;
+ }
+ latestConf = operation.edit(latestConf, op);
+ if (latestConf == null) return null;
+
+ }
+ return latestConf;
+ }
+
+ private static final Map<String, AutorizationEditOperation> ops = unmodifiableMap(asList(AutorizationEditOperation.values()).stream().collect(toMap(AutorizationEditOperation::getOperationName, identity())));
+
+
+ @Override
+ public ValidatingJsonMap getSpec() {
+ return Utils.getSpec("cluster.security.InfraRuleBasedAuthorization").getSpec();
+ }
+}
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraUserRolesLookupStrategy.java b/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraUserRolesLookupStrategy.java
similarity index 97%
rename from ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraUserRolesLookupStrategy.java
rename to ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraUserRolesLookupStrategy.java
index a54e4ad..119a339 100644
--- a/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/ambari/infra/security/InfraUserRolesLookupStrategy.java
+++ b/ambari-infra/ambari-infra-solr-plugin/src/main/java/org/apache/solr/security/InfraUserRolesLookupStrategy.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.security;
+package org.apache.solr.security;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
diff --git a/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraKerberosHostValidatorTest.java b/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraKerberosHostValidatorTest.java
similarity index 98%
rename from ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraKerberosHostValidatorTest.java
rename to ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraKerberosHostValidatorTest.java
index 828f09a..b550515 100644
--- a/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraKerberosHostValidatorTest.java
+++ b/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraKerberosHostValidatorTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.security;
+package org.apache.solr.security;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.junit.Before;
diff --git a/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraRuleBasedAuthorizationPluginTest.java b/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraRuleBasedAuthorizationPluginTest.java
similarity index 87%
rename from ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraRuleBasedAuthorizationPluginTest.java
rename to ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraRuleBasedAuthorizationPluginTest.java
index f1f842d..1348fd3 100644
--- a/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraRuleBasedAuthorizationPluginTest.java
+++ b/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraRuleBasedAuthorizationPluginTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.security;
+package org.apache.solr.security;
import java.security.Principal;
import java.util.Enumeration;
@@ -28,9 +28,9 @@ import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.security.AuthorizationContext;
+import org.apache.solr.handler.UpdateRequestHandler;
+import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.security.AuthorizationContext.RequestType;
-import org.apache.solr.security.AuthorizationResponse;
import org.junit.Test;
import static java.util.Collections.singletonList;
@@ -82,52 +82,53 @@ public class InfraRuleBasedAuthorizationPluginTest {
int FORBIDDEN = 403;
int PROMPT_FOR_CREDENTIALS = 401;
- checkRules(makeMap("resource", "/#",
- "httpMethod", "POST",
- "userPrincipal", "unknownuser",
- "collectionRequests", "freeforall" )
- , STATUS_OK);
-
checkRules(makeMap("resource", "/update/json/docs",
"httpMethod", "POST",
"userPrincipal", "tim",
- "collectionRequests", "mycoll")
+ "collectionRequests", "mycoll",
+ "handler", new UpdateRequestHandler())
, FORBIDDEN);
checkRules(makeMap("resource", "/update/json/docs",
"httpMethod", "POST",
"userPrincipal", "logsearch",
- "collectionRequests", "mycoll")
+ "collectionRequests", "mycoll",
+ "handler", new UpdateRequestHandler())
, STATUS_OK);
checkRules(makeMap("resource", "/update/json/docs",
"httpMethod", "GET",
"userPrincipal", "rangeradmin",
- "collectionRequests", "mycoll")
+ "collectionRequests", "mycoll",
+ "handler", new UpdateRequestHandler())
, FORBIDDEN);
checkRules(makeMap("resource", "/update/json/docs",
"httpMethod", "GET",
"userPrincipal", "rangeradmin",
- "collectionRequests", "mycoll2")
+ "collectionRequests", "mycoll2",
+ "handler", new UpdateRequestHandler())
, STATUS_OK);
checkRules(makeMap("resource", "/update/json/docs",
"httpMethod", "GET",
"userPrincipal", "logsearch",
- "collectionRequests", "mycoll2")
+ "collectionRequests", "mycoll2",
+ "handler", new UpdateRequestHandler())
, FORBIDDEN);
checkRules(makeMap("resource", "/update/json/docs",
"httpMethod", "POST",
"userPrincipal", "kms",
- "collectionRequests", "mycoll2")
+ "collectionRequests", "mycoll2",
+ "handler", new UpdateRequestHandler())
, STATUS_OK);
checkRules(makeMap("resource", "/admin/collections",
"userPrincipal", "tim",
"requestType", RequestType.ADMIN,
"collectionRequests", null,
+ "handler" , new CollectionsHandler(),
"params", new MapSolrParams(singletonMap("action", "CREATE")))
, FORBIDDEN);
@@ -135,6 +136,7 @@ public class InfraRuleBasedAuthorizationPluginTest {
"userPrincipal", null,
"requestType", RequestType.ADMIN,
"collectionRequests", null,
+ "handler" , new CollectionsHandler(),
"params", new MapSolrParams(singletonMap("action", "CREATE")))
, PROMPT_FOR_CREDENTIALS);
@@ -142,6 +144,7 @@ public class InfraRuleBasedAuthorizationPluginTest {
"userPrincipal", "rangeradmin",
"requestType", RequestType.ADMIN,
"collectionRequests", null,
+ "handler" , new CollectionsHandler(),
"params", new MapSolrParams(singletonMap("action", "CREATE")))
, STATUS_OK);
@@ -149,6 +152,7 @@ public class InfraRuleBasedAuthorizationPluginTest {
"userPrincipal", "kms",
"requestType", RequestType.ADMIN,
"collectionRequests", null,
+ "handler" , new CollectionsHandler(),
"params", new MapSolrParams(singletonMap("action", "CREATE")))
, FORBIDDEN);
@@ -156,6 +160,7 @@ public class InfraRuleBasedAuthorizationPluginTest {
"userPrincipal", "kms",
"requestType", RequestType.ADMIN,
"collectionRequests", null,
+ "handler" , new CollectionsHandler(),
"params", new MapSolrParams(singletonMap("action", "LIST")))
, STATUS_OK);
@@ -163,6 +168,7 @@ public class InfraRuleBasedAuthorizationPluginTest {
"userPrincipal", "rangeradmin",
"requestType", RequestType.ADMIN,
"collectionRequests", null,
+ "handler" , new CollectionsHandler(),
"params", new MapSolrParams(singletonMap("action", "LIST")))
, STATUS_OK);
}
@@ -245,7 +251,8 @@ public class InfraRuleBasedAuthorizationPluginTest {
@Override
public Object getHandler() {
- return null;
+ Object handler = values.get("handler");
+ return handler instanceof String ? (PermissionNameProvider) request -> PermissionNameProvider.Name.get((String) handler) : handler;
}
}
diff --git a/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraUserRolesLookupStrategyTest.java b/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraUserRolesLookupStrategyTest.java
similarity index 98%
rename from ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraUserRolesLookupStrategyTest.java
rename to ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraUserRolesLookupStrategyTest.java
index c1a47d1..8f56875 100644
--- a/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/ambari/infra/security/InfraUserRolesLookupStrategyTest.java
+++ b/ambari-infra/ambari-infra-solr-plugin/src/test/java/org/apache/solr/security/InfraUserRolesLookupStrategyTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.security;
+package org.apache.solr.security;
import com.google.common.collect.Sets;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
diff --git a/ambari-infra/pom.xml b/ambari-infra/pom.xml
index aab1727..335a0d4 100644
--- a/ambari-infra/pom.xml
+++ b/ambari-infra/pom.xml
@@ -25,7 +25,7 @@
<properties>
<jdk.version>1.8</jdk.version>
- <solr.version>6.6.2</solr.version>
+ <solr.version>7.2.1</solr.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<python.ver>python >= 2.6</python.ver>
<deb.python.ver>python (>= 2.6)</deb.python.ver>
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
index b1290a4..b5eafd7 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
@@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>5.0.0</luceneMatchVersion>
+ <luceneMatchVersion>7.2.1</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
@@ -233,11 +233,11 @@
will be allowed before they are merged into one.
Default is 10 for both merge policies.
-->
- <!--
- <mergeFactor>10</mergeFactor>
- -->
<!-- LogSearch customization. Set to 5 to trigger purging of deleted documents more often -->
- <mergeFactor>5</mergeFactor>
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+ <int name="maxMergeAtOnce">5</int>
+ <int name="segmentsPerTier">5</int>
+ </mergePolicyFactory>
<!-- Expert: Merge Scheduler
The Merge Scheduler in Lucene controls how merges are
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml
index f0e46a0..acbb4c6 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/solrconfig.xml
@@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>5.0.0</luceneMatchVersion>
+ <luceneMatchVersion>7.2.1</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
@@ -233,11 +233,11 @@
will be allowed before they are merged into one.
Default is 10 for both merge policies.
-->
- <!--
- <mergeFactor>10</mergeFactor>
- -->
<!-- LogSearch customization. Set to 5 to trigger purging of deleted documents more often -->
- <mergeFactor>5</mergeFactor>
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+ <int name="maxMergeAtOnce">5</int>
+ <int name="segmentsPerTier">5</int>
+ </mergePolicyFactory>
<!-- Expert: Merge Scheduler
The Merge Scheduler in Lucene controls how merges are
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml
index 1827444..514442d 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml
@@ -15,63 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-
-<!--
- For more details about configurations options that may appear in
- this file, see http://wiki.apache.org/solr/SolrConfigXml.
--->
<config>
- <!-- In all configuration below, a prefix of "solr." for class names
- is an alias that causes solr to search appropriate packages,
- including org.apache.solr.(search|update|request|core|analysis)
-
- You may also specify a fully qualified Java classname if you
- have your own custom plugins.
- -->
-
- <!-- Controls what version of Lucene various components of Solr
- adhere to. Generally, you want to use the latest version to
- get all bug fixes and improvements. It is highly recommended
- that you fully re-index after changing this setting as it can
- affect both how text is indexed and queried.
- -->
- <luceneMatchVersion>5.0.0</luceneMatchVersion>
-
- <!-- <lib/> directives can be used to instruct Solr to load any Jars
- identified and use them to resolve any "plugins" specified in
- your solrconfig.xml or schema.xml (ie: Analyzers, Request
- Handlers, etc...).
-
- All directories and paths are resolved relative to the
- instanceDir.
-
- Please note that <lib/> directives are processed in the order
- that they appear in your solrconfig.xml file, and are "stacked"
- on top of each other when building a ClassLoader - so if you have
- plugin jars with dependencies on other jars, the "lower level"
- dependency jars should be loaded first.
-
- If a "./lib" directory exists in your instanceDir, all files
- found in it are included as if you had used the following
- syntax...
-
- <lib dir="./lib" />
- -->
-
- <!-- A 'dir' option by itself adds any files found in the directory
- to the classpath, this is useful for including all jars in a
- directory.
-
- When a 'regex' is specified in addition to a 'dir', only the
- files in that directory which completely match the regex
- (anchored on both ends) will be included.
-
- If a 'dir' option (with or without a regex) is used and nothing
- is found that matches, a warning will be logged.
-
- The examples below can be used to load some solr-contribs along
- with their external dependencies.
- -->
+ <luceneMatchVersion>7.2.1</luceneMatchVersion>
+
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-dataimporthandler-.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
@@ -86,561 +32,73 @@
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
- <!-- an exact 'path' can be used instead of a 'dir' to specify a
- specific jar file. This will cause a serious error to be logged
- if it can't be loaded.
- -->
- <!--
- <lib path="../a-jar-that-does-not-exist.jar" />
- -->
-
- <!-- Data Directory
-
- Used to specify an alternate directory to hold all index data
- other than the default ./data under the Solr home. If
- replication is in use, this should match the replication
- configuration.
- -->
<dataDir>${solr.data.dir:}</dataDir>
-
- <!-- The DirectoryFactory to use for indexes.
-
- solr.StandardDirectoryFactory is filesystem
- based and tries to pick the best implementation for the current
- JVM and platform. solr.NRTCachingDirectoryFactory, the default,
- wraps solr.StandardDirectoryFactory and caches small files in memory
- for better NRT performance.
-
- One can force a particular implementation via solr.MMapDirectoryFactory,
- solr.NIOFSDirectoryFactory, or solr.SimpleFSDirectoryFactory.
-
- solr.RAMDirectoryFactory is memory based, not
- persistent, and doesn't work with replication.
- -->
<directoryFactory name="DirectoryFactory"
class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}">
-
-
- <!-- These will be used if you are using the solr.HdfsDirectoryFactory,
- otherwise they will be ignored. If you don't plan on using hdfs,
- you can safely remove this section. -->
- <!-- The root directory that collection data should be written to. -->
<str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <!-- The hadoop configuration files to use for the hdfs client. -->
<str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
- <!-- Enable/Disable the hdfs cache. -->
<str name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</str>
- <!-- Enable/Disable using one global cache for all SolrCores.
- The settings used will be from the first HdfsDirectoryFactory created. -->
<str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
-
</directoryFactory>
- <!-- The CodecFactory for defining the format of the inverted index.
- The default implementation is SchemaCodecFactory, which is the official Lucene
- index format, but hooks into the schema to provide per-field customization of
- the postings lists and per-document values in the fieldType element
- (postingsFormat/docValuesFormat). Note that most of the alternative implementations
- are experimental, so if you choose to customize the index format, it's a good
- idea to convert back to the official format e.g. via IndexWriter.addIndexes(IndexReader)
- before upgrading to a newer version to avoid unnecessary reindexing.
- -->
<codecFactory class="solr.SchemaCodecFactory"/>
-
- <!-- To enable dynamic schema REST APIs, use the following for <schemaFactory>: -->
- <schemaFactory class="ManagedIndexSchemaFactory">
- <bool name="mutable">true</bool>
- <str name="managedSchemaResourceName">managed-schema</str>
- </schemaFactory>
-<!--
- When ManagedIndexSchemaFactory is specified, Solr will load the schema from
- the resource named in 'managedSchemaResourceName', rather than from schema.xml.
- Note that the managed schema resource CANNOT be named schema.xml. If the managed
- schema does not exist, Solr will create it after reading schema.xml, then rename
- 'schema.xml' to 'schema.xml.bak'.
-
- Do NOT hand edit the managed schema - external modifications will be ignored and
- overwritten as a result of schema modification REST API calls.
-
- When ManagedIndexSchemaFactory is specified with mutable = true, schema
- modification REST API calls will be allowed; otherwise, error responses will be
- sent back for these requests.
-
- <schemaFactory class="ClassicIndexSchemaFactory"/>
- -->
-
- <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Index Config - These settings control low-level behavior of indexing
- Most example settings here show the default value, but are commented
- out, to more easily see where customizations have been made.
-
- Note: This replaces <indexDefaults> and <mainIndex> from older versions
- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
+ <schemaFactory class="ManagedIndexSchemaFactory">
+ <bool name="mutable">true</bool>
+ <str name="managedSchemaResourceName">managed-schema</str>
+ </schemaFactory>
+
<indexConfig>
- <!-- maxFieldLength was removed in 4.0. To get similar behavior, include a
- LimitTokenCountFilterFactory in your fieldType definition. E.g.
- <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10000"/>
- -->
- <!-- Maximum time to wait for a write lock (ms) for an IndexWriter. Default: 1000 -->
- <!-- <writeLockTimeout>1000</writeLockTimeout> -->
- <!-- LogSearch customization to avoid timeouts -->
<writeLockTimeout>10000</writeLockTimeout>
-
- <!-- The maximum number of simultaneous threads that may be
- indexing documents at once in IndexWriter; if more than this
- many threads arrive they will wait for others to finish.
- Default in Solr/Lucene is 8. -->
- <!-- <maxIndexingThreads>8</maxIndexingThreads> -->
- <!-- LogSearch customization of increase performance -->
<maxIndexingThreads>50</maxIndexingThreads>
-
- <!-- Expert: Enabling compound file will use less files for the index,
- using fewer file descriptors on the expense of performance decrease.
- Default in Lucene is "true". Default in Solr is "false" (since 3.6) -->
- <!-- <useCompoundFile>false</useCompoundFile> -->
-
- <!-- ramBufferSizeMB sets the amount of RAM that may be used by Lucene
- indexing for buffering added documents and deletions before they are
- flushed to the Directory.
- maxBufferedDocs sets a limit on the number of documents buffered
- before flushing.
- If both ramBufferSizeMB and maxBufferedDocs is set, then
- Lucene will flush based on whichever limit is hit first.
- The default is 100 MB. -->
- <!-- <ramBufferSizeMB>100</ramBufferSizeMB> -->
- <!-- <maxBufferedDocs>1000</maxBufferedDocs> -->
-
- <!-- Expert: Merge Policy
- The Merge Policy in Lucene controls how merging of segments is done.
- The default since Solr/Lucene 3.3 is TieredMergePolicy.
- The default since Lucene 2.3 was the LogByteSizeMergePolicy,
- Even older versions of Lucene used LogDocMergePolicy.
- -->
- <!--
- <mergePolicy class="org.apache.lucene.index.TieredMergePolicy">
- <int name="maxMergeAtOnce">10</int>
- <int name="segmentsPerTier">10</int>
- </mergePolicy>
- -->
-
- <!-- Merge Factor
- The merge factor controls how many segments will get merged at a time.
- For TieredMergePolicy, mergeFactor is a convenience parameter which
- will set both MaxMergeAtOnce and SegmentsPerTier at once.
- For LogByteSizeMergePolicy, mergeFactor decides how many new segments
- will be allowed before they are merged into one.
- Default is 10 for both merge policies.
- -->
- <!--
- <mergeFactor>10</mergeFactor>
- -->
- <!-- LogSearch customization. Set to 5 to trigger purging of deleted documents more often -->
- <mergeFactor>5</mergeFactor>
-
- <!-- Expert: Merge Scheduler
- The Merge Scheduler in Lucene controls how merges are
- performed. The ConcurrentMergeScheduler (Lucene 2.3 default)
- can perform merges in the background using separate threads.
- The SerialMergeScheduler (Lucene 2.2 default) does not.
- -->
- <!--
- <mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler"/>
- -->
-
- <!-- LockFactory
-
- This option specifies which Lucene LockFactory implementation
- to use.
-
- single = SingleInstanceLockFactory - suggested for a
- read-only index or when there is no possibility of
- another process trying to modify the index.
- native = NativeFSLockFactory - uses OS native file locking.
- Do not use when multiple solr webapps in the same
- JVM are attempting to share a single index.
- simple = SimpleFSLockFactory - uses a plain file for locking
-
- Defaults: 'native' is default for Solr3.6 and later, otherwise
- 'simple' is the default
-
- More details on the nuances of each LockFactory...
- http://wiki.apache.org/lucene-java/AvailableLockFactories
- -->
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+ <int name="maxMergeAtOnce">5</int>
+ <int name="segmentsPerTier">5</int>
+ </mergePolicyFactory>
<lockType>${solr.lock.type:native}</lockType>
-
- <!-- Unlock On Startup
-
- If true, unlock any held write or commit locks on startup.
- This defeats the locking mechanism that allows multiple
- processes to safely access a lucene index, and should be used
- with care. Default is "false".
-
- This is not needed if lock type is 'single'
- -->
- <!--
- <unlockOnStartup>false</unlockOnStartup>
- -->
-
- <!-- Commit Deletion Policy
- Custom deletion policies can be specified here. The class must
- implement org.apache.lucene.index.IndexDeletionPolicy.
-
- The default Solr IndexDeletionPolicy implementation supports
- deleting index commit points on number of commits, age of
- commit point and optimized status.
-
- The latest commit point should always be preserved regardless
- of the criteria.
- -->
- <!--
- <deletionPolicy class="solr.SolrDeletionPolicy">
- -->
- <!-- The number of commit points to be kept -->
- <!-- <str name="maxCommitsToKeep">1</str> -->
- <!-- The number of optimized commit points to be kept -->
- <!-- <str name="maxOptimizedCommitsToKeep">0</str> -->
- <!--
- Delete all commit points once they have reached the given age.
- Supports DateMathParser syntax e.g.
- -->
- <!--
- <str name="maxCommitAge">30MINUTES</str>
- <str name="maxCommitAge">1DAY</str>
- -->
- <!--
- </deletionPolicy>
- -->
-
- <!-- Lucene Infostream
-
- To aid in advanced debugging, Lucene provides an "InfoStream"
- of detailed information when indexing.
-
- Setting the value to true will instruct the underlying Lucene
- IndexWriter to write its info stream to solr's log. By default,
- this is enabled here, and controlled through log4j.properties.
- -->
- <infoStream>true</infoStream>
+ <infoStream>true</infoStream>
</indexConfig>
-
-
- <!-- JMX
-
- This example enables JMX if and only if an existing MBeanServer
- is found, use this if you want to configure JMX through JVM
- parameters. Remove this to disable exposing Solr configuration
- and statistics to JMX.
-
- For more details see http://wiki.apache.org/solr/SolrJmx
- -->
<jmx />
- <!-- If you want to connect to a particular server, specify the
- agentId
- -->
- <!-- <jmx agentId="myAgent" /> -->
- <!-- If you want to start a new MBeanServer, specify the serviceUrl -->
- <!-- <jmx serviceUrl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr"/>
- -->
-
- <!-- The default high-performance update handler -->
- <updateHandler class="solr.DirectUpdateHandler2">
- <!-- Enables a transaction log, used for real-time get, durability, and
- and solr cloud replica recovery. The log can grow as big as
- uncommitted changes to the index, so use of a hard autoCommit
- is recommended (see below).
- "dir" - the target directory for transaction logs, defaults to the
- solr data directory. -->
+ <updateHandler class="solr.DirectUpdateHandler2">
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
</updateLog>
-
- <!-- AutoCommit
-
- Perform a hard commit automatically under certain conditions.
- Instead of enabling autoCommit, consider using "commitWithin"
- when adding documents.
-
- http://wiki.apache.org/solr/UpdateXmlMessages
-
- maxDocs - Maximum number of documents to add since the last
- commit before automatically triggering a new commit.
-
- maxTime - Maximum amount of time in ms that is allowed to pass
- since a document was added before automatically
- triggering a new commit.
- openSearcher - if false, the commit causes recent index changes
- to be flushed to stable storage, but does not cause a new
- searcher to be opened to make those changes visible.
-
- If the updateLog is enabled, then it's highly recommended to
- have some sort of hard autoCommit to limit the log size.
- -->
- <autoCommit>
- <maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
- <openSearcher>false</openSearcher>
- </autoCommit>
-
- <!-- softAutoCommit is like autoCommit except it causes a
- 'soft' commit which only ensures that changes are visible
- but does not ensure that data is synced to disk. This is
- faster and more near-realtime friendly than a hard commit.
- -->
-
- <autoSoftCommit>
- <maxTime>${solr.autoSoftCommit.maxTime:5000}</maxTime>
- </autoSoftCommit>
-
- <!-- Update Related Event Listeners
-
- Various IndexWriter related events can trigger Listeners to
- take actions.
-
- postCommit - fired after every commit or optimize command
- postOptimize - fired after every optimize command
- -->
- <!-- The RunExecutableListener executes an external command from a
- hook such as postCommit or postOptimize.
-
- exe - the name of the executable to run
- dir - dir to use as the current working directory. (default=".")
- wait - the calling thread waits until the executable returns.
- (default="true")
- args - the arguments to pass to the program. (default is none)
- env - environment variables to set. (default is none)
- -->
- <!-- This example shows how RunExecutableListener could be used
- with the script based replication...
- http://wiki.apache.org/solr/CollectionDistribution
- -->
- <!--
- <listener event="postCommit" class="solr.RunExecutableListener">
- <str name="exe">solr/bin/snapshooter</str>
- <str name="dir">.</str>
- <bool name="wait">true</bool>
- <arr name="args"> <str>arg1</str> <str>arg2</str> </arr>
- <arr name="env"> <str>MYVAR=val1</str> </arr>
- </listener>
- -->
-
+ <autoCommit>
+ <maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
+ <openSearcher>false</openSearcher>
+ </autoCommit>
+ <autoSoftCommit>
+ <maxTime>${solr.autoSoftCommit.maxTime:5000}</maxTime>
+ </autoSoftCommit>
</updateHandler>
-
- <!-- IndexReaderFactory
-
- Use the following format to specify a custom IndexReaderFactory,
- which allows for alternate IndexReader implementations.
-
- ** Experimental Feature **
-
- Please note - Using a custom IndexReaderFactory may prevent
- certain other features from working. The API to
- IndexReaderFactory may change without warning or may even be
- removed from future releases if the problems cannot be
- resolved.
-
-
- ** Features that may not work with custom IndexReaderFactory **
-
- The ReplicationHandler assumes a disk-resident index. Using a
- custom IndexReader implementation may cause incompatibility
- with ReplicationHandler and may cause replication to not work
- correctly. See SOLR-1366 for details.
-
- -->
- <!--
- <indexReaderFactory name="IndexReaderFactory" class="package.class">
- <str name="someArg">Some Value</str>
- </indexReaderFactory >
- -->
-
- <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Query section - these settings control query time things like caches
- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
<query>
- <!-- Max Boolean Clauses
-
- Maximum number of clauses in each BooleanQuery, an exception
- is thrown if exceeded.
-
- ** WARNING **
-
- This option actually modifies a global Lucene property that
- will affect all SolrCores. If multiple solrconfig.xml files
- disagree on this property, the value at any given moment will
- be based on the last SolrCore to be initialized.
-
- -->
<maxBooleanClauses>1024</maxBooleanClauses>
-
-
- <!-- Solr Internal Query Caches
-
- There are two implementations of cache available for Solr,
- LRUCache, based on a synchronized LinkedHashMap, and
- FastLRUCache, based on a ConcurrentHashMap.
-
- FastLRUCache has faster gets and slower puts in single
- threaded operation and thus is generally faster than LRUCache
- when the hit ratio of the cache is high (> 75%), and may be
- faster under other scenarios on multi-cpu systems.
- -->
-
- <!-- Filter Cache
-
- Cache used by SolrIndexSearcher for filters (DocSets),
- unordered sets of *all* documents that match a query. When a
- new searcher is opened, its caches may be prepopulated or
- "autowarmed" using data from caches in the old searcher.
- autowarmCount is the number of items to prepopulate. For
- LRUCache, the autowarmed items will be the most recently
- accessed items.
-
- Parameters:
- class - the SolrCache implementation LRUCache or
- (LRUCache or FastLRUCache)
- size - the maximum number of entries in the cache
- initialSize - the initial capacity (number of entries) of
- the cache. (see java.util.HashMap)
- autowarmCount - the number of entries to prepopulate from
- and old cache.
- -->
<filterCache class="solr.FastLRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
-
- <!-- Query Result Cache
-
- Caches results of searches - ordered lists of document ids
- (DocList) based on a query, a sort, and the range of documents requested.
- -->
<queryResultCache class="solr.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
-
- <!-- Document Cache
-
- Caches Lucene Document objects (the stored fields for each
- document). Since Lucene internal document ids are transient,
- this cache will not be autowarmed.
- -->
<documentCache class="solr.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
-
- <!-- custom cache currently used by block join -->
<cache name="perSegFilter"
class="solr.search.LRUCache"
size="10"
initialSize="0"
autowarmCount="10"
regenerator="solr.NoOpRegenerator" />
-
- <!-- Field Value Cache
-
- Cache used to hold field values that are quickly accessible
- by document id. The fieldValueCache is created by default
- even if not configured here.
- -->
- <!--
- <fieldValueCache class="solr.FastLRUCache"
- size="512"
- autowarmCount="128"
- showItems="32" />
- -->
-
- <!-- Custom Cache
-
- Example of a generic cache. These caches may be accessed by
- name through SolrIndexSearcher.getCache(),cacheLookup(), and
- cacheInsert(). The purpose is to enable easy caching of
- user/application level data. The regenerator argument should
- be specified as an implementation of solr.CacheRegenerator
- if autowarming is desired.
- -->
- <!--
- <cache name="myUserCache"
- class="solr.LRUCache"
- size="4096"
- initialSize="1024"
- autowarmCount="1024"
- regenerator="com.mycompany.MyRegenerator"
- />
- -->
-
-
- <!-- Lazy Field Loading
-
- If true, stored fields that are not requested will be loaded
- lazily. This can result in a significant speed improvement
- if the usual case is to not load all stored fields,
- especially if the skipped fields are large compressed text
- fields.
- -->
<enableLazyFieldLoading>true</enableLazyFieldLoading>
- <!-- Use Filter For Sorted Query
-
- A possible optimization that attempts to use a filter to
- satisfy a search. If the requested sort does not include
- score, then the filterCache will be checked for a filter
- matching the query. If found, the filter will be used as the
- source of document ids, and then the sort will be applied to
- that.
-
- For most situations, this will not be useful unless you
- frequently get the same search repeatedly with different sort
- options, and none of them ever use "score"
- -->
- <!--
- <useFilterForSortedQuery>true</useFilterForSortedQuery>
- -->
-
- <!-- Result Window Size
-
- An optimization for use with the queryResultCache. When a search
- is requested, a superset of the requested number of document ids
- are collected. For example, if a search for a particular query
- requests matching documents 10 through 19, and queryWindowSize is 50,
- then documents 0 through 49 will be collected and cached. Any further
- requests in that range can be satisfied via the cache.
- -->
- <queryResultWindowSize>20</queryResultWindowSize>
-
- <!-- Maximum number of documents to cache for any entry in the
- queryResultCache.
- -->
- <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
-
- <!-- Query Related Event Listeners
-
- Various IndexSearcher related events can trigger Listeners to
- take actions.
-
- newSearcher - fired whenever a new searcher is being prepared
- and there is a current searcher handling requests (aka
- registered). It can be used to prime certain caches to
- prevent long request times for certain requests.
-
- firstSearcher - fired whenever a new searcher is being
- prepared but there is no current registered searcher to handle
- requests or to gain autowarming data from.
-
-
- -->
- <!-- QuerySenderListener takes an array of NamedList and executes a
- local query request for each NamedList in sequence.
- -->
+ <queryResultWindowSize>20</queryResultWindowSize>
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
<listener event="newSearcher" class="solr.QuerySenderListener">
<arr name="queries">
- <!--
- <lst><str name="q">solr</str><str name="sort">price asc</str></lst>
- <lst><str name="q">rocks</str><str name="sort">weight asc</str></lst>
- -->
</arr>
</listener>
<listener event="firstSearcher" class="solr.QuerySenderListener">
@@ -651,238 +109,36 @@
</arr>
</listener>
- <!-- Use Cold Searcher
-
- If a search request comes in and there is no current
- registered searcher, then immediately register the still
- warming searcher and use it. If "false" then all requests
- will block until the first searcher is done warming.
- -->
<useColdSearcher>true</useColdSearcher>
- <!-- Max Warming Searchers
-
- Maximum number of searchers that may be warming in the
- background concurrently. An error is returned if this limit
- is exceeded.
-
- Recommend values of 1-2 for read-only slaves, higher for
- masters w/o cache warming.
- -->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>
-
- <!-- Request Dispatcher
-
- This section contains instructions for how the SolrDispatchFilter
- should behave when processing requests for this SolrCore.
-
- handleSelect is a legacy option that affects the behavior of requests
- such as /select?qt=XXX
-
- handleSelect="true" will cause the SolrDispatchFilter to process
- the request and dispatch the query to a handler specified by the
- "qt" param, assuming "/select" isn't already registered.
-
- handleSelect="false" will cause the SolrDispatchFilter to
- ignore "/select" requests, resulting in a 404 unless a handler
- is explicitly registered with the name "/select"
-
- handleSelect="true" is not recommended for new users, but is the default
- for backwards compatibility
- -->
<requestDispatcher handleSelect="false" >
- <!-- Request Parsing
-
- These settings indicate how Solr Requests may be parsed, and
- what restrictions may be placed on the ContentStreams from
- those requests
-
- enableRemoteStreaming - enables use of the stream.file
- and stream.url parameters for specifying remote streams.
-
- multipartUploadLimitInKB - specifies the max size (in KiB) of
- Multipart File Uploads that Solr will allow in a Request.
-
- formdataUploadLimitInKB - specifies the max size (in KiB) of
- form data (application/x-www-form-urlencoded) sent via
- POST. You can use POST to pass request parameters not
- fitting into the URL.
-
- addHttpRequestToContext - if set to true, it will instruct
- the requestParsers to include the original HttpServletRequest
- object in the context map of the SolrQueryRequest under the
- key "httpRequest". It will not be used by any of the existing
- Solr components, but may be useful when developing custom
- plugins.
-
- *** WARNING ***
- The settings below authorize Solr to fetch remote files, You
- should make sure your system has some authentication before
- using enableRemoteStreaming="true"
-
- -->
+
<requestParsers enableRemoteStreaming="true"
multipartUploadLimitInKB="2048000"
formdataUploadLimitInKB="2048"
addHttpRequestToContext="false"/>
- <!-- HTTP Caching
-
- Set HTTP caching related parameters (for proxy caches and clients).
-
- The options below instruct Solr not to output any HTTP Caching
- related headers
- -->
<httpCaching never304="true" />
- <!-- If you include a <cacheControl> directive, it will be used to
- generate a Cache-Control header (as well as an Expires header
- if the value contains "max-age=")
-
- By default, no Cache-Control header is generated.
-
- You can use the <cacheControl> option even if you have set
- never304="true"
- -->
- <!--
- <httpCaching never304="true" >
- <cacheControl>max-age=30, public</cacheControl>
- </httpCaching>
- -->
- <!-- To enable Solr to respond with automatically generated HTTP
- Caching headers, and to response to Cache Validation requests
- correctly, set the value of never304="false"
-
- This will cause Solr to generate Last-Modified and ETag
- headers based on the properties of the Index.
-
- The following options can also be specified to affect the
- values of these headers...
-
- lastModFrom - the default value is "openTime" which means the
- Last-Modified value (and validation against If-Modified-Since
- requests) will all be relative to when the current Searcher
- was opened. You can change it to lastModFrom="dirLastMod" if
- you want the value to exactly correspond to when the physical
- index was last modified.
-
- etagSeed="..." is an option you can change to force the ETag
- header (and validation against If-None-Match requests) to be
- different even if the index has not changed (ie: when making
- significant changes to your config file)
-
- (lastModifiedFrom and etagSeed are both ignored if you use
- the never304="true" option)
- -->
- <!--
- <httpCaching lastModifiedFrom="openTime"
- etagSeed="Solr">
- <cacheControl>max-age=30, public</cacheControl>
- </httpCaching>
- -->
</requestDispatcher>
- <!-- Request Handlers
-
- http://wiki.apache.org/solr/SolrRequestHandler
-
- Incoming queries will be dispatched to a specific handler by name
- based on the path specified in the request.
-
- Legacy behavior: If the request path uses "/select" but no Request
- Handler has that name, and if handleSelect="true" has been specified in
- the requestDispatcher, then the Request Handler is dispatched based on
- the qt parameter. Handlers without a leading '/' are accessed this way
- like so: http://host/app/[core/]select?qt=name If no qt is
- given, then the requestHandler that declares default="true" will be
- used or the one named "standard".
-
- If a Request Handler is declared with startup="lazy", then it will
- not be initialized until the first request that uses it.
-
- -->
-
<requestHandler name="/dataimport" class="solr.DataImportHandler">
<lst name="defaults">
<str name="config">solr-data-config.xml</str>
</lst>
</requestHandler>
- <!-- SearchHandler
-
- http://wiki.apache.org/solr/SearchHandler
-
- For processing Search Queries, the primary Request Handler
- provided with Solr is "SearchHandler" It delegates to a sequent
- of SearchComponents (see below) and supports distributed
- queries across multiple shards
- -->
<requestHandler name="/select" class="solr.SearchHandler">
- <!-- default values for query parameters can be specified, these
- will be overridden by parameters in the request
- -->
<lst name="defaults">
<str name="echoParams">explicit</str>
<int name="rows">10</int>
<str name="df">text</str>
</lst>
- <!-- In addition to defaults, "appends" params can be specified
- to identify values which should be appended to the list of
- multi-val params from the query (or the existing "defaults").
- -->
- <!-- In this example, the param "fq=instock:true" would be appended to
- any query time fq params the user may specify, as a mechanism for
- partitioning the index, independent of any user selected filtering
- that may also be desired (perhaps as a result of faceted searching).
-
- NOTE: there is *absolutely* nothing a client can do to prevent these
- "appends" values from being used, so don't use this mechanism
- unless you are sure you always want it.
- -->
- <!--
- <lst name="appends">
- <str name="fq">inStock:true</str>
- </lst>
- -->
- <!-- "invariants" are a way of letting the Solr maintainer lock down
- the options available to Solr clients. Any params values
- specified here are used regardless of what values may be specified
- in either the query, the "defaults", or the "appends" params.
-
- In this example, the facet.field and facet.query params would
- be fixed, limiting the facets clients can use. Faceting is
- not turned on by default - but if the client does specify
- facet=true in the request, these are the only facets they
- will be able to see counts for; regardless of what other
- facet.field or facet.query params they may specify.
-
- NOTE: there is *absolutely* nothing a client can do to prevent these
- "invariants" values from being used, so don't use this mechanism
- unless you are sure you always want it.
- -->
- <!--
- <lst name="invariants">
- <str name="facet.field">cat</str>
- <str name="facet.field">manu_exact</str>
- <str name="facet.query">price:[* TO 500]</str>
- <str name="facet.query">price:[500 TO *]</str>
- </lst>
- -->
- <!-- If the default list of SearchComponents is not desired, that
- list can either be overridden completely, or components can be
- prepended or appended to the default list. (see below)
- -->
- <!--
- <arr name="components">
- <str>nameOfCustomComponent1</str>
- <str>nameOfCustomComponent2</str>
- </arr>
- -->
- </requestHandler>
-
- <!-- A request handler that returns indented JSON by default -->
+ </requestHandler>
+
<requestHandler name="/query" class="solr.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
@@ -892,18 +148,6 @@
</lst>
</requestHandler>
-
- <!-- realtime get handler, guaranteed to return the latest stored fields of
- any document, without the need to commit or open a new searcher. The
- current implementation relies on the updateLog feature being enabled.
-
- ** WARNING **
- Do NOT disable the realtime get handler at /get if you are using
- SolrCloud otherwise any leader election will cause a full sync in ALL
- replicas for the shard in question. Similarly, a replica recovery will
- also always fetch the complete index from the leader because a partial
- sync will not be possible in the absence of this handler.
- -->
<requestHandler name="/get" class="solr.RealTimeGetHandler">
<lst name="defaults">
<str name="omitHeader">true</str>
@@ -912,16 +156,6 @@
</lst>
</requestHandler>
-
- <!-- A Robust Example
-
- This example SearchHandler declaration shows off usage of the
- SearchHandler with many defaults declared
-
- Note that multiple instances of the same Request Handler
- (SearchHandler) can be registered multiple times with different
- names (and different init parameters)
- -->
<requestHandler name="/browse" class="solr.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
@@ -951,41 +185,9 @@
</lst>
</initParams>
- <!-- Update Request Handler.
-
- http://wiki.apache.org/solr/UpdateXmlMessages
-
- The canonical Request Handler for Modifying the Index through
- commands specified using XML, JSON, CSV, or JAVABIN
-
- Note: Since solr1.1 requestHandlers requires a valid content
- type header if posted in the body. For example, curl now
- requires: -H 'Content-type:text/xml; charset=utf-8'
-
- To override the request content type and force a specific
- Content-type, use the request parameter:
- ?update.contentType=text/csv
-
- This handler will pick a response format to match the input
- if the 'wt' parameter is not explicit
- -->
<requestHandler name="/update" class="solr.UpdateRequestHandler">
- <!-- See below for information on defining
- updateRequestProcessorChains that can be used by name
- on each Update Request
- -->
- <!--
- <lst name="defaults">
- <str name="update.chain">dedupe</str>
- </lst>
- -->
</requestHandler>
- <!-- Solr Cell Update Request Handler
-
- http://wiki.apache.org/solr/ExtractingRequestHandler
-
- -->
<requestHandler name="/update/extract"
startup="lazy"
class="solr.extraction.ExtractingRequestHandler" >
@@ -1000,109 +202,14 @@
</lst>
</requestHandler>
-
- <!-- Field Analysis Request Handler
-
- RequestHandler that provides much the same functionality as
- analysis.jsp. Provides the ability to specify multiple field
- types and field names in the same request and outputs
- index-time and query-time analysis for each of them.
-
- Request parameters are:
- analysis.fieldname - field name whose analyzers are to be used
-
- analysis.fieldtype - field type whose analyzers are to be used
- analysis.fieldvalue - text for index-time analysis
- q (or analysis.q) - text for query time analysis
- analysis.showmatch (true|false) - When set to true and when
- query analysis is performed, the produced tokens of the
- field value analysis will be marked as "matched" for every
- token that is produces by the query analysis
- -->
<requestHandler name="/analysis/field"
startup="lazy"
class="solr.FieldAnalysisRequestHandler" />
-
- <!-- Document Analysis Handler
-
- http://wiki.apache.org/solr/AnalysisRequestHandler
-
- An analysis handler that provides a breakdown of the analysis
- process of provided documents. This handler expects a (single)
- content stream with the following format:
-
- <docs>
- <doc>
- <field name="id">1</field>
- <field name="name">The Name</field>
- <field name="text">The Text Value</field>
- </doc>
- <doc>...</doc>
- <doc>...</doc>
- ...
- </docs>
-
- Note: Each document must contain a field which serves as the
- unique key. This key is used in the returned response to associate
- an analysis breakdown to the analyzed document.
-
- Like the FieldAnalysisRequestHandler, this handler also supports
- query analysis by sending either an "analysis.query" or "q"
- request parameter that holds the query text to be analyzed. It
- also supports the "analysis.showmatch" parameter which when set to
- true, all field tokens that match the query tokens will be marked
- as a "match".
- -->
<requestHandler name="/analysis/document"
class="solr.DocumentAnalysisRequestHandler"
startup="lazy" />
- <!-- Admin Handlers
-
- Admin Handlers - This will register all the standard admin
- RequestHandlers.
- -->
-
- <!-- This single handler is equivalent to the following... -->
- <!--
- <requestHandler name="/admin/luke" class="solr.admin.LukeRequestHandler" />
- <requestHandler name="/admin/system" class="solr.admin.SystemInfoHandler" />
- <requestHandler name="/admin/plugins" class="solr.admin.PluginInfoHandler" />
- <requestHandler name="/admin/threads" class="solr.admin.ThreadDumpHandler" />
- <requestHandler name="/admin/properties" class="solr.admin.PropertiesRequestHandler" />
- <requestHandler name="/admin/file" class="solr.admin.ShowFileRequestHandler" >
- -->
- <!-- If you wish to hide files under ${solr.home}/conf, explicitly
- register the ShowFileRequestHandler using the definition below.
- NOTE: The glob pattern ('*') is the only pattern supported at present, *.xml will
- not exclude all files ending in '.xml'. Use it to exclude _all_ updates
- -->
- <!--
- <requestHandler name="/admin/file"
- class="solr.admin.ShowFileRequestHandler" >
- <lst name="invariants">
- <str name="hidden">synonyms.txt</str>
- <str name="hidden">anotherfile.txt</str>
- <str name="hidden">*</str>
- </lst>
- </requestHandler>
- -->
-
- <!--
- Enabling this request handler (which is NOT a default part of the admin handler) will allow the Solr UI to edit
- all the config files. This is intended for secure/development use ONLY! Leaving available and publically
- accessible is a security vulnerability and should be done with extreme caution!
- -->
- <!--
- <requestHandler name="/admin/fileedit" class="solr.admin.EditFileRequestHandler" >
- <lst name="invariants">
- <str name="hidden">synonyms.txt</str>
- <str name="hidden">anotherfile.txt</str>
- </lst>
- </requestHandler>
- -->
- <!-- ping/healthcheck -->
<requestHandler name="/admin/ping" class="solr.PingRequestHandler">
<lst name="invariants">
<str name="q">solrpingquery</str>
@@ -1110,140 +217,35 @@
<lst name="defaults">
<str name="echoParams">all</str>
</lst>
- <!-- An optional feature of the PingRequestHandler is to configure the
- handler with a "healthcheckFile" which can be used to enable/disable
- the PingRequestHandler.
- relative paths are resolved against the data dir
- -->
- <!-- <str name="healthcheckFile">server-enabled.txt</str> -->
</requestHandler>
- <!-- Echo the request contents back to the client -->
<requestHandler name="/debug/dump" class="solr.DumpRequestHandler" >
<lst name="defaults">
<str name="echoParams">explicit</str>
<str name="echoHandler">true</str>
</lst>
</requestHandler>
-
- <!-- Solr Replication
-
- The SolrReplicationHandler supports replicating indexes from a
- "master" used for indexing and "slaves" used for queries.
-
- http://wiki.apache.org/solr/SolrReplication
-
- It is also necessary for SolrCloud to function (in Cloud mode, the
- replication handler is used to bulk transfer segments when nodes
- are added or need to recover).
-
- https://wiki.apache.org/solr/SolrCloud/
- -->
- <requestHandler name="/replication" class="solr.ReplicationHandler" >
- <!--
- To enable simple master/slave replication, uncomment one of the
- sections below, depending on whether this solr instance should be
- the "master" or a "slave". If this instance is a "slave" you will
- also need to fill in the masterUrl to point to a real machine.
- -->
- <!--
- <lst name="master">
- <str name="replicateAfter">commit</str>
- <str name="replicateAfter">startup</str>
- <str name="confFiles">schema.xml,stopwords.txt</str>
- </lst>
- -->
- <!--
- <lst name="slave">
- <str name="masterUrl">http://your-master-hostname:8983/solr</str>
- <str name="pollInterval">00:00:60</str>
- </lst>
- -->
- </requestHandler>
-
- <!-- Search Components
-
- Search components are registered to SolrCore and used by
- instances of SearchHandler (which can access them by name)
-
- By default, the following components are available:
-
- <searchComponent name="query" class="solr.QueryComponent" />
- <searchComponent name="facet" class="solr.FacetComponent" />
- <searchComponent name="mlt" class="solr.MoreLikeThisComponent" />
- <searchComponent name="highlight" class="solr.HighlightComponent" />
- <searchComponent name="stats" class="solr.StatsComponent" />
- <searchComponent name="debug" class="solr.DebugComponent" />
-
- Default configuration in a requestHandler would look like:
-
- <arr name="components">
- <str>query</str>
- <str>facet</str>
- <str>mlt</str>
- <str>highlight</str>
- <str>stats</str>
- <str>debug</str>
- </arr>
-
- If you register a searchComponent to one of the standard names,
- that will be used instead of the default.
-
- To insert components before or after the 'standard' components, use:
-
- <arr name="first-components">
- <str>myFirstComponentName</str>
- </arr>
-
- <arr name="last-components">
- <str>myLastComponentName</str>
- </arr>
-
- NOTE: The component registered with the name "debug" will
- always be executed after the "last-components"
-
- -->
-
- <!-- Spell Check
- The spell check component can return a list of alternative spelling
- suggestions.
+ <requestHandler name="/replication" class="solr.ReplicationHandler" >
+ </requestHandler>
- http://wiki.apache.org/solr/SpellCheckComponent
- -->
<searchComponent name="spellcheck" class="solr.SpellCheckComponent">
<str name="queryAnalyzerFieldType">key_lower_case</str>
- <!-- Multiple "Spell Checkers" can be declared and used by this
- component
- -->
-
- <!-- a spellchecker built from a field of the main index -->
<lst name="spellchecker">
<str name="name">default</str>
<str name="field">text</str>
<str name="classname">solr.DirectSolrSpellChecker</str>
- <!-- the spellcheck distance measure used, the default is the internal levenshtein -->
<str name="distanceMeasure">internal</str>
- <!-- minimum accuracy needed to be considered a valid spellcheck suggestion -->
<float name="accuracy">0.5</float>
- <!-- the maximum #edits we consider when enumerating terms: can be 1 or 2 -->
<int name="maxEdits">2</int>
- <!-- the minimum shared prefix when enumerating terms -->
<int name="minPrefix">1</int>
- <!-- maximum number of inspections per result. -->
<int name="maxInspections">5</int>
- <!-- minimum length of a query term to be considered for correction -->
<int name="minQueryLength">4</int>
- <!-- maximum threshold of documents a query term can appear to be considered for correction -->
<float name="maxQueryFrequency">0.01</float>
- <!-- uncomment this to require suggestions to occur in 1% of the documents
- <float name="thresholdTokenFrequency">.01</float>
- -->
</lst>
-
- <!-- a spellchecker that can break or combine words. See "/spell" handler below for usage -->
+
<lst name="spellchecker">
<str name="name">wordbreak</str>
<str name="classname">solr.WordBreakSolrSpellChecker</str>
@@ -1252,66 +254,11 @@
<str name="breakWords">true</str>
<int name="maxChanges">10</int>
</lst>
-
- <!-- a spellchecker that uses a different distance measure -->
- <!--
- <lst name="spellchecker">
- <str name="name">jarowinkler</str>
- <str name="field">spell</str>
- <str name="classname">solr.DirectSolrSpellChecker</str>
- <str name="distanceMeasure">
- org.apache.lucene.search.spell.JaroWinklerDistance
- </str>
- </lst>
- -->
-
- <!-- a spellchecker that use an alternate comparator
-
- comparatorClass be one of:
- 1. score (default)
- 2. freq (Frequency first, then score)
- 3. A fully qualified class name
- -->
- <!--
- <lst name="spellchecker">
- <str name="name">freq</str>
- <str name="field">lowerfilt</str>
- <str name="classname">solr.DirectSolrSpellChecker</str>
- <str name="comparatorClass">freq</str>
- -->
-
- <!-- A spellchecker that reads the list of words from a file -->
- <!--
- <lst name="spellchecker">
- <str name="classname">solr.FileBasedSpellChecker</str>
- <str name="name">file</str>
- <str name="sourceLocation">spellings.txt</str>
- <str name="characterEncoding">UTF-8</str>
- <str name="spellcheckIndexDir">spellcheckerFile</str>
- </lst>
- -->
</searchComponent>
-
- <!-- A request handler for demonstrating the spellcheck component.
-
- NOTE: This is purely as an example. The whole purpose of the
- SpellCheckComponent is to hook it into the request handler that
- handles your normal user queries so that a separate request is
- not needed to get suggestions.
-
- IN OTHER WORDS, THERE IS REALLY GOOD CHANCE THE SETUP BELOW IS
- NOT WHAT YOU WANT FOR YOUR PRODUCTION SYSTEM!
-
- See http://wiki.apache.org/solr/SpellCheckComponent for details
- on the request parameters.
- -->
+
<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<str name="df">text</str>
- <!-- Solr will use suggestions from both the 'default' spellchecker
- and from the 'wordbreak' spellchecker and combine them.
- collations (re-written queries) can include a combination of
- corrections from both spellcheckers -->
<str name="spellcheck.dictionary">default</str>
<str name="spellcheck.dictionary">wordbreak</str>
<str name="spellcheck">on</str>
@@ -1332,8 +279,8 @@
<searchComponent name="suggest" class="solr.SuggestComponent">
<lst name="suggester">
<str name="name">mySuggester</str>
- <str name="lookupImpl">FuzzyLookupFactory</str> <!-- org.apache.solr.spelling.suggest.fst -->
- <str name="dictionaryImpl">DocumentDictionaryFactory</str> <!-- org.apache.solr.spelling.suggest.HighFrequencyDictionaryFactory -->
+ <str name="lookupImpl">FuzzyLookupFactory</str>
+ <str name="dictionaryImpl">DocumentDictionaryFactory</str>
<str name="field">cat</str>
<str name="weightField">price</str>
<str name="suggestAnalyzerFieldType">string</str>
@@ -1349,19 +296,9 @@
<str>suggest</str>
</arr>
</requestHandler>
- <!-- Term Vector Component
- http://wiki.apache.org/solr/TermVectorComponent
- -->
<searchComponent name="tvComponent" class="solr.TermVectorComponent"/>
- <!-- A request handler for demonstrating the term vector component
-
- This is purely as an example.
-
- In reality you will likely want to add the component to your
- already specified request handlers.
- -->
<requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<str name="df">text</str>
@@ -1372,71 +309,24 @@
</arr>
</requestHandler>
- <!-- Clustering Component
-
- You'll need to set the solr.clustering.enabled system property
- when running solr to run with clustering enabled:
-
- java -Dsolr.clustering.enabled=true -jar start.jar
-
- http://wiki.apache.org/solr/ClusteringComponent
- http://carrot2.github.io/solr-integration-strategies/
- -->
<searchComponent name="clustering"
enable="${solr.clustering.enabled:false}"
class="solr.clustering.ClusteringComponent" >
<lst name="engine">
<str name="name">lingo</str>
-
- <!-- Class name of a clustering algorithm compatible with the Carrot2 framework.
-
- Currently available open source algorithms are:
- * org.carrot2.clustering.lingo.LingoClusteringAlgorithm
- * org.carrot2.clustering.stc.STCClusteringAlgorithm
- * org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
-
- See http://project.carrot2.org/algorithms.html for more information.
-
- A commercial algorithm Lingo3G (needs to be installed separately) is defined as:
- * com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
- -->
<str name="carrot.algorithm">org.carrot2.clustering.lingo.LingoClusteringAlgorithm</str>
-
- <!-- Override location of the clustering algorithm's resources
- (attribute definitions and lexical resources).
-
- A directory from which to load algorithm-specific stop words,
- stop labels and attribute definition XMLs.
-
- For an overview of Carrot2 lexical resources, see:
- http://download.carrot2.org/head/manual/#chapter.lexical-resources
-
- For an overview of Lingo3G lexical resources, see:
- http://download.carrotsearch.com/lingo3g/manual/#chapter.lexical-resources
- -->
<str name="carrot.resourcesDir">clustering/carrot2</str>
</lst>
-
- <!-- An example definition for the STC clustering algorithm. -->
<lst name="engine">
<str name="name">stc</str>
<str name="carrot.algorithm">org.carrot2.clustering.stc.STCClusteringAlgorithm</str>
</lst>
-
- <!-- An example definition for the bisecting kmeans clustering algorithm. -->
<lst name="engine">
<str name="name">kmeans</str>
<str name="carrot.algorithm">org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm</str>
</lst>
</searchComponent>
- <!-- A request handler for demonstrating the clustering component
-
- This is purely as an example.
-
- In reality you will likely want to add the component to your
- already specified request handlers.
- -->
<requestHandler name="/clustering"
startup="lazy"
enable="${solr.clustering.enabled:false}"
@@ -1444,20 +334,11 @@
<lst name="defaults">
<bool name="clustering">true</bool>
<bool name="clustering.results">true</bool>
- <!-- Field name with the logical "title" of a each document (optional) -->
<str name="carrot.title">name</str>
- <!-- Field name with the logical "URL" of a each document (optional) -->
<str name="carrot.url">id</str>
- <!-- Field name with the logical "content" of a each document (optional) -->
<str name="carrot.snippet">features</str>
- <!-- Apply highlighter to the title/ content and use this for clustering. -->
<bool name="carrot.produceSummary">true</bool>
- <!-- the maximum number of labels per cluster -->
- <!--<int name="carrot.numDescriptions">5</int>-->
- <!-- produce sub clusters -->
<bool name="carrot.outputSubClusters">false</bool>
-
- <!-- Configure the remaining request handler parameters. -->
<str name="defType">edismax</str>
<str name="qf">
text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
@@ -1470,17 +351,9 @@
<str>clustering</str>
</arr>
</requestHandler>
-
- <!-- Terms Component
-
- http://wiki.apache.org/solr/TermsComponent
- A component to return terms and document frequency of those
- terms
- -->
<searchComponent name="terms" class="solr.TermsComponent"/>
- <!-- A request handler for demonstrating the terms component -->
<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<bool name="terms">true</bool>
@@ -1491,22 +364,11 @@
</arr>
</requestHandler>
-
- <!-- Query Elevation Component
-
- http://wiki.apache.org/solr/QueryElevationComponent
-
- a search component that enables you to configure the top
- results for a given query regardless of the normal lucene
- scoring.
- -->
<searchComponent name="elevator" class="solr.QueryElevationComponent" >
- <!-- pick a fieldType to analyze queries -->
<str name="queryFieldType">string</str>
<str name="config-file">elevate.xml</str>
</searchComponent>
- <!-- A request handler for demonstrating the elevator component -->
<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
<lst name="defaults">
<str name="echoParams">explicit</str>
@@ -1517,14 +379,8 @@
</arr>
</requestHandler>
- <!-- Highlighting Component
-
- http://wiki.apache.org/solr/HighlightingParameters
- -->
<searchComponent class="solr.HighlightComponent" name="highlight">
<highlighting>
- <!-- Configure the standard fragmenter -->
- <!-- This could most likely be commented out in the "default" case -->
<fragmenter name="gap"
default="true"
class="solr.highlight.GapFragmenter">
@@ -1533,22 +389,15 @@
</lst>
</fragmenter>
- <!-- A regular-expression-based fragmenter
- (for sentence extraction)
- -->
<fragmenter name="regex"
class="solr.highlight.RegexFragmenter">
<lst name="defaults">
- <!-- slightly smaller fragsizes work better because of slop -->
<int name="hl.fragsize">70</int>
- <!-- allow 50% slop on fragment sizes -->
<float name="hl.regex.slop">0.5</float>
- <!-- a basic sentence pattern -->
<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
</lst>
</fragmenter>
- <!-- Configure the standard formatter -->
<formatter name="html"
default="true"
class="solr.highlight.HtmlFormatter">
@@ -1558,35 +407,25 @@
</lst>
</formatter>
- <!-- Configure the standard encoder -->
<encoder name="html"
class="solr.highlight.HtmlEncoder" />
- <!-- Configure the standard fragListBuilder -->
<fragListBuilder name="simple"
class="solr.highlight.SimpleFragListBuilder"/>
-
- <!-- Configure the single fragListBuilder -->
+
<fragListBuilder name="single"
class="solr.highlight.SingleFragListBuilder"/>
-
- <!-- Configure the weighted fragListBuilder -->
+
<fragListBuilder name="weighted"
default="true"
class="solr.highlight.WeightedFragListBuilder"/>
-
- <!-- default tag FragmentsBuilder -->
+
<fragmentsBuilder name="default"
default="true"
class="solr.highlight.ScoreOrderFragmentsBuilder">
- <!--
- <lst name="defaults">
- <str name="hl.multiValuedSeparatorChar">/</str>
- </lst>
- -->
+
</fragmentsBuilder>
- <!-- multi-colored tag FragmentsBuilder -->
<fragmentsBuilder name="colored"
class="solr.highlight.ScoreOrderFragmentsBuilder">
<lst name="defaults">
@@ -1612,10 +451,7 @@
<boundaryScanner name="breakIterator"
class="solr.highlight.BreakIteratorBoundaryScanner">
<lst name="defaults">
- <!-- type should be one of CHARACTER, WORD(default), LINE and SENTENCE -->
<str name="hl.bs.type">WORD</str>
- <!-- language and country are used when constructing Locale object. -->
- <!-- And the Locale object will be used when getting instance of BreakIterator -->
<str name="hl.bs.language">en</str>
<str name="hl.bs.country">US</str>
</lst>
@@ -1623,29 +459,6 @@
</highlighting>
</searchComponent>
- <!-- Update Processors
-
- Chains of Update Processor Factories for dealing with Update
- Requests can be declared, and then used by name in Update
- Request Processors
-
- http://wiki.apache.org/solr/UpdateRequestProcessor
-
- -->
-
- <!-- Add unknown fields to the schema
-
- An example field type guessing update processor that will
- attempt to parse string-typed field values as Booleans, Longs,
- Doubles, or Dates, and then add schema fields with the guessed
- field types.
-
- This requires that the schema is both managed and mutable, by
- declaring schemaFactory as ManagedIndexSchemaFactory, with
- mutable specified as true.
-
- See http://wiki.apache.org/solr/GuessingFieldTypes
- -->
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
<processor class="solr.RemoveBlankFieldUpdateProcessorFactory"/>
@@ -1698,173 +511,18 @@
<processor class="solr.RunUpdateProcessorFactory"/>
</updateRequestProcessorChain>
-
- <!-- Deduplication
-
- An example dedup update processor that creates the "id" field
- on the fly based on the hash code of some other fields. This
- example has overwriteDupes set to false since we are using the
- id field as the signatureField and Solr will maintain
- uniqueness based on that anyway.
-
- -->
- <!--
- <updateRequestProcessorChain name="dedupe">
- <processor class="solr.processor.SignatureUpdateProcessorFactory">
- <bool name="enabled">true</bool>
- <str name="signatureField">id</str>
- <bool name="overwriteDupes">false</bool>
- <str name="fields">name,features,cat</str>
- <str name="signatureClass">solr.processor.Lookup3Signature</str>
- </processor>
- <processor class="solr.LogUpdateProcessorFactory" />
- <processor class="solr.RunUpdateProcessorFactory" />
- </updateRequestProcessorChain>
- -->
-
- <!-- Language identification
-
- This example update chain identifies the language of the incoming
- documents using the langid contrib. The detected language is
- written to field language_s. No field name mapping is done.
- The fields used for detection are text, title, subject and description,
- making this example suitable for detecting languages form full-text
- rich documents injected via ExtractingRequestHandler.
- See more about langId at http://wiki.apache.org/solr/LanguageDetection
- -->
- <!--
- <updateRequestProcessorChain name="langid">
- <processor class="org.apache.solr.update.processor.TikaLanguageIdentifierUpdateProcessorFactory">
- <str name="langid.fl">text,title,subject,description</str>
- <str name="langid.langField">language_s</str>
- <str name="langid.fallback">en</str>
- </processor>
- <processor class="solr.LogUpdateProcessorFactory" />
- <processor class="solr.RunUpdateProcessorFactory" />
- </updateRequestProcessorChain>
- -->
-
- <!-- Script update processor
-
- This example hooks in an update processor implemented using JavaScript.
-
- See more about the script update processor at http://wiki.apache.org/solr/ScriptUpdateProcessor
- -->
- <!--
- <updateRequestProcessorChain name="script">
- <processor class="solr.StatelessScriptUpdateProcessorFactory">
- <str name="script">update-script.js</str>
- <lst name="params">
- <str name="config_param">example config parameter</str>
- </lst>
- </processor>
- <processor class="solr.RunUpdateProcessorFactory" />
- </updateRequestProcessorChain>
- -->
-
- <!-- Response Writers
-
- http://wiki.apache.org/solr/QueryResponseWriter
-
- Request responses will be written using the writer specified by
- the 'wt' request parameter matching the name of a registered
- writer.
-
- The "default" writer is the default and will be used if 'wt' is
- not specified in the request.
- -->
- <!-- The following response writers are implicitly configured unless
- overridden...
- -->
- <!--
- <queryResponseWriter name="xml"
- default="true"
- class="solr.XMLResponseWriter" />
- <queryResponseWriter name="json" class="solr.JSONResponseWriter"/>
- <queryResponseWriter name="python" class="solr.PythonResponseWriter"/>
- <queryResponseWriter name="ruby" class="solr.RubyResponseWriter"/>
- <queryResponseWriter name="php" class="solr.PHPResponseWriter"/>
- <queryResponseWriter name="phps" class="solr.PHPSerializedResponseWriter"/>
- <queryResponseWriter name="csv" class="solr.CSVResponseWriter"/>
- <queryResponseWriter name="schema.xml" class="solr.SchemaXmlResponseWriter"/>
- -->
-
<queryResponseWriter name="json" class="solr.JSONResponseWriter">
- <!-- For the purposes of the tutorial, JSON responses are written as
- plain text so that they are easy to read in *any* browser.
- If you expect a MIME type of "application/json" just remove this override.
- -->
<str name="content-type">text/plain; charset=UTF-8</str>
</queryResponseWriter>
-
- <!--
- Custom response writers can be declared as needed...
- -->
+
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy">
<str name="template.base.dir">${velocity.template.base.dir:}</str>
</queryResponseWriter>
- <!-- XSLT response writer transforms the XML output by any xslt file found
- in Solr's conf/xslt directory. Changes to xslt files are checked for
- every xsltCacheLifetimeSeconds.
- -->
<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
<int name="xsltCacheLifetimeSeconds">5</int>
</queryResponseWriter>
- <!-- Query Parsers
-
- http://wiki.apache.org/solr/SolrQuerySyntax
-
- Multiple QParserPlugins can be registered by name, and then
- used in either the "defType" param for the QueryComponent (used
- by SearchHandler) or in LocalParams
- -->
- <!-- example of registering a query parser -->
- <!--
- <queryParser name="myparser" class="com.mycompany.MyQParserPlugin"/>
- -->
-
- <!-- Function Parsers
-
- http://wiki.apache.org/solr/FunctionQuery
-
- Multiple ValueSourceParsers can be registered by name, and then
- used as function names when using the "func" QParser.
- -->
- <!-- example of registering a custom function parser -->
- <!--
- <valueSourceParser name="myfunc"
- class="com.mycompany.MyValueSourceParser" />
- -->
-
-
- <!-- Document Transformers
- http://wiki.apache.org/solr/DocTransformers
- -->
- <!--
- Could be something like:
- <transformer name="db" class="com.mycompany.LoadFromDatabaseTransformer" >
- <int name="connection">jdbc://....</int>
- </transformer>
-
- To add a constant value to all docs, use:
- <transformer name="mytrans2" class="org.apache.solr.response.transform.ValueAugmenterFactory" >
- <int name="value">5</int>
- </transformer>
-
- If you want the user to still be able to change it with _value:something_ use this:
- <transformer name="mytrans3" class="org.apache.solr.response.transform.ValueAugmenterFactory" >
- <double name="defaultValue">5</double>
- </transformer>
-
- If you are using the QueryElevationComponent, you may wish to mark documents that get boosted. The
- EditorialMarkerFactory will do exactly that:
- <transformer name="qecBooster" class="org.apache.solr.response.transform.EditorialMarkerFactory" />
- -->
-
-
- <!-- Legacy config for the admin interface -->
<admin>
<defaultQuery>*:*</defaultQuery>
</admin>
diff --git a/ambari-logsearch/docker/docker-compose.yml b/ambari-logsearch/docker/docker-compose.yml
index e865ec3..7ec24b2 100644
--- a/ambari-logsearch/docker/docker-compose.yml
+++ b/ambari-logsearch/docker/docker-compose.yml
@@ -26,7 +26,7 @@ services:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
- image: solr:${SOLR_VERSION:-6.6.2}
+ image: solr:${SOLR_VERSION:-7.2.1}
restart: always
hostname: solr
ports:
diff --git a/ambari-logsearch/docker/solr.yml b/ambari-logsearch/docker/solr.yml
index f64c860..4722d2c 100644
--- a/ambari-logsearch/docker/solr.yml
+++ b/ambari-logsearch/docker/solr.yml
@@ -15,7 +15,7 @@
version: '3.3'
services:
solr:
- image: solr:${SOLR_VERSION:-6.6.2}
+ image: solr:${SOLR_VERSION:-7.2.1}
restart: always
networks:
- logsearch-network
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/templates/infra-solr-security.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/templates/infra-solr-security.json.j2
index 65d38e9..d7ee2de 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/templates/infra-solr-security.json.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/templates/infra-solr-security.json.j2
@@ -20,7 +20,7 @@
"class": "org.apache.solr.security.KerberosPlugin"
},
"authorization": {
- "class": "org.apache.ambari.infra.security.InfraRuleBasedAuthorizationPlugin",
+ "class": "org.apache.solr.security.InfraRuleBasedAuthorizationPlugin",
"user-role": {
"{{infra_solr_kerberos_service_user}}@{{kerberos_realm}}": "admin",
"{{logsearch_kerberos_service_user}}@{{kerberos_realm}}": ["{{infra_solr_role_logsearch}}", "{{infra_solr_role_ranger_admin}}", "{{infra_solr_role_dev}}"],
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/properties/infra-solr-env.sh.j2 b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/properties/infra-solr-env.sh.j2
index dd904f2..3a05bb3 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/properties/infra-solr-env.sh.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/properties/infra-solr-env.sh.j2
@@ -52,7 +52,7 @@ ZK_CLIENT_TIMEOUT="60000"
# By default the start script uses "localhost"; override the hostname here
# for production SolrCloud environments to control the hostname exposed to cluster state
-#SOLR_HOST="192.168.1.1"
+SOLR_HOST=`hostname -f`
# By default the start script uses UTC; override the timezone if needed
#SOLR_TIMEZONE="UTC"
@@ -108,13 +108,12 @@ SOLR_SSL_WANT_CLIENT_AUTH=false
#SOLR_SSL_PORT=
{% if security_enabled -%}
-SOLR_HOST=`hostname -f`
SOLR_JAAS_FILE={{infra_solr_jaas_file}}
SOLR_KERB_KEYTAB={{infra_solr_web_kerberos_keytab}}
SOLR_KERB_PRINCIPAL={{infra_solr_web_kerberos_principal}}
SOLR_KERB_NAME_RULES="{{infra_solr_kerberos_name_rules}}"
SOLR_OPTS="$SOLR_OPTS {{zk_security_opts}}"
-SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer"
+SOLR_AUTH_TYPE="kerberos"
SOLR_AUTHENTICATION_OPTS=" -DauthenticationPlugin=org.apache.solr.security.KerberosPlugin -Djava.security.auth.login.config=$SOLR_JAAS_FILE -Dsolr.kerberos.principal=${SOLR_KERB_PRINCIPAL} -Dsolr.kerberos.keytab=${SOLR_KERB_KEYTAB} -Dsolr.kerberos.cookie.domain=${SOLR_HOST}"
{% endif %}
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/atlas-solrconfig.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/atlas-solrconfig.xml
index cba4a4e..95340c6 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/atlas-solrconfig.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/atlas-solrconfig.xml
@@ -62,7 +62,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>5.5.1</luceneMatchVersion>
+ <luceneMatchVersion>6.6.0</luceneMatchVersion>
<!-- Data Directory
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/audit_logs-solrconfig.xml.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/audit_logs-solrconfig.xml.j2
index d56990a..eb822b2 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/audit_logs-solrconfig.xml.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/audit_logs-solrconfig.xml.j2
@@ -35,7 +35,7 @@ this file, see http://wiki.apache.org/solr/SolrConfigXml.
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>5.0.0</luceneMatchVersion>
+ <luceneMatchVersion>7.2.1</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
@@ -237,7 +237,10 @@ this file, see http://wiki.apache.org/solr/SolrConfigXml.
<mergeFactor>10</mergeFactor>
-->
<!-- LogSearch customization. Set to 5 to trigger purging of deleted documents more often -->
- <mergeFactor>{{logsearch_audit_logs_merge_factor}}</mergeFactor>
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+ <int name="maxMergeAtOnce">{{logsearch_audit_logs_merge_factor}}</int>
+ <int name="segmentsPerTier">{{logsearch_audit_logs_merge_factor}}</int>
+ </mergePolicyFactory>
<!-- Expert: Merge Scheduler
The Merge Scheduler in Lucene controls how merges are
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2
index ed80e84..8565203 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/service_logs-solrconfig.xml.j2
@@ -35,7 +35,7 @@ this file, see http://wiki.apache.org/solr/SolrConfigXml.
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>5.0.0</luceneMatchVersion>
+ <luceneMatchVersion>7.2.1</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
@@ -237,7 +237,10 @@ this file, see http://wiki.apache.org/solr/SolrConfigXml.
<mergeFactor>10</mergeFactor>
-->
<!-- LogSearch customization. Set to 5 to trigger purging of deleted documents more often -->
- <mergeFactor>{{logsearch_service_logs_merge_factor}}</mergeFactor>
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+ <int name="maxMergeAtOnce">{{logsearch_service_logs_merge_factor}}</int>
+ <int name="segmentsPerTier">{{logsearch_service_logs_merge_factor}}</int>
+ </mergePolicyFactory>
<!-- Expert: Merge Scheduler
The Merge Scheduler in Lucene controls how merges are
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/properties/ranger-solrconfig.xml.j2 b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/properties/ranger-solrconfig.xml.j2
index 25dbb7a..e484a38 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/properties/ranger-solrconfig.xml.j2
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/properties/ranger-solrconfig.xml.j2
@@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- <luceneMatchVersion>5.2.0</luceneMatchVersion>
+ <luceneMatchVersion>6.6.0</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
@@ -234,7 +234,10 @@
-->
<!-- Ranger customization. Set to 5 to trigger purging of deleted documents more often -->
- <mergeFactor>{{ranger_audit_logs_merge_factor}}</mergeFactor>
+ <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+ <int name="maxMergeAtOnce">{{ranger_audit_logs_merge_factor}}</int>
+ <int name="segmentsPerTier">{{ranger_audit_logs_merge_factor}}</int>
+ </mergePolicyFactory>
<!-- Expert: Merge Scheduler
The Merge Scheduler in Lucene controls how merges are
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.