You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/10/23 00:05:51 UTC
[30/52] [abbrv] [partial] lucene-solr:jira/gradle: Add gradle support
for Solr
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
deleted file mode 100644
index 1747fa2..0000000
--- a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
+++ /dev/null
@@ -1,396 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.core;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.cloud.CloudDescriptor;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.util.PropertiesUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A Solr core descriptor
- *
- * @since solr 1.3
- */
-public class CoreDescriptor {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- // Properties file name constants
- public static final String CORE_NAME = "name";
- public static final String CORE_CONFIG = "config";
- public static final String CORE_DATADIR = "dataDir";
- public static final String CORE_ULOGDIR = "ulogDir";
- public static final String CORE_SCHEMA = "schema";
- public static final String CORE_SHARD = "shard";
- public static final String CORE_COLLECTION = "collection";
- public static final String CORE_ROLES = "roles";
- public static final String CORE_PROPERTIES = "properties";
- public static final String CORE_LOADONSTARTUP = "loadOnStartup";
- public static final String CORE_TRANSIENT = "transient";
- public static final String CORE_NODE_NAME = "coreNodeName";
- public static final String CORE_CONFIGSET = "configSet";
- public static final String CORE_CONFIGSET_PROPERTIES = "configSetProperties";
- public static final String SOLR_CORE_PROP_PREFIX = "solr.core.";
-
- public static final String DEFAULT_EXTERNAL_PROPERTIES_FILE = "conf" + File.separator + "solrcore.properties";
-
- /**
- * Whether this core was configured using a configSet that was trusted.
- * This helps in avoiding the loading of plugins that have potential
- * vulnerabilities, when the configSet was not uploaded from a trusted
- * user.
- */
- private boolean trustedConfigSet = true;
-
- /**
- * Get the standard properties in persistable form
- * @return the standard core properties in persistable form
- */
- public Properties getPersistableStandardProperties() {
- return originalCoreProperties;
- }
-
- /**
- * Get user-defined core properties in persistable form
- * @return user-defined core properties in persistable form
- */
- public Properties getPersistableUserProperties() {
- return originalExtraProperties;
- }
-
- private static ImmutableMap<String, String> defaultProperties = new ImmutableMap.Builder<String, String>()
- .put(CORE_CONFIG, "solrconfig.xml")
- .put(CORE_SCHEMA, "schema.xml")
- .put(CORE_CONFIGSET_PROPERTIES, ConfigSetProperties.DEFAULT_FILENAME)
- .put(CORE_DATADIR, "data" + File.separator)
- .put(CORE_TRANSIENT, "false")
- .put(CORE_LOADONSTARTUP, "true")
- .build();
-
- private static ImmutableList<String> requiredProperties = ImmutableList.of(
- CORE_NAME
- );
-
- public static ImmutableList<String> standardPropNames = ImmutableList.of(
- CORE_NAME,
- CORE_CONFIG,
- CORE_DATADIR,
- CORE_ULOGDIR,
- CORE_SCHEMA,
- CORE_PROPERTIES,
- CORE_CONFIGSET_PROPERTIES,
- CORE_LOADONSTARTUP,
- CORE_TRANSIENT,
- CORE_CONFIGSET,
- // cloud props
- CORE_SHARD,
- CORE_COLLECTION,
- CORE_ROLES,
- CORE_NODE_NAME,
- CloudDescriptor.NUM_SHARDS
- );
-
- private final CloudDescriptor cloudDesc;
-
- private final Path instanceDir;
-
- /** The original standard core properties, before substitution */
- protected final Properties originalCoreProperties = new Properties();
-
- /** The original extra core properties, before substitution */
- protected final Properties originalExtraProperties = new Properties();
-
- /** The properties for this core, as available through getProperty() */
- protected final Properties coreProperties = new Properties();
-
- /** The properties for this core, substitutable by resource loaders */
- protected final Properties substitutableProperties = new Properties();
-
- public CoreDescriptor(String name, Path instanceDir, Properties containerProperties,
- boolean isZooKeeperAware, String... properties) {
- this(name, instanceDir, toMap(properties), containerProperties, isZooKeeperAware);
- }
-
- private static Map<String, String> toMap(String... properties) {
- Map<String, String> props = new HashMap<>();
- assert properties.length % 2 == 0;
- for (int i = 0; i < properties.length; i += 2) {
- props.put(properties[i], properties[i+1]);
- }
- return props;
- }
-
- /**
- * Create a new CoreDescriptor with a given name and instancedir
- * @param name the CoreDescriptor's name
- * @param instanceDir the CoreDescriptor's instancedir
- * @param containerProperties the enclosing container properties for variable resolution
- * @param isZooKeeperAware whether we are part of SolrCloud or not.
- */
- public CoreDescriptor(String name, Path instanceDir,
- Properties containerProperties, boolean isZooKeeperAware) {
- this(name, instanceDir, Collections.emptyMap(), containerProperties, isZooKeeperAware);
- }
-
- /**
- * Create a new CoreDescriptor using the properties of an existing one
- * @param coreName the new CoreDescriptor's name
- * @param other the CoreDescriptor to copy
- */
- public CoreDescriptor(String coreName, CoreDescriptor other) {
- this.cloudDesc = other.cloudDesc;
- this.instanceDir = other.instanceDir;
- this.originalExtraProperties.putAll(other.originalExtraProperties);
- this.originalCoreProperties.putAll(other.originalCoreProperties);
- this.coreProperties.putAll(other.coreProperties);
- this.substitutableProperties.putAll(other.substitutableProperties);
- this.coreProperties.setProperty(CORE_NAME, coreName);
- this.originalCoreProperties.setProperty(CORE_NAME, coreName);
- this.substitutableProperties.setProperty(SOLR_CORE_PROP_PREFIX + CORE_NAME, coreName);
- this.trustedConfigSet = other.trustedConfigSet;
- }
-
- /**
- * Create a new CoreDescriptor.
- * @param name the CoreDescriptor's name
- * @param instanceDir a Path resolving to the instanceDir
- * @param coreProps a Map of the properties for this core
- * @param containerProperties the properties from the enclosing container.
- * @param isZooKeeperAware if true, we ar in SolrCloud mode.
- */
-
-
- public CoreDescriptor(String name, Path instanceDir, Map<String, String> coreProps,
- Properties containerProperties, boolean isZooKeeperAware) {
- this.instanceDir = instanceDir;
-
- originalCoreProperties.setProperty(CORE_NAME, name);
-
- name = PropertiesUtil.substituteProperty(checkPropertyIsNotEmpty(name, CORE_NAME),
- containerProperties);
-
- coreProperties.putAll(defaultProperties);
- coreProperties.put(CORE_NAME, name);
-
- for (String propname : coreProps.keySet()) {
-
- String propvalue = coreProps.get(propname);
-
- if (isUserDefinedProperty(propname))
- originalExtraProperties.put(propname, propvalue);
- else
- originalCoreProperties.put(propname, propvalue);
-
- if (!requiredProperties.contains(propname)) // Required props are already dealt with
- coreProperties.setProperty(propname,
- PropertiesUtil.substituteProperty(propvalue, containerProperties));
- }
-
- loadExtraProperties();
- buildSubstitutableProperties();
-
- // TODO maybe make this a CloudCoreDescriptor subclass?
- if (isZooKeeperAware) {
- cloudDesc = new CloudDescriptor(name, coreProperties, this);
- }
- else {
- cloudDesc = null;
- }
-
- log.debug("Created CoreDescriptor: " + coreProperties);
- }
-
- /**
- * Load properties specified in an external properties file.
- *
- * The file to load can be specified in a {@code properties} property on
- * the original Properties object used to create this CoreDescriptor. If
- * this has not been set, then we look for {@code conf/solrcore.properties}
- * underneath the instance dir.
- *
- * File paths are taken as read from the core's instance directory
- * if they are not absolute.
- */
- protected void loadExtraProperties() {
- String filename = coreProperties.getProperty(CORE_PROPERTIES, DEFAULT_EXTERNAL_PROPERTIES_FILE);
- Path propertiesFile = instanceDir.resolve(filename);
- if (Files.exists(propertiesFile)) {
- try (InputStream is = Files.newInputStream(propertiesFile)) {
- Properties externalProps = new Properties();
- externalProps.load(new InputStreamReader(is, StandardCharsets.UTF_8));
- coreProperties.putAll(externalProps);
- } catch (IOException e) {
- String message = String.format(Locale.ROOT, "Could not load properties from %s: %s:",
- propertiesFile.toString(), e.toString());
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message);
- }
- }
- }
-
- /**
- * Create the properties object used by resource loaders, etc, for property
- * substitution. The default solr properties are prefixed with 'solr.core.', so,
- * e.g., 'name' becomes 'solr.core.name'
- */
- protected void buildSubstitutableProperties() {
- for (String propName : coreProperties.stringPropertyNames()) {
- String propValue = coreProperties.getProperty(propName);
- if (!isUserDefinedProperty(propName))
- propName = SOLR_CORE_PROP_PREFIX + propName;
- substitutableProperties.setProperty(propName, propValue);
- }
- substitutableProperties.setProperty("solr.core.instanceDir", instanceDir.toAbsolutePath().toString());
- }
-
- /**
- * Is this property a Solr-standard property, or is it an extra property
- * defined per-core by the user?
- * @param propName the Property name
- * @return {@code true} if this property is user-defined
- */
- protected static boolean isUserDefinedProperty(String propName) {
- return !standardPropNames.contains(propName);
- }
-
- public static String checkPropertyIsNotEmpty(String value, String propName) {
- if (StringUtils.isEmpty(value)) {
- String message = String.format(Locale.ROOT, "Cannot create core with empty %s value", propName);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message);
- }
- return value;
- }
-
- public String getPropertiesName() {
- return coreProperties.getProperty(CORE_PROPERTIES);
- }
-
- public String getDataDir() {
- return coreProperties.getProperty(CORE_DATADIR);
- }
-
- public boolean usingDefaultDataDir() {
- return defaultProperties.get(CORE_DATADIR).equals(coreProperties.getProperty(CORE_DATADIR));
- }
-
- /**
- * @return the core instance directory
- */
- public Path getInstanceDir() {
- return instanceDir;
- }
-
- /**@return the core configuration resource name. */
- public String getConfigName() {
- return coreProperties.getProperty(CORE_CONFIG);
- }
-
- /**@return the core schema resource name. */
- public String getSchemaName() {
- return coreProperties.getProperty(CORE_SCHEMA);
- }
-
- /**@return the initial core name */
- public String getName() {
- return coreProperties.getProperty(CORE_NAME);
- }
-
- public void setProperty(String prop, String val) {
- if (substitutableProperties.containsKey(prop)) {
- substitutableProperties.setProperty(prop, val);
- return;
- }
- coreProperties.setProperty(prop, val);
- }
-
- public String getCollectionName() {
- return cloudDesc == null ? null : cloudDesc.getCollectionName();
- }
-
- public CloudDescriptor getCloudDescriptor() {
- return cloudDesc;
- }
-
- public boolean isLoadOnStartup() {
- String tmp = coreProperties.getProperty(CORE_LOADONSTARTUP, "false");
- return Boolean.parseBoolean(tmp);
- }
-
- public boolean isTransient() {
- String tmp = coreProperties.getProperty(CORE_TRANSIENT, "false");
- return PropertiesUtil.toBoolean(tmp);
- }
-
- public String getUlogDir() {
- return coreProperties.getProperty(CORE_ULOGDIR);
- }
-
- /**
- * Returns a specific property defined on this CoreDescriptor
- * @param prop - value to read from the properties structure.
- * @param defVal - return if no property found.
- * @return associated string. May be null.
- */
- public String getCoreProperty(String prop, String defVal) {
- return coreProperties.getProperty(prop, defVal);
- }
-
- /**
- * Returns all substitutable properties defined on this CoreDescriptor
- * @return all substitutable properties defined on this CoreDescriptor
- */
- public Properties getSubstitutableProperties() {
- return substitutableProperties;
- }
-
- @Override
- public String toString() {
- return "CoreDescriptor[name=" + this.getName() + ";instanceDir=" + this.getInstanceDir() + "]";
- }
-
- public String getConfigSet() {
- return coreProperties.getProperty(CORE_CONFIGSET);
- }
-
- public String getConfigSetPropertiesName() {
- return coreProperties.getProperty(CORE_CONFIGSET_PROPERTIES);
- }
-
- public boolean isConfigSetTrusted() {
- return trustedConfigSet;
- }
-
- public void setConfigSetTrusted(boolean trusted) {
- this.trustedConfigSet = trusted;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
deleted file mode 100644
index 76eb5c4..0000000
--- a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.core;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileVisitOption;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Lists;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.util.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Persists CoreDescriptors as properties files
- */
-public class CorePropertiesLocator implements CoresLocator {
-
- public static final String PROPERTIES_FILENAME = "core.properties";
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private final Path rootDirectory;
-
- public CorePropertiesLocator(Path coreDiscoveryRoot) {
- this.rootDirectory = coreDiscoveryRoot;
- log.debug("Config-defined core root directory: {}", this.rootDirectory);
- }
-
- @Override
- public void create(CoreContainer cc, CoreDescriptor... coreDescriptors) {
- for (CoreDescriptor cd : coreDescriptors) {
- Path propertiesFile = this.rootDirectory.resolve(cd.getInstanceDir()).resolve(PROPERTIES_FILENAME);
- if (Files.exists(propertiesFile))
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Could not create a new core in " + cd.getInstanceDir()
- + " as another core is already defined there");
- writePropertiesFile(cd, propertiesFile);
- }
- }
-
- // TODO, this isn't atomic! If we crash in the middle of a rename, we
- // could end up with two cores with identical names, in which case one of
- // them won't start up. Are we happy with this?
-
- @Override
- public void persist(CoreContainer cc, CoreDescriptor... coreDescriptors) {
- for (CoreDescriptor cd : coreDescriptors) {
- Path propFile = this.rootDirectory.resolve(cd.getInstanceDir()).resolve(PROPERTIES_FILENAME);
- writePropertiesFile(cd, propFile);
- }
- }
-
- private void writePropertiesFile(CoreDescriptor cd, Path propfile) {
- Properties p = buildCoreProperties(cd);
- try {
- FileUtils.createDirectories(propfile.getParent()); // Handling for symlinks.
- try (Writer os = new OutputStreamWriter(Files.newOutputStream(propfile), StandardCharsets.UTF_8)) {
- p.store(os, "Written by CorePropertiesLocator");
- }
- }
- catch (IOException e) {
- log.error("Couldn't persist core properties to {}: {}", propfile, e.getMessage());
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Couldn't persist core properties to " + propfile.toAbsolutePath().toString() + " : " + e.getMessage());
- }
- }
-
- @Override
- public void delete(CoreContainer cc, CoreDescriptor... coreDescriptors) {
- if (coreDescriptors == null) {
- return;
- }
- for (CoreDescriptor cd : coreDescriptors) {
- if (cd == null) continue;
- Path propfile = this.rootDirectory.resolve(cd.getInstanceDir()).resolve(PROPERTIES_FILENAME);
- try {
- Files.deleteIfExists(propfile);
- } catch (IOException e) {
- log.warn("Couldn't delete core properties file {}: {}", propfile, e.getMessage());
- }
- }
- }
-
- @Override
- public void rename(CoreContainer cc, CoreDescriptor oldCD, CoreDescriptor newCD) {
- String oldName = newCD.getPersistableStandardProperties().getProperty(CoreDescriptor.CORE_NAME);
- String newName = newCD.coreProperties.getProperty(CoreDescriptor.CORE_NAME);
- if (oldName == null ||
- (newName != null && oldName.equals(newName) == false)) {
- newCD.getPersistableStandardProperties().put(CoreDescriptor.CORE_NAME, newName);
- }
- persist(cc, newCD);
- }
-
- @Override
- public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2) {
- persist(cc, cd1, cd2);
- }
-
- @Override
- public List<CoreDescriptor> discover(final CoreContainer cc) {
- log.debug("Looking for core definitions underneath {}", rootDirectory);
- final List<CoreDescriptor> cds = Lists.newArrayList();
- try {
- Set<FileVisitOption> options = new HashSet<>();
- options.add(FileVisitOption.FOLLOW_LINKS);
- final int maxDepth = 256;
- Files.walkFileTree(this.rootDirectory, options, maxDepth, new SimpleFileVisitor<Path>() {
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
- if (file.getFileName().toString().equals(PROPERTIES_FILENAME)) {
- CoreDescriptor cd = buildCoreDescriptor(file, cc);
- if (cd != null) {
- log.debug("Found core {} in {}", cd.getName(), cd.getInstanceDir());
- cds.add(cd);
- }
- return FileVisitResult.SKIP_SIBLINGS;
- }
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
- // if we get an error on the root, then fail the whole thing
- // otherwise, log a warning and continue to try and load other cores
- if (file.equals(rootDirectory)) {
- log.error("Error reading core root directory {}: {}", file, exc);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading core root directory");
- }
- log.warn("Error visiting {}: {}", file, exc);
- return FileVisitResult.CONTINUE;
- }
- });
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Couldn't walk file tree under " + this.rootDirectory, e);
- }
- log.info("Found {} core definitions underneath {}", cds.size(), rootDirectory);
- if (cds.size() > 0) {
- log.info("Cores are: {}", cds.stream().map(CoreDescriptor::getName).collect(Collectors.toList()));
- }
- return cds;
- }
-
- protected CoreDescriptor buildCoreDescriptor(Path propertiesFile, CoreContainer cc) {
-
- Path instanceDir = propertiesFile.getParent();
- Properties coreProperties = new Properties();
- try (InputStream fis = Files.newInputStream(propertiesFile)) {
- coreProperties.load(new InputStreamReader(fis, StandardCharsets.UTF_8));
- String name = createName(coreProperties, instanceDir);
- Map<String, String> propMap = new HashMap<>();
- for (String key : coreProperties.stringPropertyNames()) {
- propMap.put(key, coreProperties.getProperty(key));
- }
- CoreDescriptor ret = new CoreDescriptor(name, instanceDir, propMap, cc.getContainerProperties(), cc.isZooKeeperAware());
- ret.loadExtraProperties();
- return ret;
- }
- catch (IOException e) {
- log.error("Couldn't load core descriptor from {}:{}", propertiesFile, e.toString());
- return null;
- }
-
- }
-
- protected static String createName(Properties p, Path instanceDir) {
- return p.getProperty(CoreDescriptor.CORE_NAME, instanceDir.getFileName().toString());
- }
-
- protected Properties buildCoreProperties(CoreDescriptor cd) {
- Properties p = new Properties();
- p.putAll(cd.getPersistableStandardProperties());
- p.putAll(cd.getPersistableUserProperties());
- return p;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/CoreSorter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreSorter.java b/solr/core/src/java/org/apache/solr/core/CoreSorter.java
deleted file mode 100644
index cccd84b..0000000
--- a/solr/core/src/java/org/apache/solr/core/CoreSorter.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.core;
-
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.solr.cloud.CloudDescriptor;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-
-import static java.util.Collections.emptyList;
-import static java.util.stream.Collectors.toList;
-
-/**
- * This is a utility class that sorts cores in such a way as to minimize other cores
- * waiting for replicas in the current node. This helps in avoiding leaderVote timeouts
- * happening in other nodes of the cluster
- */
-public class CoreSorter {
- Map<String, CountsForEachShard> shardsVsReplicaCounts = new LinkedHashMap<>();
- CoreContainer cc;
- private static final CountsForEachShard zero = new CountsForEachShard(0, 0, 0);
-
- public final static Comparator<CountsForEachShard> countsComparator = (c1, c2) -> {
- if (c1 == null) c1 = zero;//just to avoid NPE
- if (c2 == null) c2 = zero;
- if (c1.totalReplicasInDownNodes < c2.totalReplicasInDownNodes) {
- //Prioritize replicas with least no:of down nodes waiting.
- //It's better to bring up a node that is a member of a shard
- //with 0 down nodes than 1 down node because it will make the shard
- // complete earlier and avoid waiting by the other live nodes
- if (c1.totalReplicasInLiveNodes > 0) {
- //means nobody else is waiting for this , so no need to prioritize
- return -1;
- }
- }
- if (c2.totalReplicasInDownNodes < c1.totalReplicasInDownNodes) {
- //same is the above, just to take care of the case where c2 has to be prioritized
- if (c2.totalReplicasInLiveNodes > 0) {
- //means nobody else is waiting for this , so no need to priotitize
- return 1;
- }
- }
-
- //Prioritize replicas where most no:of other nodes are waiting for
- // For example if 1 other replicas are waiting for this replica, then
- // prioritize that over the replica were zero other nodes are waiting
- if (c1.totalReplicasInLiveNodes > c2.totalReplicasInLiveNodes) return -1;
- if (c2.totalReplicasInLiveNodes > c1.totalReplicasInLiveNodes) return 1;
-
- //If all else is same. prioritize fewer replicas I have because that will complete the
- //quorum for shard faster. If I have only one replica for a shard I can finish it faster
- // than a shard with 2 replicas in this node
- if (c1.myReplicas < c2.myReplicas) return -1;
- if (c2.myReplicas < c1.myReplicas) return 1;
- //if everything is same return 0
- return 0;
- };
-
-
- public CoreSorter init(CoreContainer cc) {
- this.cc = cc;
- if (cc == null || !cc.isZooKeeperAware()) {
- return this;
- }
- String myNodeName = getNodeName();
- ClusterState state = cc.getZkController().getClusterState();
- for (CloudDescriptor cloudDescriptor : getCloudDescriptors()) {
- String coll = cloudDescriptor.getCollectionName();
- String sliceName = getShardName(cloudDescriptor);
- if (shardsVsReplicaCounts.containsKey(sliceName)) continue;
- CountsForEachShard c = new CountsForEachShard(0, 0, 0);
- for (Replica replica : getReplicas(state, coll, cloudDescriptor.getShardId())) {
- if (replica.getNodeName().equals(myNodeName)) {
- c.myReplicas++;
- } else {
- Set<String> liveNodes = state.getLiveNodes();
- if (liveNodes.contains(replica.getNodeName())) {
- c.totalReplicasInLiveNodes++;
- } else {
- c.totalReplicasInDownNodes++;
- }
- }
- }
- shardsVsReplicaCounts.put(sliceName, c);
- }
-
- return this;
-
- }
-
-
- public int compare(CoreDescriptor cd1, CoreDescriptor cd2) {
- String s1 = getShardName(cd1.getCloudDescriptor());
- String s2 = getShardName(cd2.getCloudDescriptor());
- if (s1 == null || s2 == null) return cd1.getName().compareTo(cd2.getName());
- CountsForEachShard c1 = shardsVsReplicaCounts.get(s1);
- CountsForEachShard c2 = shardsVsReplicaCounts.get(s2);
- int result = countsComparator.compare(c1, c2);
- return result == 0 ? s1.compareTo(s2) : result;
- }
-
-
- static class CountsForEachShard {
- public int totalReplicasInDownNodes = 0, myReplicas = 0, totalReplicasInLiveNodes = 0;
-
- public CountsForEachShard(int totalReplicasInDownNodes, int totalReplicasInLiveNodes,int myReplicas) {
- this.totalReplicasInDownNodes = totalReplicasInDownNodes;
- this.myReplicas = myReplicas;
- this.totalReplicasInLiveNodes = totalReplicasInLiveNodes;
- }
-
- public boolean equals(Object obj) {
- if (obj instanceof CountsForEachShard) {
- CountsForEachShard that = (CountsForEachShard) obj;
- return that.totalReplicasInDownNodes == totalReplicasInDownNodes && that.myReplicas == myReplicas;
-
- }
- return false;
- }
-
- @Override
- public String toString() {
- return "down : " + totalReplicasInDownNodes + " , up : " + totalReplicasInLiveNodes + " my : " + myReplicas;
- }
-
-
- }
-
- static String getShardName(CloudDescriptor cd) {
- return cd == null ?
- null :
- cd.getCollectionName()
- + "_"
- + cd.getShardId();
- }
-
-
- String getNodeName() {
- return cc.getNodeConfig().getNodeName();
- }
-
- /**Return all replicas for a given collection+slice combo
- */
- Collection<Replica> getReplicas(ClusterState cs, String coll, String slice) {
- DocCollection c = cs.getCollectionOrNull(coll);
- if (c == null) return emptyList();
- Slice s = c.getSlice(slice);
- if (s == null) return emptyList();
- return s.getReplicas();
- }
-
-
- /**return cloud descriptors for all cores in this node
- */
- Collection<CloudDescriptor> getCloudDescriptors() {
- return cc.getCores()
- .stream()
- .map((core) -> core.getCoreDescriptor().getCloudDescriptor())
- .collect(toList());
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/CoresLocator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoresLocator.java b/solr/core/src/java/org/apache/solr/core/CoresLocator.java
deleted file mode 100644
index 52927f1..0000000
--- a/solr/core/src/java/org/apache/solr/core/CoresLocator.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.core;
-
-import java.util.List;
-
-/**
- * Manage the discovery and persistence of core definitions across Solr restarts
- */
-public interface CoresLocator {
-
- /**
- * Make new cores available for discovery
- * @param cc the CoreContainer
- * @param coreDescriptors CoreDescriptors to persist
- */
- public void create(CoreContainer cc, CoreDescriptor... coreDescriptors);
-
- /**
- * Ensure that the core definitions from the passed in CoreDescriptors
- * will persist across container restarts.
- * @param cc the CoreContainer
- * @param coreDescriptors CoreDescriptors to persist
- */
- public void persist(CoreContainer cc, CoreDescriptor... coreDescriptors);
-
- /**
- * Ensure that the core definitions from the passed in CoreDescriptors
- * are not available for discovery
- * @param cc the CoreContainer
- * @param coreDescriptors CoreDescriptors of the cores to remove
- */
- public void delete(CoreContainer cc, CoreDescriptor... coreDescriptors);
-
- /**
- * Persist the new name of a renamed core
- * @param cc the CoreContainer
- * @param oldCD the CoreDescriptor of the core before renaming
- * @param newCD the CoreDescriptor of the core after renaming
- */
- public void rename(CoreContainer cc, CoreDescriptor oldCD, CoreDescriptor newCD);
-
- /**
- * Swap two core definitions
- * @param cc the CoreContainer
- * @param cd1 the core descriptor of the first core, after swapping
- * @param cd2 the core descriptor of the second core, after swapping
- */
- public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2);
-
- /**
- * Load all the CoreDescriptors from persistence store
- * @param cc the CoreContainer
- * @return a list of all CoreDescriptors found
- */
- public List<CoreDescriptor> discover(CoreContainer cc);
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/Diagnostics.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/Diagnostics.java b/solr/core/src/java/org/apache/solr/core/Diagnostics.java
deleted file mode 100644
index d7d6178..0000000
--- a/solr/core/src/java/org/apache/solr/core/Diagnostics.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.core;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadInfo;
-
-public class Diagnostics {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public interface Callable {
- public void call(Object... data); // data depends on the context
- }
-
- public static void call(Callable callable, Object... data) {
- try {
- callable.call(data);
- } catch (Exception e) {
- log.error("TEST HOOK EXCEPTION", e);
- }
- }
-
- public static void logThreadDumps(String message) {
- StringBuilder sb = new StringBuilder(32768);
- if (message == null) message = "============ THREAD DUMP REQUESTED ============";
- sb.append(message);
- sb.append("\n");
- ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads(true, true);
- for (ThreadInfo info : threads) {
- sb.append(info);
- // sb.append("\n");
- }
- log.error(sb.toString());
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
deleted file mode 100644
index fab3300..0000000
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ /dev/null
@@ -1,434 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.core;
-
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.NoSuchFileException;
-import java.util.Arrays;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FilterDirectory;
-import org.apache.lucene.store.FlushInfo;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.LockFactory;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.core.CachingDirectoryFactory.CloseListener;
-import org.apache.solr.util.plugin.NamedListInitializedPlugin;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Provides access to a Directory implementation. You must release every
- * Directory that you get.
- */
-public abstract class DirectoryFactory implements NamedListInitializedPlugin,
- Closeable {
-
- // Estimate 10M docs, 100GB size, to avoid caching by NRTCachingDirectory
- // Stayed away from upper bounds of the int/long in case any other code tried to aggregate these numbers.
- // A large estimate should currently have no other side effects.
- public static final IOContext IOCONTEXT_NO_CACHE = new IOContext(new FlushInfo(10*1000*1000, 100L*1000*1000*1000));
-
- protected static final String INDEX_W_TIMESTAMP_REGEX = "index\\.[0-9]{17}"; // see SnapShooter.DATE_FMT
-
- // May be set by sub classes as data root, in which case getDataHome will use it as base
- protected Path dataHomePath;
-
- // hint about what the directory contains - default is index directory
- public enum DirContext {DEFAULT, META_DATA}
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- // Available lock types
- public final static String LOCK_TYPE_SIMPLE = "simple";
- public final static String LOCK_TYPE_NATIVE = "native";
- public final static String LOCK_TYPE_SINGLE = "single";
- public final static String LOCK_TYPE_NONE = "none";
- public final static String LOCK_TYPE_HDFS = "hdfs";
-
- protected volatile CoreContainer coreContainer;
-
- /**
- * Indicates a Directory will no longer be used, and when its ref count
- * hits 0, it can be closed. On close all directories will be closed
- * whether this has been called or not. This is simply to allow early cleanup.
- *
- * @throws IOException If there is a low-level I/O error.
- */
- public abstract void doneWithDirectory(Directory directory) throws IOException;
-
- /**
- * Adds a close listener for a Directory.
- */
- public abstract void addCloseListener(Directory dir, CloseListener closeListener);
-
- /**
- * Close the this and all of the Directories it contains.
- *
- * @throws IOException If there is a low-level I/O error.
- */
- @Override
- public abstract void close() throws IOException;
-
- /**
- * Creates a new Directory for a given path.
- *
- * @throws IOException If there is a low-level I/O error.
- */
- protected abstract Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException;
-
- /**
- * Creates a new LockFactory for a given path.
- * @param rawLockType A string value as passed in config. Every factory should at least support 'none' to disable locking.
- * @throws IOException If there is a low-level I/O error.
- */
- protected abstract LockFactory createLockFactory(String rawLockType) throws IOException;
-
- /**
- * Returns true if a Directory exists for a given path in the underlying (stable) storage <em>and</em>
- * contains at least one file.
- * Note that the existence of a {@link Directory} <em>Object</em> as returned by a previous call to the
- * {@link #get} method (on the specified <code>path</code>) is not enough to cause this method to return
- * true. Some prior user of that Directory must have written & synced at least one file to that
- * Directory (and at least one file must still exist)
- *
- * @throws IOException If there is a low-level I/O error.
- */
- public abstract boolean exists(String path) throws IOException;
-
- /**
- * Removes the Directory's persistent storage.
- * For example: A file system impl may remove the
- * on disk directory.
- * @throws IOException If there is a low-level I/O error.
- *
- */
- public abstract void remove(Directory dir) throws IOException;
-
- /**
- * Removes the Directory's persistent storage.
- * For example: A file system impl may remove the
- * on disk directory.
- * @throws IOException If there is a low-level I/O error.
- *
- */
- public abstract void remove(Directory dir, boolean afterCoreClose) throws IOException;
-
- /**
- * This remove is special in that it may be called even after
- * the factory has been closed. Remove only makes sense for
- * persistent directory factories.
- *
- * @param path to remove
- * @param afterCoreClose whether to wait until after the core is closed.
- * @throws IOException If there is a low-level I/O error.
- */
- public abstract void remove(String path, boolean afterCoreClose) throws IOException;
-
- /**
- * This remove is special in that it may be called even after
- * the factory has been closed. Remove only makes sense for
- * persistent directory factories.
- *
- * @param path to remove
- * @throws IOException If there is a low-level I/O error.
- */
- public abstract void remove(String path) throws IOException;
-
- /**
- * @param directory to calculate size of
- * @return size in bytes
- * @throws IOException on low level IO error
- */
- public long size(Directory directory) throws IOException {
- return sizeOfDirectory(directory);
- }
-
- /**
- * @param path to calculate size of
- * @return size in bytes
- * @throws IOException on low level IO error
- */
- public long size(String path) throws IOException {
- Directory dir = get(path, DirContext.DEFAULT, null);
- long size;
- try {
- size = sizeOfDirectory(dir);
- } finally {
- release(dir);
- }
- return size;
- }
-
- /**
- * Override for more efficient moves.
- *
- * Intended for use with replication - use
- * carefully - some Directory wrappers will
- * cache files for example.
- *
- * @throws IOException If there is a low-level I/O error.
- */
- public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
- toDir.copyFrom(fromDir, fileName, fileName, ioContext);
- fromDir.deleteFile(fileName);
- }
-
- // sub classes perform an atomic rename if possible, otherwise fall back to delete + rename
- // this is important to support for index roll over durability after crashes
- public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException {
- try {
- dir.deleteFile(toName);
- } catch (FileNotFoundException | NoSuchFileException e) {
-
- } catch (Exception e) {
- log.error("Exception deleting file", e);
- }
-
- dir.rename(fileName, toName);
- }
-
- /**
- * Returns the Directory for a given path, using the specified rawLockType.
- * Will return the same Directory instance for the same path.
- *
- *
- * @throws IOException If there is a low-level I/O error.
- */
- public abstract Directory get(String path, DirContext dirContext, String rawLockType)
- throws IOException;
-
- /**
- * Increment the number of references to the given Directory. You must call
- * release for every call to this method.
- *
- */
- public abstract void incRef(Directory directory);
-
-
- /**
- * @return true if data is kept after close.
- */
- public abstract boolean isPersistent();
-
- /**
- * @return true if storage is shared.
- */
- public boolean isSharedStorage() {
- return false;
- }
-
- /**
- * Releases the Directory so that it may be closed when it is no longer
- * referenced.
- *
- * @throws IOException If there is a low-level I/O error.
- */
- public abstract void release(Directory directory) throws IOException;
-
- /**
- * Normalize a given path.
- *
- * @param path to normalize
- * @return normalized path
- * @throws IOException on io error
- */
- public String normalize(String path) throws IOException {
- return path;
- }
-
- /**
- * @param path the path to check
- * @return true if absolute, as in not relative
- */
- public boolean isAbsolute(String path) {
- // back compat
- return new File(path).isAbsolute();
- }
-
- public static long sizeOfDirectory(Directory directory) throws IOException {
- final String[] files = directory.listAll();
- long size = 0;
-
- for (final String file : files) {
- size += sizeOf(directory, file);
- if (size < 0) {
- break;
- }
- }
-
- return size;
- }
-
- public static long sizeOf(Directory directory, String file) throws IOException {
- try {
- return directory.fileLength(file);
- } catch (IOException e) {
- // could be a race, file no longer exists, access denied, is a directory, etc.
- return 0;
- }
- }
-
- /**
- * Delete the files in the Directory
- */
- public static boolean empty(Directory dir) {
- boolean isSuccess = true;
- String contents[];
- try {
- contents = dir.listAll();
- if (contents != null) {
- for (String file : contents) {
- dir.deleteFile(file);
- }
- }
- } catch (IOException e) {
- SolrException.log(log, "Error deleting files from Directory", e);
- isSuccess = false;
- }
- return isSuccess;
- }
-
- /**
- * If your implementation can count on delete-on-last-close semantics
- * or throws an exception when trying to remove a file in use, return
- * false (eg NFS). Otherwise, return true. Defaults to returning false.
- *
- * @return true if factory impl requires that Searcher's explicitly
- * reserve commit points.
- */
- public boolean searchersReserveCommitPoints() {
- return false;
- }
-
- /**
- * Get the data home folder. If solr.data.home is set, that is used, else base on instanceDir
- * @param cd core descriptor instance
- * @return a String with absolute path to data direcotry
- */
- public String getDataHome(CoreDescriptor cd) throws IOException {
- String dataDir;
- if (dataHomePath != null) {
- String instanceDirLastPath = cd.getInstanceDir().getName(cd.getInstanceDir().getNameCount()-1).toString();
- dataDir = Paths.get(coreContainer.getSolrHome()).resolve(dataHomePath)
- .resolve(instanceDirLastPath).resolve(cd.getDataDir()).toAbsolutePath().toString();
- } else {
- // by default, we go off the instance directory
- dataDir = cd.getInstanceDir().resolve(cd.getDataDir()).toAbsolutePath().toString();
- }
- return dataDir;
- }
-
- public void cleanupOldIndexDirectories(final String dataDirPath, final String currentIndexDirPath, boolean afterCoreReload) {
- File dataDir = new File(dataDirPath);
- if (!dataDir.isDirectory()) {
- log.debug("{} does not point to a valid data directory; skipping clean-up of old index directories.", dataDirPath);
- return;
- }
-
- final File currentIndexDir = new File(currentIndexDirPath);
- File[] oldIndexDirs = dataDir.listFiles(new FileFilter() {
- @Override
- public boolean accept(File file) {
- String fileName = file.getName();
- return file.isDirectory() &&
- !file.equals(currentIndexDir) &&
- (fileName.equals("index") || fileName.matches(INDEX_W_TIMESTAMP_REGEX));
- }
- });
-
- if (oldIndexDirs == null || oldIndexDirs.length == 0)
- return; // nothing to do (no log message needed)
-
- List<File> dirsList = Arrays.asList(oldIndexDirs);
- Collections.sort(dirsList, Collections.reverseOrder());
-
- int i = 0;
- if (afterCoreReload) {
- log.info("Will not remove most recent old directory after reload {}", oldIndexDirs[0]);
- i = 1;
- }
- log.info("Found {} old index directories to clean-up under {} afterReload={}", oldIndexDirs.length - i, dataDirPath, afterCoreReload);
- for (; i < dirsList.size(); i++) {
- File dir = dirsList.get(i);
- String dirToRmPath = dir.getAbsolutePath();
- try {
- if (deleteOldIndexDirectory(dirToRmPath)) {
- log.info("Deleted old index directory: {}", dirToRmPath);
- } else {
- log.warn("Delete old index directory {} failed.", dirToRmPath);
- }
- } catch (IOException ioExc) {
- log.error("Failed to delete old directory {} due to: {}", dir.getAbsolutePath(), ioExc.toString());
- }
- }
- }
-
- // Extension point to allow sub-classes to infuse additional code when deleting old index directories
- protected boolean deleteOldIndexDirectory(String oldDirPath) throws IOException {
- File dirToRm = new File(oldDirPath);
- FileUtils.deleteDirectory(dirToRm);
- return !dirToRm.isDirectory();
- }
-
- public void initCoreContainer(CoreContainer cc) {
- this.coreContainer = cc;
- if (cc != null && cc.getConfig() != null) {
- this.dataHomePath = cc.getConfig().getSolrDataHome();
- }
- }
-
- // special hack to work with FilterDirectory
- protected Directory getBaseDir(Directory dir) {
- Directory baseDir = dir;
- while (baseDir instanceof FilterDirectory) {
- baseDir = ((FilterDirectory)baseDir).getDelegate();
- }
-
- return baseDir;
- }
-
- /**
- * Create a new DirectoryFactory instance from the given SolrConfig and tied to the specified core container.
- */
- static DirectoryFactory loadDirectoryFactory(SolrConfig config, CoreContainer cc, String registryName) {
- final PluginInfo info = config.getPluginInfo(DirectoryFactory.class.getName());
- final DirectoryFactory dirFactory;
- if (info != null) {
- log.debug(info.className);
- dirFactory = config.getResourceLoader().newInstance(info.className, DirectoryFactory.class);
- // allow DirectoryFactory instances to access the CoreContainer
- dirFactory.initCoreContainer(cc);
- dirFactory.init(info.initArgs);
- } else {
- log.debug("solr.NRTCachingDirectoryFactory");
- dirFactory = new NRTCachingDirectoryFactory();
- dirFactory.initCoreContainer(cc);
- }
- return dirFactory;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
deleted file mode 100644
index c7708ea..0000000
--- a/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.core;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-
-import org.apache.lucene.store.Directory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Directory provider for implementations that do not persist over reboots.
- *
- */
-public abstract class EphemeralDirectoryFactory extends CachingDirectoryFactory {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Override
- public boolean exists(String path) throws IOException {
- String fullPath = normalize(path);
- synchronized (this) {
- final CacheValue cacheValue = byPathCache.get(fullPath);
- if (null == cacheValue) {
- return false;
- }
- final Directory directory = cacheValue.directory;
- if (null == directory) {
- return false;
- }
- if (0 < directory.listAll().length) {
- return true;
- }
- return false;
- }
- }
-
- public boolean isPersistent() {
- return false;
- }
-
- @Override
- public boolean isAbsolute(String path) {
- return true;
- }
-
-
- @Override
- public void remove(Directory dir) throws IOException {
- // ram dir does not persist its dir anywhere
- }
-
- @Override
- public void remove(String path) throws IOException {
- // ram dir does not persist its dir anywhere
- }
-
- public void cleanupOldIndexDirectories(final String dataDirPath, final String currentIndexDirPath, boolean reload) {
- // currently a no-op
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
deleted file mode 100644
index 13e1de1..0000000
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ /dev/null
@@ -1,610 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.core;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URLEncoder;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.LockFactory;
-import org.apache.lucene.store.NRTCachingDirectory;
-import org.apache.lucene.store.NoLockFactory;
-import org.apache.lucene.store.SingleInstanceLockFactory;
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.metrics.SolrMetricProducer;
-import org.apache.solr.store.blockcache.BlockCache;
-import org.apache.solr.store.blockcache.BlockDirectory;
-import org.apache.solr.store.blockcache.BlockDirectoryCache;
-import org.apache.solr.store.blockcache.BufferStore;
-import org.apache.solr.store.blockcache.Cache;
-import org.apache.solr.store.blockcache.Metrics;
-import org.apache.solr.store.hdfs.HdfsDirectory;
-import org.apache.solr.store.hdfs.HdfsLocalityReporter;
-import org.apache.solr.store.hdfs.HdfsLockFactory;
-import org.apache.solr.util.HdfsUtil;
-import org.apache.solr.util.plugin.SolrCoreAware;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-
-public class HdfsDirectoryFactory extends CachingDirectoryFactory implements SolrCoreAware, SolrMetricProducer {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final String BLOCKCACHE_SLAB_COUNT = "solr.hdfs.blockcache.slab.count";
- public static final String BLOCKCACHE_DIRECT_MEMORY_ALLOCATION = "solr.hdfs.blockcache.direct.memory.allocation";
- public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
- public static final String BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
- public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
- public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled"; // currently buggy and disabled
-
- public static final String NRTCACHINGDIRECTORY_ENABLE = "solr.hdfs.nrtcachingdirectory.enable";
- public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB = "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
- public static final String NRTCACHINGDIRECTORY_MAXCACHEMB = "solr.hdfs.nrtcachingdirectory.maxcachedmb";
- public static final String NUMBEROFBLOCKSPERBANK = "solr.hdfs.blockcache.blocksperbank";
-
- public static final String LOCALITYMETRICS_ENABLED = "solr.hdfs.locality.metrics.enabled";
-
- public static final String KERBEROS_ENABLED = "solr.hdfs.security.kerberos.enabled";
- public static final String KERBEROS_KEYTAB = "solr.hdfs.security.kerberos.keytabfile";
- public static final String KERBEROS_PRINCIPAL = "solr.hdfs.security.kerberos.principal";
-
- public static final String HDFS_HOME = "solr.hdfs.home";
-
- public static final String CONFIG_DIRECTORY = "solr.hdfs.confdir";
-
- public static final String CACHE_MERGES = "solr.hdfs.blockcache.cachemerges";
- public static final String CACHE_READONCE = "solr.hdfs.blockcache.cachereadonce";
-
- private SolrParams params;
-
- private String hdfsDataDir;
-
- private String confDir;
-
- private boolean cacheReadOnce;
-
- private boolean cacheMerges;
-
- private static BlockCache globalBlockCache;
-
- public static Metrics metrics;
- private static Boolean kerberosInit;
-
- // we use this cache for FileSystem instances when we don't have access to a long lived instance
- private com.google.common.cache.Cache<String,FileSystem> tmpFsCache = CacheBuilder.newBuilder()
- .concurrencyLevel(10)
- .maximumSize(1000)
- .expireAfterAccess(5, TimeUnit.MINUTES).removalListener(new RemovalListener<String,FileSystem>() {
- @Override
- public void onRemoval(RemovalNotification<String,FileSystem> rn) {
- IOUtils.closeQuietly(rn.getValue());
- }
- })
- .build();
-
- private final static class MetricsHolder {
- // [JCIP SE, Goetz, 16.6] Lazy initialization
- // Won't load until MetricsHolder is referenced
- public static final Metrics metrics = new Metrics();
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- Collection<FileSystem> values = tmpFsCache.asMap().values();
- for (FileSystem fs : values) {
- IOUtils.closeQuietly(fs);
- }
- tmpFsCache.invalidateAll();
- tmpFsCache.cleanUp();
- }
-
- private final static class LocalityHolder {
- public static final HdfsLocalityReporter reporter = new HdfsLocalityReporter();
- }
-
- @Override
- public void init(NamedList args) {
- super.init(args);
- params = args.toSolrParams();
- this.hdfsDataDir = getConfig(HDFS_HOME, null);
- if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
- this.hdfsDataDir = null;
- } else {
- log.info(HDFS_HOME + "=" + this.hdfsDataDir);
- }
- cacheMerges = getConfig(CACHE_MERGES, false);
- cacheReadOnce = getConfig(CACHE_READONCE, false);
- boolean kerberosEnabled = getConfig(KERBEROS_ENABLED, false);
- log.info("Solr Kerberos Authentication "
- + (kerberosEnabled ? "enabled" : "disabled"));
- if (kerberosEnabled) {
- initKerberos();
- }
- }
-
- @Override
- protected LockFactory createLockFactory(String rawLockType) throws IOException {
- if (null == rawLockType) {
- rawLockType = DirectoryFactory.LOCK_TYPE_HDFS;
- log.warn("No lockType configured, assuming '"+rawLockType+"'.");
- }
- final String lockType = rawLockType.toLowerCase(Locale.ROOT).trim();
- switch (lockType) {
- case DirectoryFactory.LOCK_TYPE_HDFS:
- return HdfsLockFactory.INSTANCE;
- case DirectoryFactory.LOCK_TYPE_SINGLE:
- return new SingleInstanceLockFactory();
- case DirectoryFactory.LOCK_TYPE_NONE:
- return NoLockFactory.INSTANCE;
- default:
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Unrecognized lockType: " + rawLockType);
- }
- }
-
- @Override
- @SuppressWarnings("resource")
- protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
- assert params != null : "init must be called before create";
- log.info("creating directory factory for path {}", path);
- Configuration conf = getConf();
-
- if (metrics == null) {
- metrics = MetricsHolder.metrics;
- }
-
- boolean blockCacheEnabled = getConfig(BLOCKCACHE_ENABLED, true);
- boolean blockCacheGlobal = getConfig(BLOCKCACHE_GLOBAL, true);
- boolean blockCacheReadEnabled = getConfig(BLOCKCACHE_READ_ENABLED, true);
-
- final HdfsDirectory hdfsDir;
-
- final Directory dir;
- if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
- int numberOfBlocksPerBank = getConfig(NUMBEROFBLOCKSPERBANK, 16384);
-
- int blockSize = BlockDirectory.BLOCK_SIZE;
-
- int bankCount = getConfig(BLOCKCACHE_SLAB_COUNT, 1);
-
- boolean directAllocation = getConfig(BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
-
- int slabSize = numberOfBlocksPerBank * blockSize;
- log.info(
- "Number of slabs of block cache [{}] with direct memory allocation set to [{}]",
- bankCount, directAllocation);
- log.info(
- "Block cache target memory usage, slab size of [{}] will allocate [{}] slabs and use ~[{}] bytes",
- new Object[] {slabSize, bankCount,
- ((long) bankCount * (long) slabSize)});
-
- int bsBufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", blockSize);
- int bsBufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 0); // this is actually total size
-
- BlockCache blockCache = getBlockDirectoryCache(numberOfBlocksPerBank,
- blockSize, bankCount, directAllocation, slabSize,
- bsBufferSize, bsBufferCount, blockCacheGlobal);
-
- Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
- int readBufferSize = params.getInt("solr.hdfs.blockcache.read.buffersize", blockSize);
- hdfsDir = new HdfsDirectory(new Path(path), lockFactory, conf, readBufferSize);
- dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
- } else {
- hdfsDir = new HdfsDirectory(new Path(path), conf);
- dir = hdfsDir;
- }
- if (params.getBool(LOCALITYMETRICS_ENABLED, false)) {
- LocalityHolder.reporter.registerDirectory(hdfsDir);
- }
-
- boolean nrtCachingDirectory = getConfig(NRTCACHINGDIRECTORY_ENABLE, true);
- if (nrtCachingDirectory) {
- double nrtCacheMaxMergeSizeMB = getConfig(NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
- double nrtCacheMaxCacheMB = getConfig(NRTCACHINGDIRECTORY_MAXCACHEMB, 192);
-
- return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB, nrtCacheMaxCacheMB);
- }
- return dir;
- }
-
- boolean getConfig(String name, boolean defaultValue) {
- Boolean value = params.getBool(name);
- if (value == null) {
- String sysValue = System.getProperty(name);
- if (sysValue != null) {
- value = Boolean.valueOf(sysValue);
- }
- }
- return value == null ? defaultValue : value;
- }
-
- int getConfig(String name, int defaultValue) {
- Integer value = params.getInt(name);
- if (value == null) {
- String sysValue = System.getProperty(name);
- if (sysValue != null) {
- value = Integer.parseInt(sysValue);
- }
- }
- return value == null ? defaultValue : value;
- }
-
- String getConfig(String name, String defaultValue) {
- String value = params.get(name);
- if (value == null) {
- value = System.getProperty(name);
- }
- return value == null ? defaultValue : value;
- }
-
- private BlockCache getBlockDirectoryCache(int numberOfBlocksPerBank, int blockSize, int bankCount,
- boolean directAllocation, int slabSize, int bufferSize, int bufferCount, boolean staticBlockCache) {
- if (!staticBlockCache) {
- log.info("Creating new single instance HDFS BlockCache");
- return createBlockCache(numberOfBlocksPerBank, blockSize, bankCount, directAllocation, slabSize, bufferSize, bufferCount);
- }
- synchronized (HdfsDirectoryFactory.class) {
-
- if (globalBlockCache == null) {
- log.info("Creating new global HDFS BlockCache");
- globalBlockCache = createBlockCache(numberOfBlocksPerBank, blockSize, bankCount,
- directAllocation, slabSize, bufferSize, bufferCount);
- }
- }
- return globalBlockCache;
- }
-
- private BlockCache createBlockCache(int numberOfBlocksPerBank, int blockSize,
- int bankCount, boolean directAllocation, int slabSize, int bufferSize,
- int bufferCount) {
- BufferStore.initNewBuffer(bufferSize, bufferCount, metrics);
- long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
- * (long) blockSize;
-
- BlockCache blockCache;
- try {
- blockCache = new BlockCache(metrics, directAllocation, totalMemory, slabSize, blockSize);
- } catch (OutOfMemoryError e) {
- throw new RuntimeException(
- "The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args)"
- + " or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap,"
- + " your java heap size might not be large enough."
- + " Failed allocating ~" + totalMemory / 1000000.0 + " MB.",
- e);
- }
- return blockCache;
- }
-
- @Override
- public boolean exists(String path) {
- final Path hdfsDirPath = new Path(path);
- FileSystem fileSystem = getCachedFileSystem(path);
-
- try {
- return fileSystem.exists(hdfsDirPath);
- } catch (IOException e) {
- log.error("Error checking if hdfs path exists", e);
- throw new RuntimeException("Error checking if hdfs path exists", e);
- }
- }
-
- public Configuration getConf() {
- Configuration conf = new Configuration();
- confDir = getConfig(CONFIG_DIRECTORY, null);
- HdfsUtil.addHdfsResources(conf, confDir);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
- return conf;
- }
-
- protected synchronized void removeDirectory(final CacheValue cacheValue)
- throws IOException {
- FileSystem fileSystem = getCachedFileSystem(cacheValue.path);
-
- try {
- boolean success = fileSystem.delete(new Path(cacheValue.path), true);
- if (!success) {
- throw new RuntimeException("Could not remove directory");
- }
- } catch (Exception e) {
- log.error("Could not remove directory", e);
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "Could not remove directory", e);
- }
- }
-
- @Override
- public boolean isAbsolute(String path) {
- return path.startsWith("hdfs:/");
- }
-
- @Override
- public boolean isPersistent() {
- return true;
- }
-
- @Override
- public boolean isSharedStorage() {
- return true;
- }
-
- @Override
- public boolean searchersReserveCommitPoints() {
- return true;
- }
-
- @Override
- public String getDataHome(CoreDescriptor cd) throws IOException {
- if (hdfsDataDir == null) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "You must set the "
- + this.getClass().getSimpleName() + " param " + HDFS_HOME
- + " for relative dataDir paths to work");
- }
-
- // by default, we go off the instance directory
- String path;
- if (cd.getCloudDescriptor() != null) {
- path = URLEncoder.encode(cd.getCloudDescriptor().getCollectionName(),
- "UTF-8")
- + "/"
- + URLEncoder.encode(cd.getCloudDescriptor().getCoreNodeName(),
- "UTF-8");
- } else {
- path = cd.getName();
- }
-
- return normalize(SolrResourceLoader.normalizeDir(ZkController
- .trimLeadingAndTrailingSlashes(hdfsDataDir)
- + "/"
- + path
- + "/"
- + cd.getDataDir()));
- }
-
- /**
- * @param directory to calculate size of
- * @return size in bytes
- * @throws IOException on low level IO error
- */
- @Override
- public long size(Directory directory) throws IOException {
- String hdfsDirPath = getPath(directory);
- return size(hdfsDirPath);
- }
-
- /**
- * @param path to calculate size of
- * @return size in bytes
- * @throws IOException on low level IO error
- */
- @Override
- public long size(String path) throws IOException {
- Path hdfsDirPath = new Path(path);
- FileSystem fileSystem = getCachedFileSystem(path);
- try {
- return fileSystem.getContentSummary(hdfsDirPath).getLength();
- } catch (IOException e) {
- log.error("Error checking if hdfs path exists", e);
- throw new SolrException(ErrorCode.SERVER_ERROR, "Error checking if hdfs path exists", e);
- } finally {
- IOUtils.closeQuietly(fileSystem);
- }
- }
-
- private FileSystem getCachedFileSystem(String path) {
- try {
- // no need to close the fs, the cache will do it
- return tmpFsCache.get(path, () -> FileSystem.get(new Path(path).toUri(), getConf()));
- } catch (ExecutionException e) {
- throw new RuntimeException(e);
- }
- }
-
- public String getConfDir() {
- return confDir;
- }
-
- private void initKerberos() {
- String keytabFile = getConfig(KERBEROS_KEYTAB, "").trim();
- if (keytabFile.length() == 0) {
- throw new IllegalArgumentException(KERBEROS_KEYTAB + " required because "
- + KERBEROS_ENABLED + " set to true");
- }
- String principal = getConfig(KERBEROS_PRINCIPAL, "");
- if (principal.length() == 0) {
- throw new IllegalArgumentException(KERBEROS_PRINCIPAL
- + " required because " + KERBEROS_ENABLED + " set to true");
- }
- synchronized (HdfsDirectoryFactory.class) {
- if (kerberosInit == null) {
- kerberosInit = Boolean.TRUE;
- final Configuration conf = getConf();
- final String authVal = conf.get(HADOOP_SECURITY_AUTHENTICATION);
- final String kerberos = "kerberos";
- if (authVal != null && !authVal.equals(kerberos)) {
- throw new IllegalArgumentException(HADOOP_SECURITY_AUTHENTICATION
- + " set to: " + authVal + ", not kerberos, but attempting to "
- + " connect to HDFS via kerberos");
- }
- // let's avoid modifying the supplied configuration, just to be conservative
- final Configuration ugiConf = new Configuration(getConf());
- ugiConf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos);
- UserGroupInformation.setConfiguration(ugiConf);
- log.info(
- "Attempting to acquire kerberos ticket with keytab: {}, principal: {} ",
- keytabFile, principal);
- try {
- UserGroupInformation.loginUserFromKeytab(principal, keytabFile);
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- log.info("Got Kerberos ticket");
- }
- }
- }
-
- @Override
- public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) {
- MetricsHolder.metrics.initializeMetrics(manager, registry, tag, scope);
- LocalityHolder.reporter.initializeMetrics(manager, registry, tag, scope);
- }
-
- @Override
- public void inform(SolrCore core) {
- setHost(core.getCoreContainer().getHostName());
- }
-
- @VisibleForTesting
- void setHost(String hostname) {
- LocalityHolder.reporter.setHost(hostname);
- }
-
- @Override
- public void cleanupOldIndexDirectories(final String dataDir, final String currentIndexDir, boolean afterReload) {
-
- // Get the FileSystem object
- final Path dataDirPath = new Path(dataDir);
- FileSystem fileSystem = getCachedFileSystem(dataDir);
-
- boolean pathExists = false;
- try {
- pathExists = fileSystem.exists(dataDirPath);
- } catch (IOException e) {
- log.error("Error checking if hdfs path "+dataDir+" exists", e);
- }
- if (!pathExists) {
- log.warn("{} does not point to a valid data directory; skipping clean-up of old index directories.", dataDir);
- return;
- }
-
- final Path currentIndexDirPath = new Path(currentIndexDir); // make sure we don't delete the current
- final FileSystem fs = fileSystem;
- FileStatus[] oldIndexDirs = null;
- try {
- oldIndexDirs = fileSystem.listStatus(dataDirPath, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- boolean accept = false;
- String pathName = path.getName();
- try {
- accept = fs.isDirectory(path) && !path.equals(currentIndexDirPath) &&
- (pathName.equals("index") || pathName.matches(INDEX_W_TIMESTAMP_REGEX));
- } catch (IOException e) {
- log.error("Error checking if path {} is an old index directory, caused by: {}", path, e);
- }
- return accept;
- }
- });
- } catch (FileNotFoundException fnfe) {
- // already deleted - ignore
- log.debug("Old index directory already deleted - skipping...", fnfe);
- } catch (IOException ioExc) {
- log.error("Error checking for old index directories to clean-up.", ioExc);
- }
-
- if (oldIndexDirs == null || oldIndexDirs.length == 0)
- return; // nothing to clean-up
-
- List<Path> oldIndexPaths = new ArrayList<>(oldIndexDirs.length);
- for (FileStatus ofs : oldIndexDirs) {
- oldIndexPaths.add(ofs.getPath());
- }
-
- Collections.sort(oldIndexPaths, Collections.reverseOrder());
-
- Set<String> livePaths = getLivePaths();
-
- int i = 0;
- if (afterReload) {
- log.info("Will not remove most recent old directory on reload {}", oldIndexDirs[0]);
- i = 1;
- }
- log.info("Found {} old index directories to clean-up under {} afterReload={}", oldIndexDirs.length - i, dataDirPath, afterReload);
- for (; i < oldIndexPaths.size(); i++) {
- Path oldDirPath = oldIndexPaths.get(i);
- if (livePaths.contains(oldDirPath.toString())) {
- log.warn("Cannot delete directory {} because it is still being referenced in the cache.", oldDirPath);
- } else {
- try {
- if (fileSystem.delete(oldDirPath, true)) {
- log.info("Deleted old index directory {}", oldDirPath);
- } else {
- log.warn("Failed to delete old index directory {}", oldDirPath);
- }
- } catch (IOException e) {
- log.error("Failed to delete old index directory {} due to: {}", oldDirPath, e);
- }
- }
- }
- }
-
- // perform an atomic rename if possible
- public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException {
- String hdfsDirPath = getPath(dir);
- FileContext fileContext = FileContext.getFileContext(getConf());
- fileContext.rename(new Path(hdfsDirPath + "/" + fileName), new Path(hdfsDirPath + "/" + toName), Options.Rename.OVERWRITE);
- }
-
- @Override
- public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
-
- Directory baseFromDir = getBaseDir(fromDir);
- Directory baseToDir = getBaseDir(toDir);
-
- if (baseFromDir instanceof HdfsDirectory && baseToDir instanceof HdfsDirectory) {
- Path dir1 = ((HdfsDirectory) baseFromDir).getHdfsDirPath();
- Path dir2 = ((HdfsDirectory) baseToDir).getHdfsDirPath();
- Path file1 = new Path(dir1, fileName);
- Path file2 = new Path(dir2, fileName);
- FileContext fileContext = FileContext.getFileContext(getConf());
- fileContext.rename(file1, file2);
- return;
- }
-
- super.move(fromDir, toDir, fileName, ioContext);
- }
-}