You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ds...@apache.org on 2016/03/19 02:29:36 UTC
[1/3] lucene-solr:solr-5750: SOLR-5750: SolrCloud Backup/Restore DR
(Varun patch March 4th)
Repository: lucene-solr
Updated Branches:
refs/heads/solr-5750 [created] fd9c4d59e
SOLR-5750: SolrCloud Backup/Restore DR (Varun patch March 4th)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1a3e7521
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1a3e7521
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1a3e7521
Branch: refs/heads/solr-5750
Commit: 1a3e7521c086141b3be7dbb8ff61f168ba897e61
Parents: b9cc153
Author: David Smiley <ds...@apache.org>
Authored: Fri Mar 18 17:15:28 2016 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Fri Mar 18 17:15:28 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/solr/cloud/Overseer.java | 6 +-
.../cloud/OverseerCollectionMessageHandler.java | 281 +++++++++++++++++++
.../org/apache/solr/handler/RestoreCore.java | 2 +-
.../org/apache/solr/handler/SnapShooter.java | 45 ++-
.../solr/handler/admin/CollectionsHandler.java | 48 ++++
.../solr/handler/admin/CoreAdminOperation.java | 73 ++++-
.../solr/cloud/TestCloudBackupRestore.java | 166 +++++++++++
.../solrj/request/CollectionAdminRequest.java | 76 +++++
.../apache/solr/common/cloud/ZkStateReader.java | 3 +-
.../solr/common/params/CollectionParams.java | 4 +-
.../solr/common/params/CoreAdminParams.java | 5 +-
11 files changed, 700 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 0e5bded..ec701c3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -366,9 +366,13 @@ public class Overseer implements Closeable {
break;
case MODIFYCOLLECTION:
CollectionsHandler.verifyRuleParams(zkController.getCoreContainer() ,message.getProperties());
- return Collections.singletonList(new CollectionMutator(reader).modifyCollection(clusterState,message));
+ return Collections.singletonList(new CollectionMutator(reader).modifyCollection(clusterState, message));
case MIGRATESTATEFORMAT:
return Collections.singletonList(new ClusterStateMutator(reader).migrateStateFormat(clusterState, message));
+ case RESTORE:
+ break;
+ case BACKUP:
+ break;
default:
throw new RuntimeException("unknown operation:" + operation
+ " contents:" + message.getProperties());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 4d802e9..228556a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -16,12 +16,22 @@
*/
package org.apache.solr.cloud;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -29,6 +39,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -71,10 +82,12 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.handler.SnapShooter;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
@@ -273,6 +286,12 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
case MIGRATESTATEFORMAT:
migrateStateFormat(message, results);
break;
+ case BACKUP:
+ processBackupAction(message, results);
+ break;
+ case RESTORE:
+ processRestoreAction(message, results);
+ break;
default:
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ operation);
@@ -309,6 +328,268 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
collectionCmd(message, params, results, Replica.State.ACTIVE, asyncId, requestMap);
}
+ //TODO move this out into it's own class.
+ private void processRestoreAction(ZkNodeProps message, NamedList results) throws IOException, KeeperException, InterruptedException {
+ String restoreCollectionName = message.getStr(COLLECTION_PROP);
+ String name = message.getStr(NAME);
+ String location = message.getStr("location");
+ ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
+ final String asyncId = message.getStr(ASYNC);
+ Map<String, String> requestMap = null;
+ if (asyncId != null) {
+ requestMap = new HashMap<>();
+ }
+ Path backupPath = Paths.get(location).resolve(name).toAbsolutePath();
+
+ if (!Files.exists(backupPath)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Backup directory does not exist: " + backupPath.toString());
+ }
+
+ Path zkBackup = backupPath.resolve("zk_backup");
+ if (!Files.exists(zkBackup)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Backup zk directory does not exist: " + backupPath.toString());
+ }
+
+ Path propertiesPath = zkBackup.resolve("backup.properties");
+ if (!Files.exists(propertiesPath)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "backup.properties file does not exist: " + backupPath.toString());
+ }
+
+ FileInputStream in = null;
+ Properties properties = null;
+ try {
+ in = new FileInputStream(propertiesPath.toAbsolutePath().toString());
+ properties = new Properties();
+ properties.load(new InputStreamReader(in, StandardCharsets.UTF_8));
+ } catch (IOException e) {
+ String errorMsg = String.format(Locale.ROOT, "Could not load properties from %s: %s:",
+ propertiesPath.toAbsolutePath().toString(), e.toString());
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, errorMsg);
+ } finally {
+ IOUtils.closeQuietly(in);
+ }
+
+ String backupCollection = (String) properties.get("collectionName");
+ Path collectionStatePath = zkBackup.resolve("collection_state_backup.json");
+ byte[] data = Files.readAllBytes(collectionStatePath);
+ Map<String, Object> collectionProps = (Map<String, Object>) ((Map<String, Object>) Utils.fromJSON(data)).get(backupCollection);
+
+ //Download the configs
+ String configName = (String) properties.get("collection.configName");
+
+ //Use a name such as restore.<restore_name>.<original_config_name>
+ // in ZK for the configs
+ String restoreConfigName = "restore." + configName;
+ zkStateReader.getConfigManager().uploadConfigDir(zkBackup.resolve("configs").resolve(configName), restoreConfigName);
+
+ log.debug("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, name,
+ backupPath.toString());
+
+ //Create core-less collection
+ Map<String, Object> propMap = new HashMap<>();
+ propMap.put(NAME, restoreCollectionName);
+ propMap.put(CREATE_NODE_SET, CREATE_NODE_SET_EMPTY); //no cores
+ propMap.put("collection.configName", restoreConfigName);
+ // add async param
+ if (asyncId != null) {
+ propMap.put(ASYNC, asyncId);
+ }
+ String router = (String) ((Map)collectionProps.get("router")).get("name");
+ propMap.put("router.name", router);
+ Map slices = (Map) collectionProps.get(SHARDS_PROP);
+ if (slices != null) { //Implicit routers may not have a shards defined
+ propMap.put(NUM_SLICES, slices.size());
+ }
+ if (ImplicitDocRouter.NAME.equals(router)) {
+ Iterator keys = ((Map) collectionProps.get(SHARDS_PROP)).keySet().iterator();
+ StringBuilder shardsBuilder = new StringBuilder();
+ while (keys.hasNext()) {
+ String shard = (String) keys.next();
+ shardsBuilder.append(shard);
+ shardsBuilder.append(",");
+ }
+ String shards = shardsBuilder.deleteCharAt(shardsBuilder.length()-1).toString();
+ propMap.put(SHARDS_PROP, shards);
+ }
+ propMap.put(MAX_SHARDS_PER_NODE, Integer.parseInt((String) collectionProps.get(MAX_SHARDS_PER_NODE)));
+ propMap.put(ZkStateReader.AUTO_ADD_REPLICAS, Boolean.parseBoolean((String) collectionProps.get(ZkStateReader.AUTO_ADD_REPLICAS)));
+ propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString());
+
+ createCollection(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList());
+
+ //No need to wait. CreateCollection takes care of it by calling waitToSeeReplicasInState()
+ DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
+ if (restoreCollection == null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not create restore collection");
+ }
+
+ //Mark all shards in CONSTRUCTION STATE while we restore the data
+ DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient());
+ propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
+ for (Slice shard : restoreCollection.getSlices()) {
+ propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString());
+ }
+ propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
+ inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
+
+ ClusterState clusterState = zkStateReader.getClusterState();
+ //Create one replica per shard and copy backed up data to it
+ for (Slice slice: restoreCollection.getSlices()) {
+ log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
+ propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower());
+ propMap.put(COLLECTION_PROP, restoreCollectionName);
+ propMap.put(SHARD_ID_PROP, slice.getName());
+ // add async param
+ if (asyncId != null) {
+ propMap.put(ASYNC, asyncId);
+ }
+ addReplica(clusterState, new ZkNodeProps(propMap), new NamedList());
+ }
+
+ //refresh the location copy of collection state
+ restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
+
+ //Copy data from backed up index to each replica
+ for (Slice slice: restoreCollection.getSlices()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(NAME, "snapshot." + slice.getName());
+ params.set("location", backupPath.toString());
+ params.set(CoreAdminParams.ACTION, CoreAdminAction.RESTORECORE.toString());
+ sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap);
+ }
+ processResponses(new NamedList(), shardHandler, true, "Could not restore core", asyncId, requestMap);
+
+ //Mark all shards in ACTIVE STATE
+ propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
+ propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
+ for (Slice shard : restoreCollection.getSlices()) {
+ propMap.put(shard.getName(), Slice.State.ACTIVE.toString());
+ }
+ inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
+
+ //refresh the location copy of collection state
+ restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
+
+ //Update the replicationFactor to be 1 as that's what it is currently. Otherwise addreplica assigns wrong core names
+ propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower());
+ propMap.put(COLLECTION_PROP, restoreCollectionName);
+ propMap.put(REPLICATION_FACTOR, 1);
+ inQueue.offer(Utils.toJSON(message));
+
+ //Add the remaining replicas for each shard
+ int numReplicas = Integer.parseInt((String) collectionProps.get(REPLICATION_FACTOR));
+ for (Slice slice: restoreCollection.getSlices()) {
+ for(int i=1; i<numReplicas; i++) {
+ log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
+ propMap = new HashMap<>();
+ propMap.put(COLLECTION_PROP, restoreCollectionName);
+ propMap.put(SHARD_ID_PROP, slice.getName());
+ // add async param
+ if (asyncId != null) {
+ propMap.put(ASYNC, asyncId);
+ }
+ addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results);
+ }
+ }
+
+ if (numReplicas > 1) {
+ //Update the replicationFactor property in cluster state for this collection
+ log.info("Modifying replication factor to the expected value of={}", numReplicas);
+ propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower());
+ propMap.put(COLLECTION_PROP, restoreCollectionName);
+ propMap.put(REPLICATION_FACTOR, numReplicas);
+ inQueue.offer(Utils.toJSON(message));
+ }
+
+ }
+
+ private void processBackupAction(ZkNodeProps message, NamedList results) throws IOException, KeeperException, InterruptedException {
+ String collectionName = message.getStr(COLLECTION_PROP);
+ String name = message.getStr(NAME);
+ String location = message.getStr("location");
+ ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
+ String asyncId = message.getStr(ASYNC);
+ Map<String, String> requestMap = null;
+ if (asyncId != null) {
+ requestMap = new HashMap<>();
+ }
+
+ Path backupPath = Paths.get(location).resolve(name).toAbsolutePath();
+
+ //Validating if the directory already exists.
+ if (Files.exists(backupPath)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Backup directory already exists: " + backupPath.toString());
+ }
+
+ log.debug("Starting backup of collection={} with backup_name={} at location={}", collectionName, name,
+ backupPath.toString());
+
+ for (Slice slice : zkStateReader.getClusterState().getActiveSlices(collectionName)) {
+ Replica replica = slice.getLeader();
+
+ String coreName = replica.getStr(CORE_NAME_PROP);
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(NAME, slice.getName());
+ params.set("location", backupPath.toString());
+ params.set(CORE_NAME_PROP, coreName);
+ params.set(CoreAdminParams.ACTION, CoreAdminAction.BACKUPCORE.toString());
+
+ sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
+ log.debug("Sent backup request to core={} for backup_name={}", coreName, name);
+ }
+ log.debug("Sent backup requests to all shard leaders for backup_name={}", name);
+
+ processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap);
+
+ log.debug("Starting to backup ZK data for backup_name={}", name);
+
+ //Download the configs
+ String configName = zkStateReader.readConfigName(collectionName);
+ Path zkBackup = backupPath.resolve("zk_backup");
+ zkStateReader.getConfigManager().downloadConfigDir(configName, zkBackup.resolve("configs").resolve(configName));
+
+ //Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json
+ //Since we don't want to distinguish we extract the state and back it up as a separate json
+ Path collectionStatePath = zkBackup.resolve("collection_state_backup.json");
+ DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
+ byte[] bytes = Utils.toJSON(Collections.singletonMap(collectionName, collection));
+ Files.write(collectionStatePath, bytes);
+
+ Path propertiesPath = zkBackup.resolve("backup.properties");
+ Properties properties = new Properties();
+ properties.put("collectionName", collectionName);
+ properties.put("snapshotName", name);
+ properties.put("collection.configName", configName);
+ properties.put("stateFormat", Integer.toString(collection.getStateFormat()));
+ properties.put("startTime", new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()));
+ //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same.
+ //if they are not the same then we can throw and error or have a 'overwriteConfig' flag
+ //TODO save numDocs for the shardLeader. We can use it to sanity check the restore.
+ OutputStreamWriter os = new OutputStreamWriter(new FileOutputStream(propertiesPath.toAbsolutePath().toString()), StandardCharsets.UTF_8);
+
+ try {
+ properties.store(os, "Snapshot properties file");
+ } catch (IOException e) {
+ String errorMsg = String.format(Locale.ROOT, "Could not write properties to %s: %s:",
+ propertiesPath.toAbsolutePath().toString(), e.toString());
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, errorMsg);
+ } finally {
+ IOUtils.closeQuietly(os);
+ }
+
+ log.debug("Completed backing up ZK data for backup={}", name);
+ }
+
@SuppressWarnings("unchecked")
private void processRebalanceLeaders(ZkNodeProps message) throws KeeperException, InterruptedException {
checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, CORE_NAME_PROP, ELECTION_NODE_PROP,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
index a6c1da9..a8ee719 100644
--- a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
+++ b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
@@ -52,7 +52,7 @@ public class RestoreCore implements Callable<Boolean> {
return doRestore();
}
- private boolean doRestore() throws Exception {
+ public boolean doRestore() throws Exception {
Path backupPath = Paths.get(backupLocation).resolve(backupName);
String restoreIndexName = "restore." + backupName;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
index 9197e87..2e20b72 100644
--- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
+++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
@@ -37,6 +37,8 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,7 +76,7 @@ public class SnapShooter {
}
void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, final ReplicationHandler replicationHandler) {
- replicationHandler.core.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration());
+ solrCore.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration());
new Thread() {
@Override
@@ -112,7 +114,7 @@ public class SnapShooter {
}.start();
}
- void validateCreateSnapshot() throws IOException {
+ public void validateCreateSnapshot() throws IOException {
snapShotDir = new File(snapDir, directoryName);
if (snapShotDir.exists()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
@@ -124,6 +126,43 @@ public class SnapShooter {
}
}
+ //nocommit - copy pasted from createSnapshot. Need to reconcile tho
+ public NamedList createSnapshot() {
+ RefCounted<SolrIndexSearcher> searcher = solrCore.getSearcher();
+ IndexCommit indexCommit = null;
+
+ NamedList<Object> details = new NamedList<>();
+ try {
+ details.add("startTime", new Date().toString());
+ LOG.info("Creating backup snapshot " + (snapshotName == null ? "<not named>" : snapshotName) + " at " + snapDir);
+
+ indexCommit = searcher.get().getIndexReader().getIndexCommit();
+ Collection<String> files = indexCommit.getFileNames();
+
+ Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
+ try {
+ copyFiles(dir, files, snapShotDir);
+ } finally {
+ solrCore.getDirectoryFactory().release(dir);
+ }
+
+ details.add("fileCount", files.size());
+ details.add("status", "success");
+ details.add("snapshotCompletedAt", new Date().toString());
+ details.add("snapshotName", snapshotName);
+ LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "<not named>" : snapshotName) +
+ " at " + snapDir);
+ } catch (Exception e) {
+ IndexFetcher.delTree(snapShotDir);
+ LOG.error("Exception while creating snapshot", e);
+ details.add("snapShootException", e.getMessage());
+ } finally {
+ solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
+ searcher.decref();
+ }
+ return details;
+ }
+
void createSnapshot(final IndexCommit indexCommit, ReplicationHandler replicationHandler) {
LOG.info("Creating backup snapshot " + (snapshotName == null ? "<not named>" : snapshotName) + " at " + snapDir);
NamedList<Object> details = new NamedList<>();
@@ -149,7 +188,7 @@ public class SnapShooter {
LOG.error("Exception while creating snapshot", e);
details.add("snapShootException", e.getMessage());
} finally {
- replicationHandler.core.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
+ solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
replicationHandler.snapShootDetails = details;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index de2104f..490767e 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -791,6 +791,54 @@ public class CollectionsHandler extends RequestHandlerBase {
throws Exception {
return req.getParams().required().getAll(null, COLLECTION_PROP);
}
+ },
+ BACKUP_OP(BACKUP) {
+ @Override
+ Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h) throws Exception {
+ req.getParams().required().check(NAME, COLLECTION_PROP);
+
+ String collectionName = req.getParams().get(COLLECTION_PROP);
+ ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
+ if (!clusterState.hasCollection(collectionName)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
+ }
+
+ String location = req.getParams().get("location");
+ if (location == null) {
+ location = (String) h.coreContainer.getZkController().getZkStateReader().getClusterProps().get("location");
+ }
+ if (location == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "'location' is not specified as a query parameter or set as a cluster property");
+ }
+ Map<String, Object> params = req.getParams().getAll(null, NAME, COLLECTION_PROP);
+ params.put("location", location);
+ return params;
+ }
+ },
+ RESTORE_OP(RESTORE) {
+ @Override
+ Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h) throws Exception {
+ req.getParams().required().check(NAME, COLLECTION_PROP);
+
+ String collectionName = req.getParams().get(COLLECTION_PROP);
+ ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
+ //We always want to restore into an collection name which doesn't exist yet.
+ if (clusterState.hasCollection(collectionName)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' exists, no action taken.");
+ }
+
+ String location = req.getParams().get("location");
+ if (location == null) {
+ location = (String) h.coreContainer.getZkController().getZkStateReader().getClusterProps().get("location");
+ }
+ if (location == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "'location' is not specified as a query parameter or set as a cluster property");
+ }
+
+ Map<String, Object> params = req.getParams().getAll(null, NAME, COLLECTION_PROP);
+ params.put("location", location);
+ return params;
+ }
};
CollectionAction action;
long timeOut;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
index 8240189..b4ef7a3 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
@@ -60,6 +60,8 @@ import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.handler.RestoreCore;
+import org.apache.solr.handler.SnapShooter;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
@@ -80,6 +82,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.CommonParams.PATH;
+import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.BACKUPCORE;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.CREATE;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.FORCEPREPAREFORLEADERSHIP;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.INVOKE;
@@ -94,6 +97,7 @@ import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQU
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTRECOVERY;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSTATUS;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSYNCSHARD;
+import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.RESTORECORE;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.SPLIT;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.STATUS;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.SWAP;
@@ -852,6 +856,73 @@ enum CoreAdminOperation {
}
}
+ },
+ BACKUPCORE_OP(BACKUPCORE) {
+ @Override
+ public void call(CallInfo callInfo) throws IOException {
+ ZkController zkController = callInfo.handler.coreContainer.getZkController();
+ if (zkController == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Internal SolrCloud API");
+ }
+
+ final SolrParams params = callInfo.req.getParams();
+ String cname = params.get(CoreAdminParams.CORE);
+ if (cname == null) {
+ throw new IllegalArgumentException(CoreAdminParams.CORE + " is required");
+ }
+
+ String name = params.get(NAME);
+ if (name == null) {
+ throw new IllegalArgumentException(CoreAdminParams.NAME + " is required");
+ }
+
+ String location = params.get("location");
+ if (name == null) {
+ throw new IllegalArgumentException("location is required");
+ }
+
+ try (SolrCore core = callInfo.handler.coreContainer.getCore(cname)) {
+ SnapShooter snapShooter = new SnapShooter(core, location, name);
+ snapShooter.validateCreateSnapshot();
+ NamedList details = snapShooter.createSnapshot();
+ if (details.get("snapShootException") != null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed to backup core=" + core.getName());
+ }
+ }
+ }
+ },
+ RESTORECORE_OP(RESTORECORE) {
+ @Override
+ public void call(CallInfo callInfo) throws Exception {
+ ZkController zkController = callInfo.handler.coreContainer.getZkController();
+ if (zkController == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only valid for SolrCloud");
+ }
+
+ final SolrParams params = callInfo.req.getParams();
+ String cname = params.get(CoreAdminParams.CORE);
+ if (cname == null) {
+ throw new IllegalArgumentException(CoreAdminParams.CORE + " is required");
+ }
+
+ String name = params.get(NAME);
+ if (name == null) {
+ throw new IllegalArgumentException(CoreAdminParams.NAME + " is required");
+ }
+
+ String location = params.get("location");
+ if (name == null) {
+ throw new IllegalArgumentException("location is required");
+ }
+
+ try (SolrCore core = callInfo.handler.coreContainer.getCore(cname)) {
+ RestoreCore restoreCore = new RestoreCore(core, location, name);
+ boolean success = restoreCore.doRestore();
+ if (!success) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName());
+ }
+ }
+ }
};
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -937,4 +1008,4 @@ enum CoreAdminOperation {
return size;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java
new file mode 100644
index 0000000..a2a01a7
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.io.File;
+
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ImplicitDocRouter;
+import org.apache.solr.common.util.NamedList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.params.ShardParams._ROUTE_;
+
+public class TestCloudBackupRestore extends SolrTestCaseJ4 {
+
+ static Logger log = LoggerFactory.getLogger(TestCloudBackupRestore.class);
+
+ private MiniSolrCloudCluster solrCluster;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ solrCluster = new MiniSolrCloudCluster(2, createTempDir(), buildJettyConfig("/solr"));
+ final File configDir = getFile("solr").toPath().resolve("collection1/conf").toFile();
+ solrCluster.uploadConfigDir(configDir, "conf1");
+ System.setProperty("solr.test.sys.prop1", "propone");
+ System.setProperty("solr.test.sys.prop2", "proptwo");
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ solrCluster.shutdown();
+ super.tearDown();
+ }
+
+ @Test
+ public void test() throws Exception {
+ String collectionName = "backuprestore";
+ String restoreCollectionName = collectionName + "_restored";
+ boolean isImplicit = random().nextBoolean();
+ CollectionAdminRequest.Create create = new CollectionAdminRequest.Create()
+ .setCollectionName(collectionName)
+ .setConfigName("conf1")
+ .setReplicationFactor(TestUtil.nextInt(random(), 1, 2))
+ .setMaxShardsPerNode(2);
+ if (isImplicit) { //implicit router
+ create.setRouterName(ImplicitDocRouter.NAME);
+ create.setShards("shard1,shard2");
+ create.setRouterField("shard_s");
+ } else {
+ create.setNumShards(2);
+ }
+
+ create.process(solrCluster.getSolrClient());
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish("backuprestore", solrCluster.getSolrClient().getZkStateReader(), false, true, 30);
+ indexDocs(collectionName);
+ testBackupAndRestore(collectionName, restoreCollectionName, isImplicit);
+ }
+
+ private void indexDocs(String collectionName) throws Exception {
+ int numDocs = TestUtil.nextInt(random(), 10, 100);
+ CloudSolrClient client = solrCluster.getSolrClient();
+ client.setDefaultCollection(collectionName);
+ for (int i=0; i<numDocs; i++) {
+ //We index the shard_s fields for whichever router gets chosen but only use it when implicit router was selected
+ if (random().nextBoolean()) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", i);
+ doc.addField("shard_s", "shard1");
+ client.add(doc);
+ } else {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", i);
+ doc.addField("shard_s", "shard2");
+ client.add(doc);
+ }
+ }
+ client.commit();
+ }
+
+ private void testBackupAndRestore(String collectionName, String restoreCollectionName, boolean isImplicit) throws Exception {
+ String backupName = "mytestbackup";
+ CloudSolrClient client = solrCluster.getSolrClient();
+ long totalDocs = client.query(collectionName, new SolrQuery("*:*")).getResults().getNumFound();
+ long shard1Docs = 0, shard2Docs = 0;
+ if (isImplicit) {
+ shard1Docs = client.query(collectionName, new SolrQuery("*:*").setParam(_ROUTE_, "shard1")).getResults().getNumFound();
+ shard2Docs = client.query(collectionName, new SolrQuery("*:*").setParam(_ROUTE_, "shard2")).getResults().getNumFound();
+ assertTrue(totalDocs == shard1Docs + shard2Docs);
+ }
+
+ String location = createTempDir().toFile().getAbsolutePath();
+
+ log.info("Triggering Backup command");
+ //Run backup command
+ CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(backupName, collectionName)
+ .setLocation(location);
+ NamedList<Object> rsp = solrCluster.getSolrClient().request(backup);
+ assertEquals(0, ((NamedList)rsp.get("responseHeader")).get("status"));
+
+ log.info("Triggering Restore command");
+
+ //Restore
+ CollectionAdminRequest.Restore restore = new CollectionAdminRequest.Restore(backupName, restoreCollectionName)
+ .setLocation(location);
+ rsp = solrCluster.getSolrClient().request(restore);
+ assertEquals(0, ((NamedList)rsp.get("responseHeader")).get("status"));
+
+ client.getZkStateReader().updateClusterState();
+ DocCollection restoreCollection = null;
+ while (restoreCollection == null) {
+ try {
+ restoreCollection = client.getZkStateReader().getClusterState().getCollection(restoreCollectionName);
+ } catch (SolrException e) {
+ Thread.sleep(100); //wait for cluster state to update
+ }
+ }
+
+ //Check the number of results are the same
+ long restoredNumDocs = client.query(restoreCollectionName, new SolrQuery("*:*")).getResults().getNumFound();
+ assertEquals(totalDocs, restoredNumDocs);
+
+ if (isImplicit) {
+ long restoredShard1Docs = client.query(restoreCollectionName, new SolrQuery("*:*").setParam(_ROUTE_, "shard1")).getResults().getNumFound();
+ long restoredShard2Docs = client.query(restoreCollectionName, new SolrQuery("*:*").setParam(_ROUTE_, "shard2")).getResults().getNumFound();
+
+ assertEquals(shard2Docs, restoredShard2Docs);
+ assertEquals(shard1Docs, restoredShard1Docs);
+ }
+
+ DocCollection backupCollection = client.getZkStateReader().getClusterState().getCollection(collectionName);
+ assertEquals(backupCollection.getReplicationFactor(), restoreCollection.getReplicationFactor());
+
+ assertEquals( "restore.conf1", solrCluster.getSolrClient().getZkStateReader().readConfigName(restoreCollectionName));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index a7d71ca..053c417 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -410,6 +410,82 @@ public abstract class CollectionAdminRequest <Q extends CollectionAdminRequest<Q
}
}
+ // BACKUP request
+ public static class Backup extends AsyncCollectionAdminRequest<Backup> {
+ protected String location;
+ protected String name;
+ protected String collection;
+
+ public String getLocation() {
+ return location;
+ }
+
+ public Backup setLocation(String location) {
+ this.location = location;
+ return this;
+ }
+
+ public Backup(String name, String collection) {
+ this.name = name;
+ this.collection = collection;
+ action = CollectionAction.BACKUP;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set(CoreAdminParams.COLLECTION, collection);
+ params.set(CoreAdminParams.NAME, name);
+ if (location != null) {
+ params.set("location", location);
+ }
+ return params;
+ }
+
+ @Override
+ protected Backup getThis() {
+ return this;
+ }
+ }
+
+ // RESTORE request
+ public static class Restore extends AsyncCollectionAdminRequest<Restore> {
+ protected String location;
+ protected String name;
+ protected String collection;
+
+ public String getLocation() {
+ return location;
+ }
+
+ public Restore setLocation(String location) {
+ this.location = location;
+ return this;
+ }
+
+ public Restore(String name, String collection) {
+ this.name = name;
+ this.collection = collection;
+ action = CollectionAction.RESTORE;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set(CoreAdminParams.COLLECTION, collection);
+ params.set(CoreAdminParams.NAME, name);
+ if (location != null) {
+ params.set("location", location);
+ }
+ return params;
+ }
+
+ @Override
+ protected Restore getThis() {
+ return this;
+ }
+ }
+
// CREATESHARD request
public static class CreateShard extends CollectionShardAsyncAdminRequest<CreateShard> {
protected String nodeSet;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 3dbc6d2..7c390f7 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -133,7 +133,8 @@ public class ZkStateReader implements Closeable {
public static final Set<String> KNOWN_CLUSTER_PROPS = unmodifiableSet(new HashSet<>(asList(
LEGACY_CLOUD,
URL_SCHEME,
- AUTO_ADD_REPLICAS)));
+ AUTO_ADD_REPLICAS,
+ "location")));
/**
* Returns config set name for collection.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index cb34b36..cc505f8 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -53,7 +53,9 @@ public interface CollectionParams
BALANCESHARDUNIQUE(true),
REBALANCELEADERS(true),
MODIFYCOLLECTION(true),
- MIGRATESTATEFORMAT(true);
+ MIGRATESTATEFORMAT(true),
+ BACKUP(true),
+ RESTORE(true);
public final boolean isWrite;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a3e7521/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
index 0a9e749..4219277 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
@@ -128,7 +128,10 @@ public abstract class CoreAdminParams
REJOINLEADERELECTION,
//internal API used by force shard leader election
FORCEPREPAREFORLEADERSHIP,
- INVOKE;
+ INVOKE,
+ //Internal APIs to backup and restore a core
+ BACKUPCORE,
+ RESTORECORE;
public static CoreAdminAction get( String p ) {
if (p != null) {
[2/3] lucene-solr:solr-5750: Merge branch 'master' into solr5750
Posted by ds...@apache.org.
Merge branch 'master' into solr5750
# Conflicts:
# solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/31a28f3d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/31a28f3d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/31a28f3d
Branch: refs/heads/solr-5750
Commit: 31a28f3da8e071b90f9856400394d17993df45d5
Parents: 1a3e752 a22099a
Author: David Smiley <ds...@apache.org>
Authored: Fri Mar 18 17:27:36 2016 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Fri Mar 18 17:27:36 2016 -0400
----------------------------------------------------------------------
dev-tools/idea/solr/core/src/java/solr-core.iml | 1 +
.../idea/solr/core/src/solr-core-tests.iml | 1 +
dev-tools/scripts/buildAndPushRelease.py | 2 +-
dev-tools/scripts/smokeTestRelease.py | 5 +-
lucene/CHANGES.txt | 46 +-
lucene/MIGRATE.txt | 14 +-
.../lucene/codecs/lucene50/Lucene50Codec.java | 6 +-
.../lucene/codecs/lucene53/Lucene53Codec.java | 8 +-
.../lucene/codecs/lucene54/Lucene54Codec.java | 8 +-
.../lucene50/TestLucene50DocValuesFormat.java | 2 +-
.../index/TestBackwardsCompatibility.java | 2 +-
.../lucene/benchmark/byTask/feeds/DocMaker.java | 44 +-
.../benchmark/byTask/tasks/ReadTokensTask.java | 13 +-
lucene/build.xml | 2 +-
.../classification/ClassificationTestBase.java | 7 +-
.../DocumentClassificationTestBase.java | 4 +-
.../classification/utils/DataSplitterTest.java | 5 +-
.../lucene/codecs/memory/FSTTermsWriter.java | 2 +-
.../codecs/simpletext/SimpleTextBKDReader.java | 6 +-
.../codecs/simpletext/SimpleTextCodec.java | 8 +-
.../simpletext/SimpleTextPointFormat.java | 53 -
.../simpletext/SimpleTextPointReader.java | 302 ----
.../simpletext/SimpleTextPointWriter.java | 241 ---
.../simpletext/SimpleTextPointsFormat.java | 53 +
.../simpletext/SimpleTextPointsReader.java | 302 ++++
.../simpletext/SimpleTextPointsWriter.java | 244 +++
.../simpletext/TestSimpleTextPointFormat.java | 33 -
.../simpletext/TestSimpleTextPointsFormat.java | 33 +
.../java/org/apache/lucene/codecs/Codec.java | 2 +-
.../org/apache/lucene/codecs/FilterCodec.java | 4 +-
.../org/apache/lucene/codecs/PointFormat.java | 111 --
.../org/apache/lucene/codecs/PointReader.java | 51 -
.../org/apache/lucene/codecs/PointWriter.java | 144 --
.../org/apache/lucene/codecs/PointsFormat.java | 111 ++
.../org/apache/lucene/codecs/PointsReader.java | 51 +
.../org/apache/lucene/codecs/PointsWriter.java | 169 ++
.../lucene54/Lucene54DocValuesConsumer.java | 21 +-
.../lucene/codecs/lucene60/Lucene60Codec.java | 6 +-
.../codecs/lucene60/Lucene60PointFormat.java | 106 --
.../codecs/lucene60/Lucene60PointReader.java | 241 ---
.../codecs/lucene60/Lucene60PointWriter.java | 220 ---
.../codecs/lucene60/Lucene60PointsFormat.java | 106 ++
.../codecs/lucene60/Lucene60PointsReader.java | 241 +++
.../codecs/lucene60/Lucene60PointsWriter.java | 241 +++
.../lucene/codecs/lucene60/package-info.java | 11 +-
.../org/apache/lucene/document/BinaryPoint.java | 2 +
.../org/apache/lucene/document/Document.java | 6 +-
.../org/apache/lucene/document/DoublePoint.java | 26 +-
.../java/org/apache/lucene/document/Field.java | 19 +-
.../org/apache/lucene/document/FieldType.java | 24 +-
.../org/apache/lucene/document/FloatPoint.java | 26 +-
.../org/apache/lucene/document/IntPoint.java | 26 +-
.../org/apache/lucene/document/LongPoint.java | 24 +-
.../document/SortedNumericDocValuesField.java | 4 +-
.../org/apache/lucene/index/CheckIndex.java | 17 +-
.../org/apache/lucene/index/CodecReader.java | 19 +-
.../apache/lucene/index/CompositeReader.java | 2 -
.../lucene/index/DefaultIndexingChain.java | 22 +-
.../java/org/apache/lucene/index/FieldInfo.java | 2 +-
.../apache/lucene/index/FilterCodecReader.java | 8 +-
.../apache/lucene/index/FilterLeafReader.java | 10 +-
.../org/apache/lucene/index/IndexReader.java | 2 -
.../org/apache/lucene/index/IndexWriter.java | 1 +
.../org/apache/lucene/index/LeafReader.java | 2 +-
.../org/apache/lucene/index/MergeState.java | 12 +-
.../org/apache/lucene/index/MultiDocValues.java | 6 +-
.../org/apache/lucene/index/PointValues.java | 142 +-
.../apache/lucene/index/PointValuesWriter.java | 24 +-
.../apache/lucene/index/SegmentCoreReaders.java | 10 +-
.../org/apache/lucene/index/SegmentMerger.java | 4 +-
.../org/apache/lucene/index/SegmentReader.java | 8 +-
.../index/SingletonSortedSetDocValues.java | 1 -
.../lucene/index/SlowCodecReaderWrapper.java | 8 +-
.../index/SlowCompositeReaderWrapper.java | 271 ---
.../apache/lucene/search/FilterCollector.java | 2 +-
.../lucene/search/FilterLeafCollector.java | 2 +-
.../lucene/search/LegacyNumericRangeQuery.java | 12 +-
.../apache/lucene/search/PointInSetQuery.java | 23 +-
.../apache/lucene/search/PointRangeQuery.java | 194 +-
.../apache/lucene/store/FilterDirectory.java | 2 +-
.../org/apache/lucene/util/ByteBlockPool.java | 22 +
.../org/apache/lucene/util/BytesRefArray.java | 41 +-
.../org/apache/lucene/util/IntroSorter.java | 6 +-
.../org/apache/lucene/util/OfflineSorter.java | 47 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 226 +--
.../apache/lucene/util/bkd/HeapPointReader.java | 17 +-
.../apache/lucene/util/bkd/HeapPointWriter.java | 47 +-
.../lucene/util/bkd/OfflinePointReader.java | 102 +-
.../lucene/util/bkd/OfflinePointWriter.java | 24 +-
.../org/apache/lucene/util/bkd/PointReader.java | 37 +-
.../lucene54/TestLucene54DocValuesFormat.java | 6 +-
.../lucene60/TestLucene60PointFormat.java | 83 -
.../lucene60/TestLucene60PointsFormat.java | 83 +
.../apache/lucene/document/TestDocument.java | 4 +-
.../apache/lucene/document/TestFieldType.java | 15 +-
.../org/apache/lucene/index/Test2BPoints.java | 148 ++
.../index/TestAllFilesCheckIndexHeader.java | 8 +-
.../index/TestAllFilesDetectTruncation.java | 4 +-
.../index/TestBinaryDocValuesUpdates.java | 28 +-
.../apache/lucene/index/TestCustomNorms.java | 4 +-
.../index/TestDemoParallelLeafReader.java | 21 +-
.../lucene/index/TestDirectoryReader.java | 2 +-
.../org/apache/lucene/index/TestDocValues.java | 14 +-
.../lucene/index/TestDocValuesIndexing.java | 25 +-
.../lucene/index/TestDocsAndPositions.java | 4 +-
.../apache/lucene/index/TestDocumentWriter.java | 2 +-
.../apache/lucene/index/TestDuelingCodecs.java | 4 +-
.../index/TestExitableDirectoryReader.java | 17 +-
.../lucene/index/TestFilterLeafReader.java | 10 +-
.../test/org/apache/lucene/index/TestFlex.java | 2 +-
.../index/TestFlushByRamOrCountsPolicy.java | 2 +-
.../lucene/index/TestForceMergeForever.java | 2 +-
.../lucene/index/TestIndexReaderClose.java | 52 +-
.../apache/lucene/index/TestIndexWriter.java | 6 +-
.../lucene/index/TestLazyProxSkipping.java | 2 +-
.../apache/lucene/index/TestMultiDocValues.java | 16 +-
.../lucene/index/TestMultiLevelSkipList.java | 2 +-
.../test/org/apache/lucene/index/TestNorms.java | 6 +-
.../index/TestNumericDocValuesUpdates.java | 30 +-
.../org/apache/lucene/index/TestOmitNorms.java | 8 +-
.../apache/lucene/index/TestOmitPositions.java | 2 +-
.../org/apache/lucene/index/TestOmitTf.java | 6 +-
.../org/apache/lucene/index/TestOrdinalMap.java | 5 +-
.../index/TestParallelCompositeReader.java | 34 +-
.../lucene/index/TestParallelLeafReader.java | 28 +-
.../index/TestParallelReaderEmptyIndex.java | 22 +-
.../lucene/index/TestParallelTermEnum.java | 4 +-
.../org/apache/lucene/index/TestPayloads.java | 11 +-
.../apache/lucene/index/TestPointValues.java | 166 +-
.../lucene/index/TestPostingsOffsets.java | 5 +-
.../apache/lucene/index/TestReaderClosed.java | 6 +-
.../index/TestReaderWrapperDVTypeCheck.java | 5 +-
.../apache/lucene/index/TestRollingUpdates.java | 2 +-
.../lucene/index/TestSegmentTermEnum.java | 2 +-
.../lucene/index/TestSortedSetDocValues.java | 27 +
.../apache/lucene/index/TestStressAdvance.java | 2 +-
.../lucene/index/TestSwappedIndexFiles.java | 3 +-
.../org/apache/lucene/index/TestTermsEnum.java | 8 +-
.../lucene/search/TestDisjunctionMaxQuery.java | 6 +-
.../lucene/search/TestMinShouldMatch2.java | 2 +-
.../lucene/search/TestMultiPhraseEnum.java | 8 +-
.../apache/lucene/search/TestPhraseQuery.java | 4 +-
.../apache/lucene/search/TestPointQueries.java | 159 +-
.../lucene/search/TestPositionIncrement.java | 10 +-
.../lucene/search/TestSimilarityProvider.java | 11 +-
.../apache/lucene/search/TestTermScorer.java | 9 +-
.../TestUsageTrackingFilterCachingPolicy.java | 17 +-
.../search/spans/TestFieldMaskingSpanQuery.java | 11 +-
.../search/spans/TestNearSpansOrdered.java | 29 +-
.../lucene/search/spans/TestSpanCollection.java | 11 +-
.../search/spans/TestSpanContainQuery.java | 7 +-
.../apache/lucene/search/spans/TestSpans.java | 13 +-
.../lucene/store/TestFilterDirectory.java | 4 +-
.../lucene/store/TestNRTCachingDirectory.java | 2 +-
.../apache/lucene/util/TestOfflineSorter.java | 2 +-
.../apache/lucene/util/bkd/Test2BBKDPoints.java | 121 ++
.../org/apache/lucene/util/bkd/TestBKD.java | 22 +-
.../org/apache/lucene/util/fst/TestFSTs.java | 2 +-
.../apache/lucene/facet/range/DoubleRange.java | 3 +-
.../DefaultSortedSetDocValuesReaderState.java | 55 +-
.../facet/range/TestRangeFacetCounts.java | 194 +-
.../sortedset/TestSortedSetDocValuesFacets.java | 36 -
.../lucene/search/grouping/TestGrouping.java | 12 +-
lucene/ivy-versions.properties | 2 +-
.../lucene/search/join/TestBlockJoin.java | 87 +-
.../apache/lucene/search/join/TestJoinUtil.java | 8 +-
.../jetty-continuation-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-continuation-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-http-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-http-9.3.8.v20160314.jar.sha1 | 1 +
.../licenses/jetty-io-9.3.6.v20151106.jar.sha1 | 1 -
.../licenses/jetty-io-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-server-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-server-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-servlet-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-servlet-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-util-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-util-9.3.8.v20160314.jar.sha1 | 1 +
.../apache/lucene/index/memory/MemoryIndex.java | 628 +++++--
.../lucene/index/memory/TestMemoryIndex.java | 293 ++++
.../memory/TestMemoryIndexAgainstRAMDir.java | 204 ++-
.../index/SlowCompositeReaderWrapper.java | 275 +++
.../apache/lucene/uninverting/FieldCache.java | 113 +-
.../lucene/uninverting/FieldCacheImpl.java | 164 +-
.../lucene/uninverting/UninvertingReader.java | 97 +-
.../index/TestSlowCompositeReaderWrapper.java | 91 +
.../lucene/uninverting/TestDocTermOrds.java | 14 +-
.../lucene/uninverting/TestFieldCache.java | 147 +-
.../uninverting/TestFieldCacheReopen.java | 11 +-
.../TestFieldCacheSanityChecker.java | 10 +-
.../lucene/uninverting/TestFieldCacheSort.java | 684 +++++++-
.../uninverting/TestFieldCacheSortRandom.java | 8 +-
.../uninverting/TestFieldCacheVsDocValues.java | 6 +-
.../uninverting/TestFieldCacheWithThreads.java | 13 +-
.../uninverting/TestLegacyFieldCache.java | 498 ++++++
.../lucene/uninverting/TestNumericTerms32.java | 6 +-
.../lucene/uninverting/TestNumericTerms64.java | 8 +-
.../uninverting/TestUninvertingReader.java | 6 +-
.../lucene/queries/CommonTermsQueryTest.java | 6 +-
.../apache/lucene/queries/TermsQueryTest.java | 4 +-
.../queries/function/FunctionTestSetup.java | 8 +-
.../queries/function/TestFunctionQuerySort.java | 4 +-
.../function/TestSortedSetFieldSource.java | 2 +-
.../queries/function/TestValueSources.java | 16 -
.../lucene/queries/payloads/PayloadHelper.java | 3 +-
.../queries/payloads/TestPayloadSpans.java | 42 +-
.../queries/payloads/TestPayloadTermQuery.java | 11 +-
.../flexible/core/nodes/package-info.java | 4 +-
.../flexible/standard/StandardQueryParser.java | 29 +-
.../LegacyNumericRangeQueryNodeBuilder.java | 93 +
.../builders/NumericRangeQueryNodeBuilder.java | 91 -
.../builders/PointRangeQueryNodeBuilder.java | 137 ++
.../builders/StandardQueryTreeBuilder.java | 12 +-
.../standard/config/LegacyNumericConfig.java | 166 ++
.../LegacyNumericFieldConfigListener.java | 75 +
.../flexible/standard/config/NumericConfig.java | 164 --
.../config/NumericFieldConfigListener.java | 73 -
.../flexible/standard/config/PointsConfig.java | 124 ++
.../standard/config/PointsConfigListener.java | 65 +
.../config/StandardQueryConfigHandler.java | 39 +-
.../standard/nodes/LegacyNumericQueryNode.java | 153 ++
.../nodes/LegacyNumericRangeQueryNode.java | 153 ++
.../standard/nodes/NumericQueryNode.java | 151 --
.../standard/nodes/NumericRangeQueryNode.java | 151 --
.../flexible/standard/nodes/PointQueryNode.java | 151 ++
.../standard/nodes/PointRangeQueryNode.java | 124 ++
.../LegacyNumericQueryNodeProcessor.java | 154 ++
.../LegacyNumericRangeQueryNodeProcessor.java | 170 ++
.../processors/NumericQueryNodeProcessor.java | 152 --
.../NumericRangeQueryNodeProcessor.java | 168 --
.../processors/PointQueryNodeProcessor.java | 136 ++
.../PointRangeQueryNodeProcessor.java | 148 ++
.../StandardQueryNodeProcessorPipeline.java | 6 +-
.../lucene/queryparser/xml/CoreParser.java | 1 +
.../LegacyNumericRangeQueryBuilder.java | 2 +
.../xml/builders/PointRangeQueryBuilder.java | 95 +
.../standard/TestLegacyNumericQueryParser.java | 535 ++++++
.../standard/TestNumericQueryParser.java | 535 ------
.../flexible/standard/TestPointQueryParser.java | 82 +
.../lucene/queryparser/xml/PointRangeQuery.xml | 31 +
.../lucene/queryparser/xml/TestCoreParser.java | 9 +-
.../apache/lucene/document/BigIntegerPoint.java | 2 +
.../lucene/document/InetAddressPoint.java | 25 +-
.../org/apache/lucene/document/LatLonPoint.java | 80 +-
.../document/LatLonPointDistanceComparator.java | 213 +++
.../document/LatLonPointDistanceQuery.java | 150 +-
.../document/LatLonPointInPolygonQuery.java | 65 +-
.../lucene/document/LatLonPointSortField.java | 106 ++
.../lucene/document/TestBigIntegerPoint.java | 19 +-
.../lucene/document/TestInetAddressPoint.java | 25 +-
.../apache/lucene/document/TestLatLonPoint.java | 35 +-
.../document/TestLatLonPointDistanceQuery.java | 28 +-
.../document/TestLatLonPointDistanceSort.java | 289 +++
.../document/TestLatLonPointInPolygonQuery.java | 49 +
.../lucene/search/TestLatLonPointQueries.java | 86 +-
.../lucene/spatial/spatial4j/Geo3dShape.java | 12 +-
.../apache/lucene/spatial/SpatialTestCase.java | 4 +-
.../lucene/spatial/spatial4j/Geo3dRptTest.java | 14 +-
.../Geo3dShapeRectRelationTestCase.java | 18 +-
.../Geo3dShapeSphereModelRectRelationTest.java | 16 +-
.../Geo3dShapeWGS84ModelRectRelationTest.java | 16 +-
.../spatial/spatial4j/geo3d/GeoPointTest.java | 80 -
.../org/apache/lucene/spatial/util/GeoRect.java | 5 +
.../org/apache/lucene/geo3d/ArcDistance.java | 56 -
.../apache/lucene/geo3d/BasePlanetObject.java | 52 -
.../org/apache/lucene/geo3d/BaseXYZSolid.java | 167 --
.../java/org/apache/lucene/geo3d/Bounds.java | 113 --
.../org/apache/lucene/geo3d/DistanceStyle.java | 83 -
.../org/apache/lucene/geo3d/Geo3DPoint.java | 117 --
.../java/org/apache/lucene/geo3d/Geo3DUtil.java | 59 -
.../java/org/apache/lucene/geo3d/GeoArea.java | 67 -
.../org/apache/lucene/geo3d/GeoAreaFactory.java | 55 -
.../java/org/apache/lucene/geo3d/GeoBBox.java | 36 -
.../org/apache/lucene/geo3d/GeoBBoxFactory.java | 111 --
.../org/apache/lucene/geo3d/GeoBaseBBox.java | 72 -
.../org/apache/lucene/geo3d/GeoBaseCircle.java | 34 -
.../lucene/geo3d/GeoBaseDistanceShape.java | 56 -
.../lucene/geo3d/GeoBaseMembershipShape.java | 56 -
.../org/apache/lucene/geo3d/GeoBasePolygon.java | 34 -
.../org/apache/lucene/geo3d/GeoBaseShape.java | 59 -
.../java/org/apache/lucene/geo3d/GeoCircle.java | 25 -
.../apache/lucene/geo3d/GeoCircleFactory.java | 43 -
.../geo3d/GeoCompositeMembershipShape.java | 117 --
.../lucene/geo3d/GeoCompositePolygon.java | 31 -
.../apache/lucene/geo3d/GeoConvexPolygon.java | 288 ---
.../geo3d/GeoDegenerateHorizontalLine.java | 215 ---
.../lucene/geo3d/GeoDegenerateLatitudeZone.java | 138 --
.../geo3d/GeoDegenerateLongitudeSlice.java | 153 --
.../apache/lucene/geo3d/GeoDegeneratePoint.java | 135 --
.../lucene/geo3d/GeoDegenerateVerticalLine.java | 205 ---
.../org/apache/lucene/geo3d/GeoDistance.java | 59 -
.../apache/lucene/geo3d/GeoDistanceShape.java | 27 -
.../apache/lucene/geo3d/GeoLatitudeZone.java | 198 ---
.../apache/lucene/geo3d/GeoLongitudeSlice.java | 204 ---
.../apache/lucene/geo3d/GeoMembershipShape.java | 27 -
.../lucene/geo3d/GeoNorthLatitudeZone.java | 165 --
.../apache/lucene/geo3d/GeoNorthRectangle.java | 263 ---
.../apache/lucene/geo3d/GeoOutsideDistance.java | 55 -
.../java/org/apache/lucene/geo3d/GeoPath.java | 797 ---------
.../java/org/apache/lucene/geo3d/GeoPoint.java | 193 --
.../org/apache/lucene/geo3d/GeoPolygon.java | 26 -
.../apache/lucene/geo3d/GeoPolygonFactory.java | 187 --
.../org/apache/lucene/geo3d/GeoRectangle.java | 288 ---
.../java/org/apache/lucene/geo3d/GeoShape.java | 63 -
.../org/apache/lucene/geo3d/GeoSizeable.java | 40 -
.../lucene/geo3d/GeoSouthLatitudeZone.java | 168 --
.../apache/lucene/geo3d/GeoSouthRectangle.java | 259 ---
.../apache/lucene/geo3d/GeoStandardCircle.java | 168 --
.../geo3d/GeoWideDegenerateHorizontalLine.java | 238 ---
.../lucene/geo3d/GeoWideLongitudeSlice.java | 208 ---
.../lucene/geo3d/GeoWideNorthRectangle.java | 286 ---
.../apache/lucene/geo3d/GeoWideRectangle.java | 319 ----
.../lucene/geo3d/GeoWideSouthRectangle.java | 284 ---
.../java/org/apache/lucene/geo3d/GeoWorld.java | 106 --
.../org/apache/lucene/geo3d/LatLonBounds.java | 322 ----
.../org/apache/lucene/geo3d/LinearDistance.java | 56 -
.../lucene/geo3d/LinearSquaredDistance.java | 56 -
.../org/apache/lucene/geo3d/Membership.java | 46 -
.../org/apache/lucene/geo3d/NormalDistance.java | 56 -
.../lucene/geo3d/NormalSquaredDistance.java | 56 -
.../src/java/org/apache/lucene/geo3d/Plane.java | 1657 ------------------
.../org/apache/lucene/geo3d/PlanetModel.java | 277 ---
.../lucene/geo3d/PointInGeo3DShapeQuery.java | 212 ---
.../org/apache/lucene/geo3d/SidedPlane.java | 175 --
.../apache/lucene/geo3d/StandardXYZSolid.java | 417 -----
.../src/java/org/apache/lucene/geo3d/Tools.java | 41 -
.../java/org/apache/lucene/geo3d/Vector.java | 378 ----
.../java/org/apache/lucene/geo3d/XYZBounds.java | 267 ---
.../java/org/apache/lucene/geo3d/XYZSolid.java | 26 -
.../apache/lucene/geo3d/XYZSolidFactory.java | 67 -
.../java/org/apache/lucene/geo3d/XYdZSolid.java | 213 ---
.../java/org/apache/lucene/geo3d/XdYZSolid.java | 212 ---
.../org/apache/lucene/geo3d/XdYdZSolid.java | 138 --
.../java/org/apache/lucene/geo3d/dXYZSolid.java | 216 ---
.../org/apache/lucene/geo3d/dXYdZSolid.java | 138 --
.../org/apache/lucene/geo3d/dXdYZSolid.java | 138 --
.../org/apache/lucene/geo3d/dXdYdZSolid.java | 146 --
.../org/apache/lucene/geo3d/package-info.java | 21 -
.../org/apache/lucene/spatial3d/Geo3DPoint.java | 115 ++
.../org/apache/lucene/spatial3d/Geo3DUtil.java | 59 +
.../spatial3d/PointInGeo3DShapeQuery.java | 215 +++
.../lucene/spatial3d/geom/ArcDistance.java | 56 +
.../lucene/spatial3d/geom/BasePlanetObject.java | 57 +
.../lucene/spatial3d/geom/BaseXYZSolid.java | 167 ++
.../apache/lucene/spatial3d/geom/Bounds.java | 113 ++
.../lucene/spatial3d/geom/DistanceStyle.java | 83 +
.../apache/lucene/spatial3d/geom/GeoArea.java | 67 +
.../lucene/spatial3d/geom/GeoAreaFactory.java | 55 +
.../apache/lucene/spatial3d/geom/GeoBBox.java | 36 +
.../lucene/spatial3d/geom/GeoBBoxFactory.java | 111 ++
.../lucene/spatial3d/geom/GeoBaseBBox.java | 72 +
.../lucene/spatial3d/geom/GeoBaseCircle.java | 34 +
.../spatial3d/geom/GeoBaseDistanceShape.java | 56 +
.../spatial3d/geom/GeoBaseMembershipShape.java | 56 +
.../lucene/spatial3d/geom/GeoBasePolygon.java | 34 +
.../lucene/spatial3d/geom/GeoBaseShape.java | 59 +
.../apache/lucene/spatial3d/geom/GeoCircle.java | 25 +
.../lucene/spatial3d/geom/GeoCircleFactory.java | 43 +
.../geom/GeoCompositeMembershipShape.java | 117 ++
.../spatial3d/geom/GeoCompositePolygon.java | 31 +
.../lucene/spatial3d/geom/GeoConvexPolygon.java | 288 +++
.../geom/GeoDegenerateHorizontalLine.java | 215 +++
.../geom/GeoDegenerateLatitudeZone.java | 138 ++
.../geom/GeoDegenerateLongitudeSlice.java | 153 ++
.../spatial3d/geom/GeoDegeneratePoint.java | 135 ++
.../geom/GeoDegenerateVerticalLine.java | 205 +++
.../lucene/spatial3d/geom/GeoDistance.java | 59 +
.../lucene/spatial3d/geom/GeoDistanceShape.java | 27 +
.../lucene/spatial3d/geom/GeoLatitudeZone.java | 198 +++
.../spatial3d/geom/GeoLongitudeSlice.java | 204 +++
.../spatial3d/geom/GeoMembershipShape.java | 27 +
.../spatial3d/geom/GeoNorthLatitudeZone.java | 165 ++
.../spatial3d/geom/GeoNorthRectangle.java | 263 +++
.../spatial3d/geom/GeoOutsideDistance.java | 55 +
.../apache/lucene/spatial3d/geom/GeoPath.java | 797 +++++++++
.../apache/lucene/spatial3d/geom/GeoPoint.java | 193 ++
.../lucene/spatial3d/geom/GeoPolygon.java | 26 +
.../spatial3d/geom/GeoPolygonFactory.java | 187 ++
.../lucene/spatial3d/geom/GeoRectangle.java | 288 +++
.../apache/lucene/spatial3d/geom/GeoShape.java | 63 +
.../lucene/spatial3d/geom/GeoSizeable.java | 40 +
.../spatial3d/geom/GeoSouthLatitudeZone.java | 168 ++
.../spatial3d/geom/GeoSouthRectangle.java | 259 +++
.../spatial3d/geom/GeoStandardCircle.java | 168 ++
.../geom/GeoWideDegenerateHorizontalLine.java | 238 +++
.../spatial3d/geom/GeoWideLongitudeSlice.java | 208 +++
.../spatial3d/geom/GeoWideNorthRectangle.java | 286 +++
.../lucene/spatial3d/geom/GeoWideRectangle.java | 319 ++++
.../spatial3d/geom/GeoWideSouthRectangle.java | 284 +++
.../apache/lucene/spatial3d/geom/GeoWorld.java | 106 ++
.../lucene/spatial3d/geom/LatLonBounds.java | 322 ++++
.../lucene/spatial3d/geom/LinearDistance.java | 56 +
.../spatial3d/geom/LinearSquaredDistance.java | 56 +
.../lucene/spatial3d/geom/Membership.java | 46 +
.../lucene/spatial3d/geom/NormalDistance.java | 56 +
.../spatial3d/geom/NormalSquaredDistance.java | 56 +
.../org/apache/lucene/spatial3d/geom/Plane.java | 1657 ++++++++++++++++++
.../lucene/spatial3d/geom/PlanetModel.java | 277 +++
.../lucene/spatial3d/geom/SidedPlane.java | 175 ++
.../lucene/spatial3d/geom/StandardXYZSolid.java | 417 +++++
.../org/apache/lucene/spatial3d/geom/Tools.java | 41 +
.../apache/lucene/spatial3d/geom/Vector.java | 378 ++++
.../apache/lucene/spatial3d/geom/XYZBounds.java | 267 +++
.../apache/lucene/spatial3d/geom/XYZSolid.java | 26 +
.../lucene/spatial3d/geom/XYZSolidFactory.java | 67 +
.../apache/lucene/spatial3d/geom/XYdZSolid.java | 213 +++
.../apache/lucene/spatial3d/geom/XdYZSolid.java | 212 +++
.../lucene/spatial3d/geom/XdYdZSolid.java | 138 ++
.../apache/lucene/spatial3d/geom/dXYZSolid.java | 216 +++
.../lucene/spatial3d/geom/dXYdZSolid.java | 138 ++
.../lucene/spatial3d/geom/dXdYZSolid.java | 138 ++
.../lucene/spatial3d/geom/dXdYdZSolid.java | 146 ++
.../lucene/spatial3d/geom/package-info.java | 22 +
.../apache/lucene/spatial3d/package-info.java | 21 +
lucene/spatial3d/src/java/overview.html | 3 +-
.../org/apache/lucene/geo3d/GeoBBoxTest.java | 364 ----
.../org/apache/lucene/geo3d/GeoCircleTest.java | 415 -----
.../lucene/geo3d/GeoConvexPolygonTest.java | 91 -
.../org/apache/lucene/geo3d/GeoModelTest.java | 110 --
.../org/apache/lucene/geo3d/GeoPathTest.java | 270 ---
.../org/apache/lucene/geo3d/GeoPolygonTest.java | 165 --
.../test/org/apache/lucene/geo3d/PlaneTest.java | 64 -
.../org/apache/lucene/geo3d/TestGeo3DPoint.java | 804 ---------
.../org/apache/lucene/geo3d/XYZSolidTest.java | 220 ---
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 824 +++++++++
.../lucene/spatial3d/geom/GeoBBoxTest.java | 364 ++++
.../lucene/spatial3d/geom/GeoCircleTest.java | 410 +++++
.../spatial3d/geom/GeoConvexPolygonTest.java | 91 +
.../lucene/spatial3d/geom/GeoModelTest.java | 110 ++
.../lucene/spatial3d/geom/GeoPathTest.java | 270 +++
.../lucene/spatial3d/geom/GeoPointTest.java | 77 +
.../lucene/spatial3d/geom/GeoPolygonTest.java | 165 ++
.../apache/lucene/spatial3d/geom/PlaneTest.java | 64 +
.../lucene/spatial3d/geom/XYZSolidTest.java | 220 +++
.../search/suggest/SortedInputIterator.java | 16 +-
.../analyzing/TestFreeTextSuggester.java | 2 +-
.../lucene/codecs/asserting/AssertingCodec.java | 8 +-
.../codecs/asserting/AssertingPointFormat.java | 276 ---
.../codecs/asserting/AssertingPointsFormat.java | 276 +++
.../lucene/codecs/cranky/CrankyCodec.java | 6 +-
.../lucene/codecs/cranky/CrankyPointFormat.java | 185 --
.../codecs/cranky/CrankyPointsFormat.java | 185 ++
.../index/BaseDocValuesFormatTestCase.java | 64 +-
.../index/BaseIndexFileFormatTestCase.java | 6 +-
.../lucene/index/BasePointFormatTestCase.java | 950 ----------
.../lucene/index/BasePointsFormatTestCase.java | 950 ++++++++++
.../index/BasePostingsFormatTestCase.java | 66 +-
.../index/BaseStoredFieldsFormatTestCase.java | 55 +-
.../index/BaseTermVectorsFormatTestCase.java | 12 +-
.../lucene/index/MockRandomMergePolicy.java | 2 +-
.../org/apache/lucene/index/RandomCodec.java | 24 +-
.../ThreadedIndexingAndSearchingTestCase.java | 4 +-
.../lucene/mockfile/FilterFileChannel.java | 2 +-
.../apache/lucene/mockfile/FilterFileStore.java | 2 +-
.../lucene/mockfile/FilterFileSystem.java | 2 +-
.../mockfile/FilterFileSystemProvider.java | 2 +-
.../lucene/mockfile/FilterOutputStream2.java | 2 +-
.../org/apache/lucene/search/QueryUtils.java | 6 +-
.../lucene/search/ShardSearchingTestBase.java | 2 +-
.../lucene/search/spans/MultiSpansWrapper.java | 51 -
.../lucene/store/MockDirectoryWrapper.java | 9 +-
.../org/apache/lucene/util/LineFileDocs.java | 58 +-
.../org/apache/lucene/util/LuceneTestCase.java | 53 +-
.../java/org/apache/lucene/util/TestUtil.java | 37 +-
.../lucene/analysis/TestMockAnalyzer.java | 2 +-
.../asserting/TestAssertingPointFormat.java | 30 -
.../asserting/TestAssertingPointsFormat.java | 30 +
.../TestCompressingStoredFieldsFormat.java | 5 +-
.../TestCompressingTermVectorsFormat.java | 8 +-
.../lucene/index/TestAssertingLeafReader.java | 5 +-
lucene/tools/javadoc/ecj.javadocs.prefs | 4 +-
lucene/tools/junit4/cached-timehints.txt | 2 +-
solr/CHANGES.txt | 84 +-
solr/build.xml | 16 +-
solr/common-build.xml | 4 +-
.../solr/hadoop/MorphlineGoLiveMiniMRTest.java | 1 -
.../solr/collection1/conf/solrconfig.xml | 6 -
.../test-files/solr/minimr/conf/solrconfig.xml | 6 -
.../test-files/solr/mrunit/conf/solrconfig.xml | 7 -
.../collection1/conf/solrconfig.xml | 6 -
.../solr/solrcloud/conf/solrconfig.xml | 6 -
.../org/apache/solr/cloud/ElectionContext.java | 2 +-
.../cloud/LeaderInitiatedRecoveryThread.java | 6 -
.../cloud/OverseerCollectionMessageHandler.java | 9 +-
.../org/apache/solr/cloud/ZkController.java | 2 +-
.../org/apache/solr/core/CoreContainer.java | 15 +-
.../src/java/org/apache/solr/core/SolrCore.java | 64 +-
.../apache/solr/handler/CdcrRequestHandler.java | 2 +-
.../org/apache/solr/handler/RestoreCore.java | 6 +-
.../org/apache/solr/handler/SchemaHandler.java | 78 +-
.../solr/handler/admin/ClusterStatus.java | 19 +-
.../solr/handler/admin/CollectionsHandler.java | 105 +-
.../solr/handler/admin/CoreAdminOperation.java | 4 +-
.../solr/handler/admin/LukeRequestHandler.java | 12 +-
.../solr/handler/admin/RebalanceLeaders.java | 2 +-
.../apache/solr/handler/loader/JsonLoader.java | 6 +-
.../org/apache/solr/request/SimpleFacets.java | 12 +-
.../solr/response/GeoJSONResponseWriter.java | 345 ++++
.../solr/response/JSONResponseWriter.java | 2 +-
.../transform/GeoTransformerFactory.java | 224 +++
.../response/transform/TransformerFactory.java | 1 +
.../response/transform/WriteableGeoJSON.java | 55 +
.../org/apache/solr/rest/SolrSchemaRestApi.java | 56 +-
.../solr/rest/schema/BaseFieldResource.java | 146 --
.../solr/rest/schema/BaseFieldTypeResource.java | 98 --
.../schema/CopyFieldCollectionResource.java | 198 ---
.../schema/DynamicFieldCollectionResource.java | 207 ---
.../solr/rest/schema/DynamicFieldResource.java | 197 ---
.../rest/schema/FieldCollectionResource.java | 225 ---
.../apache/solr/rest/schema/FieldResource.java | 201 ---
.../schema/FieldTypeCollectionResource.java | 197 ---
.../solr/rest/schema/FieldTypeResource.java | 203 ---
.../solr/schema/AbstractSpatialFieldType.java | 87 +-
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../java/org/apache/solr/schema/EnumField.java | 2 +-
.../org/apache/solr/schema/IndexSchema.java | 11 +-
.../java/org/apache/solr/schema/TrieField.java | 8 +-
.../apache/solr/search/SolrIndexSearcher.java | 4 +-
.../apache/solr/search/facet/FacetField.java | 13 +-
.../solr/search/facet/UnInvertedField.java | 9 +-
.../org/apache/solr/servlet/HttpSolrCall.java | 3 +
.../org/apache/solr/update/SolrCoreState.java | 8 +-
.../org/apache/solr/update/TransactionLog.java | 4 +-
.../org/apache/solr/update/VersionInfo.java | 8 +-
.../ClassificationUpdateProcessor.java | 102 ++
.../ClassificationUpdateProcessorFactory.java | 223 +++
.../collection1/conf/schema-classification.xml | 43 +
.../conf/schema-non-stored-docvalues.xml | 131 +-
.../solr/collection1/conf/schema-spatial.xml | 1 +
.../conf/solrconfig-classification.xml | 53 +
.../configsets/cloud-minimal/conf/schema.xml | 32 +
.../cloud-minimal/conf/solrconfig.xml | 48 +
.../org/apache/solr/TestRandomDVFaceting.java | 3 -
.../solr/cloud/BaseCdcrDistributedZkTest.java | 1 -
.../solr/cloud/BasicDistributedZkTest.java | 4 +-
.../cloud/ChaosMonkeyNothingIsSafeTest.java | 2 +-
.../solr/cloud/ChaosMonkeyShardSplitTest.java | 2 +-
.../apache/solr/cloud/CollectionReloadTest.java | 2 +-
.../cloud/CollectionTooManyReplicasTest.java | 6 +-
.../CollectionsAPIAsyncDistributedZkTest.java | 174 +-
.../cloud/CollectionsAPIDistributedZkTest.java | 9 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 474 +++++
.../solr/cloud/CollectionsAPISolrJTests.java | 472 -----
.../apache/solr/cloud/CustomCollectionTest.java | 1 -
.../org/apache/solr/cloud/DeleteShardTest.java | 2 -
.../org/apache/solr/cloud/DeleteStatusTest.java | 172 +-
.../org/apache/solr/cloud/ForceLeaderTest.java | 11 +-
.../apache/solr/cloud/HttpPartitionTest.java | 9 +-
.../cloud/LeaderFailoverAfterPartitionTest.java | 2 -
.../LeaderInitiatedRecoveryOnCommitTest.java | 4 +-
.../apache/solr/cloud/MigrateRouteKeyTest.java | 4 +-
.../org/apache/solr/cloud/OverseerTest.java | 12 +-
.../solr/cloud/ReplicaPropertiesBase.java | 3 -
.../org/apache/solr/cloud/ShardSplitTest.java | 1 -
.../apache/solr/cloud/SolrCloudExampleTest.java | 1 -
.../org/apache/solr/cloud/SyncSliceTest.java | 1 -
.../solr/cloud/TestCloudDeleteByQuery.java | 1 -
.../apache/solr/cloud/TestCollectionAPI.java | 23 +-
.../TestLeaderInitiatedRecoveryThread.java | 1 -
.../solr/cloud/TestMiniSolrCloudCluster.java | 47 +-
.../cloud/TestMiniSolrCloudClusterBase.java | 3 +-
.../cloud/TestRandomRequestDistribution.java | 4 +-
.../apache/solr/cloud/TestRebalanceLeaders.java | 1 -
.../solr/cloud/TestReplicaProperties.java | 1 -
.../cloud/TestSolrCloudWithKerberosAlt.java | 1 +
.../solr/cloud/UnloadDistributedZkTest.java | 4 +-
.../org/apache/solr/cloud/ZkControllerTest.java | 2 +-
.../apache/solr/cloud/hdfs/StressHdfsTest.java | 3 +-
.../solr/cloud/overseer/ZkStateReaderTest.java | 6 +-
.../solr/cloud/overseer/ZkStateWriterTest.java | 10 +-
.../org/apache/solr/handler/JsonLoaderTest.java | 20 +
.../apache/solr/handler/TestRestoreCore.java | 55 +-
.../apache/solr/request/SimpleFacetsTest.java | 12 +-
.../org/apache/solr/request/TestFaceting.java | 8 +-
.../response/TestGeoJSONResponseWriter.java | 279 +++
.../rest/schema/TestClassNameShortening.java | 3 +-
.../schema/TestCopyFieldCollectionResource.java | 96 +-
.../TestDynamicFieldCollectionResource.java | 29 -
.../rest/schema/TestDynamicFieldResource.java | 7 -
.../schema/TestFieldCollectionResource.java | 45 -
.../solr/rest/schema/TestFieldResource.java | 23 +-
.../schema/TestFieldTypeCollectionResource.java | 1 +
.../solr/rest/schema/TestFieldTypeResource.java | 17 +-
.../TestManagedSchemaDynamicFieldResource.java | 366 ----
.../schema/TestManagedSchemaFieldResource.java | 369 ----
.../TestManagedSchemaFieldTypeResource.java | 350 ----
.../schema/TestRemoveLastDynamicCopyField.java | 80 -
.../schema/TestSchemaSimilarityResource.java | 1 -
.../analysis/TestManagedStopFilterFactory.java | 2 +-
.../TestManagedSynonymFilterFactory.java | 6 +-
.../solr/schema/SpatialRPTFieldTypeTest.java | 38 +-
.../TestCloudManagedSchemaConcurrent.java | 3 +-
.../solr/schema/TestUseDocValuesAsStored.java | 217 ++-
.../solr/search/facet/TestJsonFacets.java | 71 +-
.../solr/security/BasicAuthIntegrationTest.java | 2 +-
.../solr/update/DirectUpdateHandlerTest.java | 2 +-
.../test/org/apache/solr/update/TestUpdate.java | 21 +
...lassificationUpdateProcessorFactoryTest.java | 234 +++
.../example-DIH/solr/db/conf/solrconfig.xml | 6 -
.../example-DIH/solr/mail/conf/solrconfig.xml | 6 -
.../example-DIH/solr/rss/conf/solrconfig.xml | 6 -
.../example-DIH/solr/solr/conf/solrconfig.xml | 6 -
.../example-DIH/solr/tika/conf/solrconfig.xml | 7 -
solr/example/files/conf/solrconfig.xml | 6 -
.../jetty-continuation-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-continuation-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-deploy-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-deploy-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-http-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-http-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/jetty-io-9.3.6.v20151106.jar.sha1 | 1 -
solr/licenses/jetty-io-9.3.8.v20160314.jar.sha1 | 1 +
.../licenses/jetty-jmx-9.3.6.v20151106.jar.sha1 | 1 -
.../licenses/jetty-jmx-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-rewrite-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-rewrite-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-security-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-security-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-server-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-server-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-servlet-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-servlet-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-servlets-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-servlets-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-util-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-util-9.3.8.v20160314.jar.sha1 | 1 +
.../jetty-webapp-9.3.6.v20151106.jar.sha1 | 1 -
.../jetty-webapp-9.3.8.v20160314.jar.sha1 | 1 +
.../licenses/jetty-xml-9.3.6.v20151106.jar.sha1 | 1 -
.../licenses/jetty-xml-9.3.8.v20160314.jar.sha1 | 1 +
solr/licenses/start.jar.sha1 | 2 +-
.../basic_configs/conf/managed-schema | 32 +-
.../basic_configs/conf/solrconfig.xml | 5 -
.../conf/managed-schema | 53 +-
.../conf/solrconfig.xml | 7 -
.../conf/managed-schema | 43 +-
.../conf/solrconfig.xml | 6 -
.../solr/client/solrj/io/sql/ResultSetImpl.java | 2 +-
.../client/solrj/io/stream/DaemonStream.java | 6 +-
.../solrj/request/CollectionAdminRequest.java | 1211 ++++++++++---
.../client/solrj/request/CoreAdminRequest.java | 23 +-
.../solrj/util/SolrIdentifierValidator.java | 28 +-
.../apache/solr/common/cloud/ClusterState.java | 4 +-
.../org/apache/solr/common/cloud/DocRouter.java | 27 +-
.../apache/solr/common/cloud/ZkStateReader.java | 46 +
.../common/params/CollectionAdminParams.java | 2 +
.../apache/solr/common/util/JavaBinCodec.java | 4 +-
.../solr/client/solrj/io/sql/JdbcTest.java | 15 +
.../solrj/io/stream/StreamExpressionTest.java | 8 +-
.../solr/client/solrj/request/SchemaTest.java | 4 -
.../request/TestCollectionAdminRequest.java | 9 +-
.../client/solrj/request/TestCoreAdmin.java | 13 +-
.../solr/cloud/AbstractDistribZkTestBase.java | 4 +-
.../cloud/AbstractFullDistribZkTestBase.java | 10 +-
.../java/org/apache/solr/cloud/ChaosMonkey.java | 114 +-
solr/webapp/web/css/angular/common.css | 13 +-
solr/webapp/web/css/styles/common.css | 16 +-
solr/webapp/web/index.html | 6 +-
solr/webapp/web/old.html | 5 +-
solr/webapp/web/partials/query.html | 12 +-
660 files changed, 31728 insertions(+), 26640 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31a28f3d/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31a28f3d/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/handler/RestoreCore.java
index a8ee719,9949d3f..fcda176
--- a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
+++ b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
@@@ -52,10 -55,11 +55,11 @@@ public class RestoreCore implements Cal
return doRestore();
}
- private boolean doRestore() throws Exception {
+ public boolean doRestore() throws Exception {
Path backupPath = Paths.get(backupLocation).resolve(backupName);
- String restoreIndexName = "restore." + backupName;
+ SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT);
+ String restoreIndexName = "restore." + dateFormat.format(new Date());
String restoreIndexPath = core.getDataDir() + restoreIndexName;
Directory restoreIndexDir = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31a28f3d/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31a28f3d/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31a28f3d/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 053c417,0c25e09..f01bede
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@@ -410,84 -527,16 +527,92 @@@ public abstract class CollectionAdminRe
}
}
+ // BACKUP request
+ public static class Backup extends AsyncCollectionAdminRequest<Backup> {
+ protected String location;
+ protected String name;
+ protected String collection;
+
+ public String getLocation() {
+ return location;
+ }
+
+ public Backup setLocation(String location) {
+ this.location = location;
+ return this;
+ }
+
+ public Backup(String name, String collection) {
+ this.name = name;
+ this.collection = collection;
+ action = CollectionAction.BACKUP;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set(CoreAdminParams.COLLECTION, collection);
+ params.set(CoreAdminParams.NAME, name);
+ if (location != null) {
+ params.set("location", location);
+ }
+ return params;
+ }
+
+ @Override
+ protected Backup getThis() {
+ return this;
+ }
+ }
+
+ // RESTORE request
+ public static class Restore extends AsyncCollectionAdminRequest<Restore> {
+ protected String location;
+ protected String name;
+ protected String collection;
+
+ public String getLocation() {
+ return location;
+ }
+
+ public Restore setLocation(String location) {
+ this.location = location;
+ return this;
+ }
+
+ public Restore(String name, String collection) {
+ this.name = name;
+ this.collection = collection;
+ action = CollectionAction.RESTORE;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set(CoreAdminParams.COLLECTION, collection);
+ params.set(CoreAdminParams.NAME, name);
+ if (location != null) {
+ params.set("location", location);
+ }
+ return params;
+ }
+
+ @Override
+ protected Restore getThis() {
+ return this;
+ }
+ }
+
+ /**
+ * Returns a SolrRequest to create a new shard in a collection
+ */
+ public static CreateShard createShard(String collection, String shard) {
+ return new CreateShard(collection, shard);
+ }
+
// CREATESHARD request
- public static class CreateShard extends CollectionShardAsyncAdminRequest<CreateShard> {
+ public static class CreateShard extends AsyncShardSpecificAdminRequest {
+
protected String nodeSet;
protected Properties properties;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31a28f3d/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
[3/3] lucene-solr:solr-5750: SOLR-5750: test passes but more
nocommits. Incl fixes in Overseer & ClusterStateMutator
Posted by ds...@apache.org.
SOLR-5750: test passes but more nocommits. Incl fixes in Overseer & ClusterStateMutator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fd9c4d59
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fd9c4d59
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fd9c4d59
Branch: refs/heads/solr-5750
Commit: fd9c4d59e8dbe0e9fbd99a40ed2ff746c1ae7a0c
Parents: 31a28f3
Author: David Smiley <ds...@apache.org>
Authored: Fri Mar 18 21:29:12 2016 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Fri Mar 18 21:29:12 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/solr/cloud/Overseer.java | 17 +-
.../cloud/OverseerCollectionMessageHandler.java | 192 +++++++++----------
.../cloud/overseer/ClusterStateMutator.java | 2 +-
.../solr/cloud/TestCloudBackupRestore.java | 91 ++++-----
.../solrj/request/CollectionAdminRequest.java | 82 ++++----
.../apache/solr/common/cloud/ZkStateReader.java | 4 +-
.../solr/cloud/AbstractDistribZkTestBase.java | 2 +-
.../cloud/AbstractFullDistribZkTestBase.java | 2 +-
8 files changed, 199 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index ec701c3..7244514 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -140,6 +140,7 @@ public class Overseer implements Closeable {
try {
ZkStateWriter zkStateWriter = null;
ClusterState clusterState = null;
+ //nocommit: can't this simply be the same as clusterState being null?
boolean refreshClusterState = true; // let's refresh in the first iteration
while (!this.isClosed) {
isLeader = amILeader();
@@ -155,6 +156,7 @@ public class Overseer implements Closeable {
try {
reader.updateClusterState();
clusterState = reader.getClusterState();
+ assert clusterState != null : "should clusterState be null?";//nocommit
zkStateWriter = new ZkStateWriter(reader, stats);
refreshClusterState = false;
@@ -169,12 +171,14 @@ public class Overseer implements Closeable {
// force flush to ZK after each message because there is no fallback if workQueue items
// are removed from workQueue but fail to be written to ZK
clusterState = processQueueItem(message, clusterState, zkStateWriter, false, null);
+ assert clusterState != null : "should clusterState be null?";//nocommit
workQueue.poll(); // poll-ing removes the element we got by peek-ing
data = workQueue.peek();
}
// force flush at the end of the loop
if (hadWorkItems) {
clusterState = zkStateWriter.writePendingUpdates();
+ assert clusterState != null : "should clusterState be null?";//nocommit
}
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
@@ -188,6 +192,8 @@ public class Overseer implements Closeable {
} catch (Exception e) {
log.error("Exception in Overseer work queue loop", e);
}
+ } else {
+ assert clusterState != null : "should clusterState be null?";//nocommit
}
byte[] head = null;
@@ -225,6 +231,7 @@ public class Overseer implements Closeable {
while (workQueue.poll() != null);
}
});
+ assert clusterState != null : "should clusterState be null?";//nocommit
// it is safer to keep this poll here because an invalid message might never be queued
// and therefore we can't rely on the ZkWriteCallback to remove the item
@@ -236,7 +243,13 @@ public class Overseer implements Closeable {
}
// we should force write all pending updates because the next iteration might sleep until there
// are more items in the main queue
- clusterState = zkStateWriter.writePendingUpdates();
+
+ ClusterState clusterState2 = zkStateWriter.writePendingUpdates();//note: may return null?
+ if (clusterState2 == null) {
+ log.error("zkStateWriter.writePendingUpdates returned null clusterState");
+ } else {
+ clusterState = clusterState2;
+ }
// clean work queue
while (workQueue.poll() != null);
@@ -369,7 +382,7 @@ public class Overseer implements Closeable {
return Collections.singletonList(new CollectionMutator(reader).modifyCollection(clusterState, message));
case MIGRATESTATEFORMAT:
return Collections.singletonList(new ClusterStateMutator(reader).migrateStateFormat(clusterState, message));
- case RESTORE:
+ case RESTORE: // nocommit needs explaination
break;
case BACKUP:
break;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 9c79470..8a35b6d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -17,8 +17,10 @@
package org.apache.solr.cloud;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.lang.invoke.MethodHandles;
@@ -50,6 +52,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
@@ -335,46 +338,27 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
String location = message.getStr("location");
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
final String asyncId = message.getStr(ASYNC);
- Map<String, String> requestMap = null;
- if (asyncId != null) {
- requestMap = new HashMap<>();
- }
- Path backupPath = Paths.get(location).resolve(name).toAbsolutePath();
+ Map<String, String> requestMap = new HashMap<>();
- if (!Files.exists(backupPath)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Backup directory does not exist: " + backupPath.toString());
- }
+ Path backupPath = Paths.get(location).resolve(name).toAbsolutePath();
- Path zkBackup = backupPath.resolve("zk_backup");
- if (!Files.exists(zkBackup)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Backup zk directory does not exist: " + backupPath.toString());
- }
+ Path backupZkPath = backupPath.resolve("zk_backup");
- Path propertiesPath = zkBackup.resolve("backup.properties");
- if (!Files.exists(propertiesPath)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "backup.properties file does not exist: " + backupPath.toString());
- }
+ Path backupZkPropsPath = backupZkPath.resolve("backup.properties");
- FileInputStream in = null;
- Properties properties = null;
- try {
- in = new FileInputStream(propertiesPath.toAbsolutePath().toString());
- properties = new Properties();
+ Properties properties = new Properties();
+ try (InputStream in = Files.newInputStream(backupZkPropsPath)) {
properties.load(new InputStreamReader(in, StandardCharsets.UTF_8));
} catch (IOException e) {
String errorMsg = String.format(Locale.ROOT, "Could not load properties from %s: %s:",
- propertiesPath.toAbsolutePath().toString(), e.toString());
+ backupZkPropsPath.toAbsolutePath().toString(), e.toString());
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, errorMsg);
- } finally {
- IOUtils.closeQuietly(in);
}
String backupCollection = (String) properties.get("collectionName");
- Path collectionStatePath = zkBackup.resolve("collection_state_backup.json");
+ Path collectionStatePath = backupZkPath.resolve("collection_state_backup.json");
byte[] data = Files.readAllBytes(collectionStatePath);
+ @SuppressWarnings("unchecked")
Map<String, Object> collectionProps = (Map<String, Object>) ((Map<String, Object>) Utils.fromJSON(data)).get(backupCollection);
//Download the configs
@@ -383,42 +367,46 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
//Use a name such as restore.<restore_name>.<original_config_name>
// in ZK for the configs
String restoreConfigName = "restore." + configName;
- zkStateReader.getConfigManager().uploadConfigDir(zkBackup.resolve("configs").resolve(configName), restoreConfigName);
+ zkStateReader.getConfigManager().uploadConfigDir(backupZkPath.resolve("configs").resolve(configName), restoreConfigName);
log.debug("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, name,
- backupPath.toString());
+ backupPath);
//Create core-less collection
- Map<String, Object> propMap = new HashMap<>();
- propMap.put(NAME, restoreCollectionName);
- propMap.put(CREATE_NODE_SET, CREATE_NODE_SET_EMPTY); //no cores
- propMap.put("collection.configName", restoreConfigName);
- // add async param
- if (asyncId != null) {
- propMap.put(ASYNC, asyncId);
- }
- String router = (String) ((Map)collectionProps.get("router")).get("name");
- propMap.put("router.name", router);
- Map slices = (Map) collectionProps.get(SHARDS_PROP);
- if (slices != null) { //Implicit routers may not have a shards defined
- propMap.put(NUM_SLICES, slices.size());
- }
- if (ImplicitDocRouter.NAME.equals(router)) {
- Iterator keys = ((Map) collectionProps.get(SHARDS_PROP)).keySet().iterator();
- StringBuilder shardsBuilder = new StringBuilder();
- while (keys.hasNext()) {
- String shard = (String) keys.next();
- shardsBuilder.append(shard);
- shardsBuilder.append(",");
+ {
+ Map<String, Object> propMap = new HashMap<>();
+ propMap.put(NAME, restoreCollectionName);
+ propMap.put(CREATE_NODE_SET, CREATE_NODE_SET_EMPTY); //no cores
+ propMap.put("collection.configName", restoreConfigName);
+ // add async param
+ if (asyncId != null) {
+ propMap.put(ASYNC, asyncId);
}
- String shards = shardsBuilder.deleteCharAt(shardsBuilder.length()-1).toString();
- propMap.put(SHARDS_PROP, shards);
- }
- propMap.put(MAX_SHARDS_PER_NODE, Integer.parseInt((String) collectionProps.get(MAX_SHARDS_PER_NODE)));
- propMap.put(ZkStateReader.AUTO_ADD_REPLICAS, Boolean.parseBoolean((String) collectionProps.get(ZkStateReader.AUTO_ADD_REPLICAS)));
- propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString());
+ String router = (String) ((Map)collectionProps.get("router")).get("name");
+ propMap.put("router.name", router);
+ Map slices = (Map) collectionProps.get(SHARDS_PROP);
+ if (slices != null) { //Implicit routers may not have a shards defined
+ propMap.put(NUM_SLICES, slices.size());
+ }
+ if (ImplicitDocRouter.NAME.equals(router)) {
+ Iterator keys = ((Map) collectionProps.get(SHARDS_PROP)).keySet().iterator();
+ StringBuilder shardsBuilder = new StringBuilder();
+ while (keys.hasNext()) {
+ String shard = (String) keys.next();
+ shardsBuilder.append(shard);
+ shardsBuilder.append(",");
+ }
+ String shards = shardsBuilder.deleteCharAt(shardsBuilder.length()-1).toString();//delete trailing comma
+ propMap.put(SHARDS_PROP, shards);
+ }
+ // TODO nocommit loop all from a list and blacklist those we know could never work.
+ // The user could always edit the properties file if they need to.
+ propMap.put(MAX_SHARDS_PER_NODE, Integer.parseInt((String) collectionProps.get(MAX_SHARDS_PER_NODE)));
+ propMap.put(ZkStateReader.AUTO_ADD_REPLICAS, Boolean.parseBoolean((String) collectionProps.get(ZkStateReader.AUTO_ADD_REPLICAS)));
+ propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString());
- createCollection(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList());
+ createCollection(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList());
+ }
//No need to wait. CreateCollection takes care of it by calling waitToSeeReplicasInState()
DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
@@ -426,21 +414,24 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
throw new SolrException(ErrorCode.SERVER_ERROR, "Could not create restore collection");
}
- //Mark all shards in CONSTRUCTION STATE while we restore the data
DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient());
- propMap = new HashMap<>();
- propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
- for (Slice shard : restoreCollection.getSlices()) {
- propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString());
+
+ //Mark all shards in CONSTRUCTION STATE while we restore the data
+ {
+ Map<String, Object> propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
+ for (Slice shard : restoreCollection.getSlices()) {
+ propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString());
+ }
+ propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
+ inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
}
- propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
- inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
ClusterState clusterState = zkStateReader.getClusterState();
//Create one replica per shard and copy backed up data to it
for (Slice slice: restoreCollection.getSlices()) {
log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
- propMap = new HashMap<>();
+ HashMap<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower());
propMap.put(COLLECTION_PROP, restoreCollectionName);
propMap.put(SHARD_ID_PROP, slice.getName());
@@ -465,50 +456,55 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
processResponses(new NamedList(), shardHandler, true, "Could not restore core", asyncId, requestMap);
//Mark all shards in ACTIVE STATE
- propMap = new HashMap<>();
- propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
- propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
- for (Slice shard : restoreCollection.getSlices()) {
- propMap.put(shard.getName(), Slice.State.ACTIVE.toString());
+
+ {
+ HashMap<String, Object> propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
+ propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
+ for (Slice shard : restoreCollection.getSlices()) {
+ propMap.put(shard.getName(), Slice.State.ACTIVE.toString());
+ }
+ inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
}
- inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
//refresh the location copy of collection state
restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
//Update the replicationFactor to be 1 as that's what it is currently. Otherwise addreplica assigns wrong core names
- propMap = new HashMap<>();
- propMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower());
- propMap.put(COLLECTION_PROP, restoreCollectionName);
- propMap.put(REPLICATION_FACTOR, 1);
- inQueue.offer(Utils.toJSON(message));
+ {
+ HashMap<String, Object> propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower());
+ propMap.put(COLLECTION_PROP, restoreCollectionName);
+ propMap.put(REPLICATION_FACTOR, 1);
+ inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
+ }
//Add the remaining replicas for each shard
int numReplicas = Integer.parseInt((String) collectionProps.get(REPLICATION_FACTOR));
- for (Slice slice: restoreCollection.getSlices()) {
- for(int i=1; i<numReplicas; i++) {
- log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
- propMap = new HashMap<>();
- propMap.put(COLLECTION_PROP, restoreCollectionName);
- propMap.put(SHARD_ID_PROP, slice.getName());
- // add async param
- if (asyncId != null) {
- propMap.put(ASYNC, asyncId);
+ if (numReplicas > 1) {
+ for (Slice slice: restoreCollection.getSlices()) {
+ for(int i=1; i<numReplicas; i++) {
+ log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
+ HashMap<String, Object> propMap = new HashMap<>();
+ propMap.put(COLLECTION_PROP, restoreCollectionName);
+ propMap.put(SHARD_ID_PROP, slice.getName());
+ // add async param
+ if (asyncId != null) {
+ propMap.put(ASYNC, asyncId);
+ }
+ addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results);
}
- addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results);
}
- }
- if (numReplicas > 1) {
//Update the replicationFactor property in cluster state for this collection
log.info("Modifying replication factor to the expected value of={}", numReplicas);
- propMap = new HashMap<>();
+ HashMap<String, Object>propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower());
propMap.put(COLLECTION_PROP, restoreCollectionName);
propMap.put(REPLICATION_FACTOR, numReplicas);
- inQueue.offer(Utils.toJSON(message));
+ inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
+ //nocommit do we need to wait for the result to be done before returning?
}
-
}
private void processBackupAction(ZkNodeProps message, NamedList results) throws IOException, KeeperException, InterruptedException {
@@ -527,11 +523,11 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
//Validating if the directory already exists.
if (Files.exists(backupPath)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Backup directory already exists: " + backupPath.toString());
+ "Backup directory already exists: " + backupPath);
}
log.debug("Starting backup of collection={} with backup_name={} at location={}", collectionName, name,
- backupPath.toString());
+ backupPath);
for (Slice slice : zkStateReader.getClusterState().getActiveSlices(collectionName)) {
Replica replica = slice.getLeader();
@@ -575,16 +571,14 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
//TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same.
//if they are not the same then we can throw and error or have a 'overwriteConfig' flag
//TODO save numDocs for the shardLeader. We can use it to sanity check the restore.
- OutputStreamWriter os = new OutputStreamWriter(new FileOutputStream(propertiesPath.toAbsolutePath().toString()), StandardCharsets.UTF_8);
- try {
+ try (OutputStreamWriter os = new OutputStreamWriter(
+ new FileOutputStream(propertiesPath.toAbsolutePath().toString()), StandardCharsets.UTF_8)) {
properties.store(os, "Snapshot properties file");
} catch (IOException e) {
String errorMsg = String.format(Locale.ROOT, "Could not write properties to %s: %s:",
propertiesPath.toAbsolutePath().toString(), e.toString());
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, errorMsg);
- } finally {
- IOUtils.closeQuietly(os);
+ throw new SolrException(ErrorCode.SERVER_ERROR, errorMsg);
}
log.debug("Completed backing up ZK data for backup={}", name);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
index 7ffa8c1..6864742 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
@@ -101,7 +101,7 @@ public class ClusterStateMutator {
collectionProps.put("autoCreated", "true");
}
- String znode = message.getInt(DocCollection.STATE_FORMAT, 1) == 1 ? null
+ String znode = message.getInt(DocCollection.STATE_FORMAT, 2) == 1 ? null
: ZkStateReader.getCollectionPath(cName);
DocCollection newCollection = new DocCollection(cName,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java
index a2a01a7..7dbefda 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudBackupRestore.java
@@ -17,49 +17,32 @@
package org.apache.solr.cloud;
-import java.io.File;
-
import org.apache.lucene.util.TestUtil;
-import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.util.NamedList;
-import org.junit.After;
-import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.ShardParams._ROUTE_;
-public class TestCloudBackupRestore extends SolrTestCaseJ4 {
-
- static Logger log = LoggerFactory.getLogger(TestCloudBackupRestore.class);
+public class TestCloudBackupRestore extends SolrCloudTestCase {
- private MiniSolrCloudCluster solrCluster;
+ private static Logger log = LoggerFactory.getLogger(TestCloudBackupRestore.class);
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- solrCluster = new MiniSolrCloudCluster(2, createTempDir(), buildJettyConfig("/solr"));
- final File configDir = getFile("solr").toPath().resolve("collection1/conf").toFile();
- solrCluster.uploadConfigDir(configDir, "conf1");
- System.setProperty("solr.test.sys.prop1", "propone");
- System.setProperty("solr.test.sys.prop2", "proptwo");
- }
+ private static final int NUM_SHARDS = 2;
- @Override
- @After
- public void tearDown() throws Exception {
- solrCluster.shutdown();
- super.tearDown();
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .configure();
}
@Test
@@ -67,28 +50,26 @@ public class TestCloudBackupRestore extends SolrTestCaseJ4 {
String collectionName = "backuprestore";
String restoreCollectionName = collectionName + "_restored";
boolean isImplicit = random().nextBoolean();
- CollectionAdminRequest.Create create = new CollectionAdminRequest.Create()
- .setCollectionName(collectionName)
- .setConfigName("conf1")
- .setReplicationFactor(TestUtil.nextInt(random(), 1, 2))
- .setMaxShardsPerNode(2);
+ int numReplicas = TestUtil.nextInt(random(), 1, 2);
+ CollectionAdminRequest.Create create =
+ CollectionAdminRequest.createCollection(collectionName, "conf1", NUM_SHARDS, numReplicas);
+ create.setMaxShardsPerNode(NUM_SHARDS);
if (isImplicit) { //implicit router
create.setRouterName(ImplicitDocRouter.NAME);
- create.setShards("shard1,shard2");
+ create.setNumShards(null);//erase it
+ create.setShards("shard1,shard2"); // however still same number as NUM_SHARDS; we assume this later
create.setRouterField("shard_s");
- } else {
- create.setNumShards(2);
}
-
- create.process(solrCluster.getSolrClient());
- AbstractDistribZkTestBase.waitForRecoveriesToFinish("backuprestore", solrCluster.getSolrClient().getZkStateReader(), false, true, 30);
+//TODO nocommit test shard split & custom doc route?
+ create.process(cluster.getSolrClient());
+ waitForCollection(collectionName);
indexDocs(collectionName);
testBackupAndRestore(collectionName, restoreCollectionName, isImplicit);
}
private void indexDocs(String collectionName) throws Exception {
int numDocs = TestUtil.nextInt(random(), 10, 100);
- CloudSolrClient client = solrCluster.getSolrClient();
+ CloudSolrClient client = cluster.getSolrClient();
client.setDefaultCollection(collectionName);
for (int i=0; i<numDocs; i++) {
//We index the shard_s fields for whichever router gets chosen but only use it when implicit router was selected
@@ -109,7 +90,7 @@ public class TestCloudBackupRestore extends SolrTestCaseJ4 {
private void testBackupAndRestore(String collectionName, String restoreCollectionName, boolean isImplicit) throws Exception {
String backupName = "mytestbackup";
- CloudSolrClient client = solrCluster.getSolrClient();
+ CloudSolrClient client = cluster.getSolrClient();
long totalDocs = client.query(collectionName, new SolrQuery("*:*")).getResults().getNumFound();
long shard1Docs = 0, shard2Docs = 0;
if (isImplicit) {
@@ -121,29 +102,19 @@ public class TestCloudBackupRestore extends SolrTestCaseJ4 {
String location = createTempDir().toFile().getAbsolutePath();
log.info("Triggering Backup command");
- //Run backup command
- CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(backupName, collectionName)
+
+ CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
.setLocation(location);
- NamedList<Object> rsp = solrCluster.getSolrClient().request(backup);
+ NamedList<Object> rsp = cluster.getSolrClient().request(backup);
assertEquals(0, ((NamedList)rsp.get("responseHeader")).get("status"));
log.info("Triggering Restore command");
- //Restore
- CollectionAdminRequest.Restore restore = new CollectionAdminRequest.Restore(backupName, restoreCollectionName)
+ CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
.setLocation(location);
- rsp = solrCluster.getSolrClient().request(restore);
+ rsp = cluster.getSolrClient().request(restore);
assertEquals(0, ((NamedList)rsp.get("responseHeader")).get("status"));
-
- client.getZkStateReader().updateClusterState();
- DocCollection restoreCollection = null;
- while (restoreCollection == null) {
- try {
- restoreCollection = client.getZkStateReader().getClusterState().getCollection(restoreCollectionName);
- } catch (SolrException e) {
- Thread.sleep(100); //wait for cluster state to update
- }
- }
+ waitForCollection(restoreCollectionName);
//Check the number of results are the same
long restoredNumDocs = client.query(restoreCollectionName, new SolrQuery("*:*")).getResults().getNumFound();
@@ -158,9 +129,17 @@ public class TestCloudBackupRestore extends SolrTestCaseJ4 {
}
DocCollection backupCollection = client.getZkStateReader().getClusterState().getCollection(collectionName);
+ DocCollection restoreCollection = client.getZkStateReader().getClusterState().getCollection(restoreCollectionName);
+
assertEquals(backupCollection.getReplicationFactor(), restoreCollection.getReplicationFactor());
- assertEquals( "restore.conf1", solrCluster.getSolrClient().getZkStateReader().readConfigName(restoreCollectionName));
+ assertEquals("restore.conf1", cluster.getSolrClient().getZkStateReader().readConfigName(restoreCollectionName));
+ }
+
+ public void waitForCollection(String collection) throws Exception {
+ AbstractFullDistribZkTestBase.waitForCollection(cluster.getSolrClient().getZkStateReader(), collection, NUM_SHARDS);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collection, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30);
+ AbstractDistribZkTestBase.assertAllActive(collection, cluster.getSolrClient().getZkStateReader());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index f01bede..c18900b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -527,11 +527,33 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
}
+ public static Backup backupCollection(String collection, String backupName) {
+ return new Backup(collection, backupName);
+ }
+
// BACKUP request
- public static class Backup extends AsyncCollectionAdminRequest<Backup> {
+ public static class Backup extends AsyncCollectionSpecificAdminRequest {
+ protected final String name;
protected String location;
- protected String name;
- protected String collection;
+
+ public Backup(String collection, String name) {
+ super(CollectionAction.BACKUP, collection);
+ this.name = name;
+ }
+
+ @Override
+ @Deprecated
+ public Backup setAsyncId(String id) {
+ this.asyncId = id;
+ return this;
+ }
+
+ @Override
+ @Deprecated
+ public Backup setCollectionName(String collection) {
+ this.collection = collection;
+ return this;
+ }
public String getLocation() {
return location;
@@ -542,34 +564,42 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
- public Backup(String name, String collection) {
- this.name = name;
- this.collection = collection;
- action = CollectionAction.BACKUP;
- }
-
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set(CoreAdminParams.COLLECTION, collection);
params.set(CoreAdminParams.NAME, name);
- if (location != null) {
- params.set("location", location);
- }
+ params.set("location", location); //note: optional
return params;
}
- @Override
- protected Backup getThis() {
- return this;
- }
+ }
+
+ public static Restore restoreCollection(String collection, String backupName) {
+ return new Restore(collection, backupName);
}
// RESTORE request
- public static class Restore extends AsyncCollectionAdminRequest<Restore> {
+ public static class Restore extends AsyncCollectionSpecificAdminRequest {
+ protected final String name;
protected String location;
- protected String name;
- protected String collection;
+
+ public Restore(String collection, String name) {
+ super(CollectionAction.RESTORE, collection);
+ this.name = name;
+ }
+
+ @Override
+ public Restore setAsyncId(String id) {
+ this.asyncId = id;
+ return this;
+ }
+
+ @Override
+ public Restore setCollectionName(String collection) {
+ this.collection = collection;
+ return this;
+ }
public String getLocation() {
return location;
@@ -580,27 +610,15 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
- public Restore(String name, String collection) {
- this.name = name;
- this.collection = collection;
- action = CollectionAction.RESTORE;
- }
-
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set(CoreAdminParams.COLLECTION, collection);
params.set(CoreAdminParams.NAME, name);
- if (location != null) {
- params.set("location", location);
- }
+ params.set("location", location); //note: optional
return params;
}
- @Override
- protected Restore getThis() {
- return this;
- }
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 37c275b..a001af0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -97,6 +97,8 @@ public class ZkStateReader implements Closeable {
public static final String URL_SCHEME = "urlScheme";
+ public static final String BACKUP_LOCATION = "location";
+
/** A view of the current state of all collections; combines all the different state sources into a single view. */
protected volatile ClusterState clusterState;
@@ -134,7 +136,7 @@ public class ZkStateReader implements Closeable {
LEGACY_CLOUD,
URL_SCHEME,
AUTO_ADD_REPLICAS,
- "location")));
+ BACKUP_LOCATION)));
/**
* Returns config set name for collection.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
index 7b3617b..4ac6d5a 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
@@ -234,7 +234,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
fail("Illegal state, was: " + coreState + " expected:" + expectedState + " clusterState:" + reader.getClusterState());
}
- protected void assertAllActive(String collection,ZkStateReader zkStateReader)
+ protected static void assertAllActive(String collection, ZkStateReader zkStateReader)
throws KeeperException, InterruptedException {
zkStateReader.forceUpdateCollection(collection);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd9c4d59/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index a584dbd..347d0f7 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -336,7 +336,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
}
- protected void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception {
+ protected static void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception {
// wait until shards have started registering...
int cnt = 30;
while (!reader.getClusterState().hasCollection(collection)) {