You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2015/02/05 11:37:26 UTC
svn commit: r1657523 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/handler/
core/src/test-files/solr/collection1/conf/
core/src/test/org/apache/solr/cloud/ core/src/test/org/apache/solr/handler/
Author: shalin
Date: Thu Feb 5 10:37:25 2015
New Revision: 1657523
URL: http://svn.apache.org/r1657523
Log:
SOLR-6775: Creating backup snapshot results in null pointer exception
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1657523&r1=1657522&r2=1657523&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Feb 5 10:37:25 2015
@@ -105,6 +105,9 @@ Bug Fixes
* SOLR-7072: Multiple mlt.fl does not work. (Constantin Mitocaru, shalin)
+* SOLR-6775: Creating backup snapshot results in null pointer exception.
+ (Ryan Hesson, Varun Thacker via shalin)
+
Optimizations
----------------------
* SOLR-7049: Move work done by the LIST Collections API call to the Collections
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1657523&r1=1657522&r2=1657523&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Thu Feb 5 10:37:25 2015
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
-import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
@@ -54,7 +53,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RateLimiter;
-import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@@ -72,7 +70,6 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrDeletionPolicy;
import org.apache.solr.core.SolrEventListener;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.BinaryQueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.SolrIndexWriter;
@@ -1058,6 +1055,7 @@ public class ReplicationHandler extends
numberToKeep = Integer.MAX_VALUE;
}
SnapShooter snapShooter = new SnapShooter(core, null, null);
+ snapShooter.validateCreateSnapshot();
snapShooter.createSnapAsync(currentCommitPoint, numberToKeep, ReplicationHandler.this);
} catch (Exception e) {
LOG.error("Exception while snapshooting", e);
Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml?rev=1657523&r1=1657522&r2=1657523&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml (original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1-keepOneBackup.xml Thu Feb 5 10:37:25 2015
@@ -32,6 +32,7 @@
<lst name="master">
<str name="replicateAfter">commit</str>
<str name="confFiles">schema-replication2.xml:schema.xml</str>
+ <str name="backupAfter">commit</str>
</lst>
<str name="maxNumberOfBackups">1</str>
</requestHandler>
Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml?rev=1657523&r1=1657522&r2=1657523&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml (original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-master1.xml Thu Feb 5 10:37:25 2015
@@ -34,6 +34,7 @@
<requestHandler name="/replication" class="solr.ReplicationHandler">
<lst name="master">
<str name="replicateAfter">commit</str>
+ <str name="backupAfter">commit</str>
<str name="confFiles">schema-replication2.xml:schema.xml</str>
</lst>
</requestHandler>
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1657523&r1=1657522&r2=1657523&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Thu Feb 5 10:37:25 2015
@@ -17,14 +17,18 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.BasicResponseHandler;
-import org.apache.lucene.util.IOUtils;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.Lists;
+import org.apache.lucene.mockfile.FilterPath;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
import org.apache.solr.client.solrj.request.QueryRequest;
@@ -36,17 +40,9 @@ import org.apache.solr.common.cloud.ZkNo
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.handler.ReplicationHandler;
+import org.apache.solr.handler.CheckBackupStatus;
import org.junit.Test;
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
/**
* This test simply does a bunch of basic things in solrcloud mode and asserts things
* work as expected.
@@ -405,84 +401,29 @@ public class BasicDistributedZk2Test ext
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/replication");
params.set("command", "backup");
- File location = createTempDir().toFile();
- params.set("location", location.getAbsolutePath());
+ Path location = createTempDir();
+ location = FilterPath.unwrap(location).toRealPath();
+ params.set("location", location.toString());
QueryRequest request = new QueryRequest(params);
- NamedList<Object> results = client.request(request );
+ client.request(request);
checkForBackupSuccess(client, location);
}
- private void checkForBackupSuccess(final HttpSolrClient client, File location)
- throws InterruptedException, IOException {
- class CheckStatus extends Thread {
- volatile String fail = null;
- volatile String response = null;
- volatile boolean success = false;
- final Pattern p = Pattern
- .compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
-
- CheckStatus() {}
-
- @Override
- public void run() {
- String masterUrl = client.getBaseURL() + "/replication?command="
- + ReplicationHandler.CMD_DETAILS;
-
- try {
- response = client.getHttpClient().execute(new HttpGet(masterUrl), new BasicResponseHandler());
- if (response.contains("<str name=\"status\">success</str>")) {
- Matcher m = p.matcher(response);
- if (!m.find()) {
- fail("could not find the completed timestamp in response.");
- }
-
- success = true;
- }
- } catch (Exception e) {
- e.printStackTrace();
- fail = e.getMessage();
- }
-
- };
+ private void checkForBackupSuccess(HttpSolrClient client, Path location) throws InterruptedException, IOException {
+ CheckBackupStatus checkBackupStatus = new CheckBackupStatus(client);
+ while (!checkBackupStatus.success) {
+ checkBackupStatus.fetchStatus();
+ Thread.sleep(1000);
}
-
- int waitCnt = 0;
- CheckStatus checkStatus = new CheckStatus();
- while (true) {
- checkStatus.run();
- if (checkStatus.fail != null) {
- fail(checkStatus.fail);
- }
- if (checkStatus.success) {
- break;
- }
- Thread.sleep(500);
- if (waitCnt == 90) {
- fail("Backup success not detected:" + checkStatus.response);
- }
- waitCnt++;
- }
-
- File[] files = location.listFiles(new FilenameFilter() {
-
- @Override
- public boolean accept(File dir, String name) {
- if (name.startsWith("snapshot")) {
- return true;
- }
- return false;
- }
- });
- assertEquals(Arrays.asList(files).toString(), 1, files.length);
- File snapDir = files[0];
-
- IOUtils.rm(snapDir.toPath());
+ ArrayList<Path> files = Lists.newArrayList(Files.newDirectoryStream(location, "snapshot*").iterator());
+
+ assertEquals(Arrays.asList(files).toString(), 1, files.size());
+
}
private void addNewReplica() throws Exception {
- JettySolrRunner newReplica = createJettys(1).get(0);
waitForRecoveriesToFinish(false);
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java?rev=1657523&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java Thu Feb 5 10:37:25 2015
@@ -0,0 +1,68 @@
+package org.apache.solr.handler;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+
+public class CheckBackupStatus extends SolrTestCaseJ4 {
+ String response = null;
+ public boolean success = false;
+ String backupTimestamp = null;
+ final String lastBackupTimestamp;
+ final Pattern p = Pattern.compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
+ final Pattern pException = Pattern.compile("<str name=\"snapShootException\">(.*?)</str>");
+ final HttpSolrClient client;
+
+ public CheckBackupStatus(final HttpSolrClient client, String lastBackupTimestamp) {
+ this.client = client;
+ this.lastBackupTimestamp = lastBackupTimestamp;
+ }
+
+ public CheckBackupStatus(final HttpSolrClient client) {
+ this(client, null);
+ }
+
+ public void fetchStatus() throws IOException {
+ String masterUrl = client.getBaseURL() + "/replication?command=" + ReplicationHandler.CMD_DETAILS;
+ response = client.getHttpClient().execute(new HttpGet(masterUrl), new BasicResponseHandler());
+ if(pException.matcher(response).find()) {
+ fail("Failed to create backup");
+ }
+ if(response.contains("<str name=\"status\">success</str>")) {
+ Matcher m = p.matcher(response);
+ if(!m.find()) {
+ fail("could not find the completed timestamp in response.");
+ }
+ if (lastBackupTimestamp != null) {
+ backupTimestamp = m.group(1);
+ if (backupTimestamp.equals(lastBackupTimestamp)) {
+ success = true;
+ }
+ } else {
+ success = true;
+ }
+ }
+ }
+}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java?rev=1657523&r1=1657522&r2=1657523&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java Thu Feb 5 10:37:25 2015
@@ -17,6 +17,16 @@ package org.apache.solr.handler;
* limitations under the License.
*/
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -29,6 +39,7 @@ import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.SolrInputDocument;
@@ -37,15 +48,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.file.Path;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
public class TestReplicationHandlerBackup extends SolrJettyTestBase {
@@ -53,8 +55,7 @@ public class TestReplicationHandlerBacku
TestReplicationHandler.SolrInstance master = null;
SolrClient masterClient;
- private static final String CONF_DIR = "solr"
- + File.separator + "collection1" + File.separator + "conf"
+ private static final String CONF_DIR = "solr" + File.separator + "collection1" + File.separator + "conf"
+ File.separator;
private static String context = "/solr";
@@ -116,10 +117,35 @@ public class TestReplicationHandlerBacku
master = null;
}
-
@Test
- public void doTestBackup() throws Exception {
+ public void testBackupOnCommit() throws Exception {
+ //Index
+ int nDocs = indexDocs();
+
+ //Confirm if completed
+ CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient);
+ while (!checkBackupStatus.success) {
+ checkBackupStatus.fetchStatus();
+ Thread.sleep(1000);
+ }
+
+ //Validate
+ Path snapDir = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot*").iterator().next();
+ verify(snapDir, nDocs);
+ }
+ private void verify(Path backup, int nDocs) throws IOException {
+ try (Directory dir = new SimpleFSDirectory(backup)) {
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
+ assertEquals(nDocs, hits.totalHits);
+ reader.close();
+ dir.close();
+ }
+ }
+
+ private int indexDocs() throws IOException, SolrServerException {
int nDocs = TestUtil.nextInt(random(), 1, 100);
masterClient.deleteByQuery("*:*");
for (int i = 0; i < nDocs; i++) {
@@ -130,8 +156,16 @@ public class TestReplicationHandlerBacku
}
masterClient.commit();
+ return nDocs;
+ }
+
+
+ @Test
+ public void doTestBackup() throws Exception {
+
+ int nDocs = indexDocs();
- File[] snapDir = new File[2];
+ Path[] snapDir = new Path[2];
boolean namedBackup = random().nextBoolean();
try {
String firstBackupTimestamp = null;
@@ -150,60 +184,29 @@ public class TestReplicationHandlerBacku
backupNames[i] = backupName;
}
backupCommand.runCommand();
+ if (backupCommand.fail != null) {
+ fail(backupCommand.fail);
+ }
- File dataDir = new File(master.getDataDir());
-
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus(firstBackupTimestamp);
- while (true) {
+ CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, firstBackupTimestamp);
+ while (!checkBackupStatus.success) {
checkBackupStatus.fetchStatus();
- if (checkBackupStatus.success) {
- if (i == 0) {
- firstBackupTimestamp = checkBackupStatus.backupTimestamp;
- Thread.sleep(1000); //ensure the next backup will have a different timestamp.
- }
- break;
- }
- Thread.sleep(200);
+ Thread.sleep(1000);
}
-
- if (backupCommand.fail != null) {
- fail(backupCommand.fail);
+ if (i == 0) {
+ firstBackupTimestamp = checkBackupStatus.backupTimestamp;
}
- File[] files = null;
+
if (!namedBackup) {
- files = dataDir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- if (name.startsWith("snapshot")) {
- return true;
- }
- return false;
- }
- });
+ snapDir[i] = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot*").iterator().next();
} else {
- files = dataDir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- if (name.equals("snapshot." + backupName)) {
- return true;
- }
- return false;
- }
- });
+ snapDir[i] = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot." + backupName).iterator().next();
}
- assertEquals(1, files.length);
- snapDir[i] = files[0];
- Directory dir = new SimpleFSDirectory(snapDir[i].getAbsoluteFile().toPath());
- IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
- TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
- assertEquals(nDocs, hits.totalHits);
- reader.close();
- dir.close();
+ verify(snapDir[i], nDocs);
}
- if (!namedBackup && snapDir[0].exists()) {
+ if (!namedBackup && Files.exists(snapDir[0])) {
fail("The first backup should have been cleaned up because " + backupKeepParamName + " was set to 1.");
}
@@ -214,11 +217,9 @@ public class TestReplicationHandlerBacku
} finally {
if(!namedBackup) {
- Path toDelete[] = new Path[snapDir.length];
for (int i = 0; i < snapDir.length; i++) {
- toDelete[i] = snapDir[i].toPath();
+ org.apache.lucene.util.IOUtils.rm(snapDir[i]);
}
- org.apache.lucene.util.IOUtils.rm(toDelete);
}
}
}
@@ -247,47 +248,6 @@ public class TestReplicationHandlerBacku
}
}
- private class CheckBackupStatus {
- String response = null;
- boolean success = false;
- String backupTimestamp = null;
- final String lastBackupTimestamp;
- final Pattern p = Pattern.compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
- final Pattern pException = Pattern.compile("<str name=\"snapShootException\">(.*?)</str>");
-
- CheckBackupStatus(String lastBackupTimestamp) {
- this.lastBackupTimestamp = lastBackupTimestamp;
- }
-
- public void fetchStatus() throws IOException {
- String masterUrl = buildUrl(masterJetty.getLocalPort(), "/solr") + "/" + DEFAULT_TEST_CORENAME + "/replication?command=" + ReplicationHandler.CMD_DETAILS;
- URL url;
- InputStream stream = null;
- try {
- url = new URL(masterUrl);
- stream = url.openStream();
- response = IOUtils.toString(stream, "UTF-8");
- if(pException.matcher(response).find()) {
- fail("Failed to create backup");
- }
- if(response.contains("<str name=\"status\">success</str>")) {
- Matcher m = p.matcher(response);
- if(!m.find()) {
- fail("could not find the completed timestamp in response.");
- }
- backupTimestamp = m.group(1);
- if(!backupTimestamp.equals(lastBackupTimestamp)) {
- success = true;
- }
- }
- stream.close();
- } finally {
- IOUtils.closeQuietly(stream);
- }
-
- };
- }
-
private class BackupCommand {
String fail = null;
final boolean addNumberToKeepInRequest;
@@ -307,7 +267,7 @@ public class TestReplicationHandlerBacku
}
public void runCommand() {
- String masterUrl = null;
+ String masterUrl;
if(backupName != null) {
masterUrl = buildUrl(masterJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME + "/replication?command=" + cmd +
"&name=" + backupName;
@@ -316,10 +276,9 @@ public class TestReplicationHandlerBacku
(addNumberToKeepInRequest ? "&" + backupKeepParamName + "=1" : "");
}
- URL url;
InputStream stream = null;
try {
- url = new URL(masterUrl);
+ URL url = new URL(masterUrl);
stream = url.openStream();
stream.close();
} catch (Exception e) {
@@ -328,7 +287,7 @@ public class TestReplicationHandlerBacku
IOUtils.closeQuietly(stream);
}
- };
+ }
}
private class CheckDeleteBackupStatus {