You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/08/21 01:30:58 UTC
[31/50] [abbrv] lucene-solr:jira/http2: SOLR-12475: Fix failing
MaxSizeAutoCommitTest
SOLR-12475: Fix failing MaxSizeAutoCommitTest
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4309ae6f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4309ae6f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4309ae6f
Branch: refs/heads/jira/http2
Commit: 4309ae6f9fdb70d780ee08bfc32ec3a36e68389a
Parents: 585ba16
Author: Anshum Gupta <an...@apple.com>
Authored: Thu Aug 16 23:03:27 2018 -0700
Committer: Anshum Gupta <an...@apple.com>
Committed: Thu Aug 16 23:05:21 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/update/CommitTracker.java | 13 +-
.../solr/update/MaxSizeAutoCommitTest.java | 268 ++++---------------
3 files changed, 60 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4309ae6f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7f851f4..29a5b52 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -243,6 +243,8 @@ Bug Fixes
* SOLR-12668: Autoscaling trigger listeners should be executed in the order of their creation. (ab)
+* SOLR-12475: Fix MaxSizeAutoCommitTest failures (Rupa Shankar, Anshum Gupta)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4309ae6f/solr/core/src/java/org/apache/solr/update/CommitTracker.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/CommitTracker.java b/solr/core/src/java/org/apache/solr/update/CommitTracker.java
index 8f06d11..7da9651 100644
--- a/solr/core/src/java/org/apache/solr/update/CommitTracker.java
+++ b/solr/core/src/java/org/apache/solr/update/CommitTracker.java
@@ -18,6 +18,7 @@ package org.apache.solr.update;
import java.lang.invoke.MethodHandles;
+import java.util.Locale;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
@@ -282,12 +283,14 @@ public final class CommitTracker implements Runnable {
@Override
public String toString() {
- if (timeUpperBound > 0 || docsUpperBound > 0) {
- return (timeUpperBound > 0 ? ("if uncommited for " + timeUpperBound + "ms; ")
+ if (timeUpperBound > 0 || docsUpperBound > 0 || tLogFileSizeUpperBound > 0) {
+ return (timeUpperBound > 0 ? ("if uncommitted for " + timeUpperBound + "ms; ")
: "")
- + (docsUpperBound > 0 ? ("if " + docsUpperBound + " uncommited docs ")
- : "");
-
+ + (docsUpperBound > 0 ? ("if " + docsUpperBound + " uncommitted docs; ")
+ : "")
+ + (tLogFileSizeUpperBound > 0 ? String.format(Locale.ROOT, "if tlog file size has exceeded %d bytes",
+ tLogFileSizeUpperBound)
+ : "");
} else {
return "disabled";
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4309ae6f/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java b/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java
index 57390d9..33fd7de 100644
--- a/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java
+++ b/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java
@@ -17,25 +17,18 @@
package org.apache.solr.update;
-import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.UpdateRequestHandler;
-import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.response.SolrQueryResponse;
import org.junit.After;
@@ -43,25 +36,22 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-//commented 2-Aug-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18
public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 {
// Given an ID, returns an XML string for an "add document" request
private static final Function<Integer, String> ADD_DOC_FN = (id) -> adoc("id", Integer.toString(id));
// Given an ID, returns an XML string for a "delete document" request
private static final Function<Integer, String> DELETE_DOC_FN = (id) -> delI(Integer.toString(id));
+ // How long to sleep while checking for commits
+ private static final int COMMIT_CHECKING_SLEEP_TIME_MS = 50;
- private ObjectMapper objectMapper; // for JSON parsing
private SolrCore core;
private DirectUpdateHandler2 updateHandler;
private CommitTracker hardCommitTracker;
private UpdateRequestHandler updateRequestHandler;
- private String tlogDirPath;
@Before
public void setup() throws Exception {
- objectMapper = new ObjectMapper();
- System.setProperty("solr.autoCommit.maxSize", "5k");
System.setProperty("solr.ulog", "solr.UpdateLog");
initCore("solrconfig-tlog.xml", "schema.xml");
core = h.getCore();
@@ -72,65 +62,45 @@ public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 {
hardCommitTracker.setDocsUpperBound(-1);
updateRequestHandler = new UpdateRequestHandler();
updateRequestHandler.init( null );
- tlogDirPath = core.getDataDir() + "/tlog";
}
@After
public void tearDown() throws Exception {
super.tearDown();
- System.clearProperty("solr.autoCommit.maxSize");
System.clearProperty("solr.ulog");
deleteCore();
}
@Test
- public void simpleTest() throws Exception {
+ public void testAdds() throws Exception {
int maxFileSizeBound = 1000;
- int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25);
// Set max size bound
hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
- // Adding these docs will place the tlog size just under the threshold
- int numDocs = 27;
- int batchSize = 3;
- int numBatches = numDocs / batchSize;
+ // Add a large number of docs - should trigger a commit
+ int numDocsToAdd = 500;
SolrQueryResponse updateResp = new SolrQueryResponse();
- int numTlogs = -1;
- TreeMap<String, Long> tlogsInfo = null;
- for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) {
- int docStartId = batchSize * batchCounter;
+ Assert.assertEquals("There have been no updates yet, so there shouldn't have been any commits", 0,
+ hardCommitTracker.getCommitCount());
- // Send batch update request
- updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp);
+ long tlogSizePreUpdates = updateHandler.getUpdateLog().getCurrentLogSizeFromStream();
+ Assert.assertEquals("There have been no updates yet, so tlog should be empty", 0, tlogSizePreUpdates);
- // The sleep is to allow existing commits to finish (or at least mostly finish) before querying/submitting more documents
- waitForCommit(200);
-
- // There should just be 1 tlog and its size should be within the (buffered) file size bound
- tlogsInfo = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
- numTlogs = parseTotalNumTlogs(tlogsInfo);
- Assert.assertEquals(1, numTlogs);
- }
-
- // Now that the core's tlog size is just under the threshold, one more update should induce a commit
- int docStartId = batchSize * numBatches;
- updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp);
- waitForCommit(200);
+ updateRequestHandler.handleRequest(constructBatchAddDocRequest(0, numDocsToAdd), updateResp);
- // Verify that a commit happened. There should now be 2 tlogs, both of which are < maxFileSizeBound.
- TreeMap<String, Long> tlogsInfoPostCommit = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
- Assert.assertEquals(2, parseTotalNumTlogs(tlogsInfoPostCommit));
+ // The long sleep is to allow for the triggered commit to finish
+ waitForCommit(1000);
- // And the current tlog's size should be less than the previous tlog's size
- Assert.assertTrue(tlogsInfoPostCommit.lastEntry().getValue() < tlogsInfo.lastEntry().getValue());
+ // Verify commit information
+ Assert.assertTrue("At least one commit should have occurred", hardCommitTracker.getCommitCount() > 0);
+ long tlogSizePostUpdates = updateHandler.getUpdateLog().getCurrentLogSizeFromStream();
+ Assert.assertTrue("Current tlog size is larger than the max bound", tlogSizePostUpdates < maxFileSizeBound);
}
@Test
public void testRedundantDeletes() throws Exception {
int maxFileSizeBound = 1000;
- int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25);
-
// Set max size bound
hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
@@ -138,125 +108,73 @@ public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 {
int numDocsToAdd = 150;
SolrQueryResponse updateResp = new SolrQueryResponse();
updateRequestHandler.handleRequest(constructBatchAddDocRequest(0, numDocsToAdd), updateResp);
- waitForCommit(200);
+ waitForCommit(1000);
- // Get the tlog file info
- TreeMap<String, Long> tlogsInfoPreDeletes = getTlogFileSizes(tlogDirPath);
+ // Get the current commit info
+ int commitCountPreDeletes = hardCommitTracker.getCommitCount();
// Send a bunch of redundant deletes
- int numDeletesToSend = 5000;
+ int numDeletesToSend = 500;
int docIdToDelete = 100;
- SolrQueryRequestBase requestWithOneDelete = new SolrQueryRequestBase(core, new MapSolrParams(new HashMap<String, String>())) {};
+ SolrQueryRequestBase batchSingleDeleteRequest = new SolrQueryRequestBase(core, new MapSolrParams(new HashMap<>())) {};
List<String> docs = new ArrayList<>();
- docs.add(delI(Integer.toString(docIdToDelete)));
-
- requestWithOneDelete.setContentStreams(toContentStreams(docs));
-
for (int i = 0; i < numDeletesToSend; i++) {
- if (i % 50 == 0) {
- // Wait periodically to allow existing commits to finish before
- // sending more delete requests
- waitForCommit(200);
- }
- updateRequestHandler.handleRequest(requestWithOneDelete, updateResp);
+ docs.add(delI(Integer.toString(docIdToDelete)));
}
+ batchSingleDeleteRequest.setContentStreams(toContentStreams(docs));
+
+ updateRequestHandler.handleRequest(batchSingleDeleteRequest, updateResp);
- // Verify that new tlogs have been created, and that their sizes are as expected
- TreeMap<String, Long> tlogsInfoPostDeletes = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
- Assert.assertTrue(parseTotalNumTlogs(tlogsInfoPreDeletes) < parseTotalNumTlogs(tlogsInfoPostDeletes));
+ // The long sleep is to allow for the expected triggered commit to finish
+ waitForCommit(1000);
+
+ // Verify commit information
+ Assert.assertTrue("At least one commit should have occurred",
+ hardCommitTracker.getCommitCount() > commitCountPreDeletes);
+ long tlogSizePostDeletes = updateHandler.getUpdateLog().getCurrentLogSizeFromStream();
+ Assert.assertTrue("Current tlog size is larger than the max bound", tlogSizePostDeletes < maxFileSizeBound);
}
@Test
- //commented 2-Aug-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
- public void deleteTest() throws Exception {
+ public void testDeletes() throws Exception {
int maxFileSizeBound = 1000;
- int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25);
// Set max size bound
hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
// Add docs
- int numDocsToAdd = 150;
+ int numDocsToAdd = 500;
SolrQueryResponse updateResp = new SolrQueryResponse();
updateRequestHandler.handleRequest(constructBatchAddDocRequest(0, numDocsToAdd), updateResp);
- waitForCommit(200);
+ waitForCommit(1000);
- // Get the tlog file info
- TreeMap<String, Long> tlogsInfoPreDeletes = getTlogFileSizes(tlogDirPath);
+ // Get the current commit info
+ int commitCountPreDeletes = hardCommitTracker.getCommitCount();
- // Delete documents (in batches, so we can allow commits to finish and new tlog files to be created)
- int batchSize = 15;
- int numBatches = numDocsToAdd / batchSize;
- for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) {
- int docStartId = batchSize * batchCounter;
-
- // Send batch delete doc request
- updateRequestHandler.handleRequest(constructBatchDeleteDocRequest(docStartId, batchSize), updateResp);
-
- // The sleep is to allow existing commits to finish before deleting more documents
- waitForCommit(200);
- }
-
- // Verify that the commit happened by seeing if a new tlog file was opened
- TreeMap<String, Long> tlogsInfoPostDeletes = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
- Assert.assertTrue(parseTotalNumTlogs(tlogsInfoPreDeletes) < parseTotalNumTlogs(tlogsInfoPostDeletes));
- }
-
- @Test
- @Repeat(iterations = 5)
- //commented 2-Aug-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
- public void endToEndTest() throws Exception {
- int maxFileSizeBound = 5000;
- // Set max size bound
- hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
-
- // Giving a 10% buffer for the max size bound
- int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.1);
-
- SolrQueryRequest selectQuery = req("*:*");
- List<Integer> docCounts = new ArrayList<>();
-
- int numDocs = 1000;
- int batchSize = 20;
- int numBatches = numDocs / batchSize;
- for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) {
- SolrQueryResponse updateResp = new SolrQueryResponse();
- int docStartId = batchSize * batchCounter;
-
- // Send batch add doc request
- updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp);
-
- // The sleep is to allow existing commits to finish before querying/submitting more documents
- waitForCommit(200);
-
- // Check tlog file sizes
- getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
-
- // See how many documents are currently visible. This should increase as more commits occur.
- docCounts.add(queryCore(selectQuery));
- }
+ // Delete all documents - should trigger a commit
+ updateRequestHandler.handleRequest(constructBatchDeleteDocRequest(0, numDocsToAdd), updateResp);
- // One final commit, after which all documents should be visible
- CommitUpdateCommand commitUpdateCommand = new CommitUpdateCommand(req(), false);
- updateHandler.commit(commitUpdateCommand);
- waitForCommit(200);
- docCounts.add(queryCore(selectQuery));
+ // The long sleep is to allow for the expected triggered commit to finish
+ waitForCommit(1000);
- // Evaluate the document counts
- checkNumFoundDocuments(docCounts, numDocs);
+ // Verify commit information
+ Assert.assertTrue("At least one commit should have occurred",
+ hardCommitTracker.getCommitCount() > commitCountPreDeletes);
+ long tlogSizePostDeletes = updateHandler.getUpdateLog().getCurrentLogSizeFromStream();
+ Assert.assertTrue("Current tlog size is larger than the max bound", tlogSizePostDeletes < maxFileSizeBound);
}
/**
- * Sleeps in increments of 50 ms while checking to see if a commit completed. If it did, then return. If not, continue
- * this cycle for at most the amount of time specified
+ * Sleeps in increments of COMMIT_CHECKING_SLEEP_TIME_MS while checking to see if a commit completed. If it did,
+ * then return. If not, continue this cycle for at most the amount of time specified
* @param maxTotalWaitTimeMillis the max amount of time (in ms) to wait/check for a commit
*/
private void waitForCommit(long maxTotalWaitTimeMillis) throws Exception {
long startTimeNanos = System.nanoTime();
long maxTotalWaitTimeNanos = TimeUnit.MILLISECONDS.toNanos(maxTotalWaitTimeMillis);
while (System.nanoTime() - startTimeNanos < maxTotalWaitTimeNanos) {
- Thread.sleep(50);
+ Thread.sleep(COMMIT_CHECKING_SLEEP_TIME_MS);
if (!updateHandler.getUpdateLog().hasUncommittedChanges()) {
return;
}
@@ -264,32 +182,6 @@ public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 {
}
/**
- * Returns the total number of tlogs that have been created for the core.
- *
- * The tlogs in a core's tlog directory are named: tlog.0000000000000000000, tlog.0000000000000000001, tlog.0000000000000000002, etc.
- * Because old tlogs are periodically deleted, we can't just count the number of existing files. Instead, we take the
- * highest ordering tlog file name (which would be the newest) and parse the extension.
- *
- * e.g if the most recently created tlog file is tlog.0000000000000000003, we know that this core has had 4 tlogs.
- *
- * @param tlogsInfo TreeMap of (tlog file name, tlog file size (in bytes)) pairs
- * @return total number of tlogs created for this core
- */
- private int parseTotalNumTlogs(TreeMap<String, Long> tlogsInfo) {
- String mostRecentFileName = tlogsInfo.lastKey();
- int extensionDelimiterIndex = mostRecentFileName.lastIndexOf(".");
- if (extensionDelimiterIndex == -1) {
- throw new RuntimeException("Invalid tlog filename: " + mostRecentFileName);
- }
- String extension = mostRecentFileName.substring(extensionDelimiterIndex + 1);
- try {
- return Integer.parseInt(extension) + 1;
- } catch (NumberFormatException e) {
- throw new RuntimeException("Could not parse tlog filename: " + mostRecentFileName, e);
- }
- }
-
- /**
* Construct a batch add document request with a series of very simple Solr docs with increasing IDs.
* @param startId the document ID to begin with
* @param batchSize the number of documents to include in the batch
@@ -327,66 +219,6 @@ public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 {
}
/**
- * Executes the given query
- * @param query the query to execute
- * @return the number of documents found
- */
- public int queryCore(SolrQueryRequest query) throws Exception {
- String responseStr = h.query(query);
- try {
- Map<String, Object> root = (Map<String, Object>) objectMapper.readValue(responseStr, Object.class);
- Map<String, Object> rootResponse = (Map<String, Object>) root.get("response");
- return (int) rootResponse.get("numFound");
- } catch (Exception e) {
- throw new RuntimeException("Unable to parse Solr query response", e);
- }
- }
-
- /**
- * Checks the given list of document counts to make sure that they are increasing (as commits occur).
- * @param numDocList list of the number of documents found in a given core. Ascending from oldest to newest
- */
- private void checkNumFoundDocuments(List<Integer> numDocList, int finalValue) {
- long currentTotal = 0;
- for (Integer count : numDocList) {
- Assert.assertTrue(count >= currentTotal);
- currentTotal = count;
- }
- Assert.assertEquals(finalValue, numDocList.get(numDocList.size() - 1).intValue());
- }
-
-
- /**
- * Goes through the given tlog directory and inspects each tlog.
- * @param tlogDirPath tlog directory path
- * @return a TreeMap of (tlog file name, tlog file size (in bytes)) pairs
- */
- private TreeMap<String, Long> getTlogFileSizes(String tlogDirPath) {
- return getTlogFileSizes(tlogDirPath, Integer.MAX_VALUE);
- }
-
- /**
- * Goes through the given tlog directory and inspects each tlog. Asserts that each tlog's size is <= the given max size bound.
- * @param tlogDirPath tlog directory path
- * @param maxSizeBound the max tlog size
- * @return a TreeMap of (tlog file name, tlog file size (in bytes)) pairs
- */
- private TreeMap<String, Long> getTlogFileSizes(String tlogDirPath, int maxSizeBound) {
- File tlogDir = new File(tlogDirPath);
- File[] tlogs = tlogDir.listFiles();
- TreeMap<String, Long> tlogInfo = new TreeMap<>();
- if (tlogs != null) {
- for (File tlog : tlogs) {
- String message = String.format(Locale.getDefault(), "Tlog size exceeds the max size bound. Tlog path: %s, tlog size: %d",
- tlog.getPath(), tlog.length());
- Assert.assertTrue(message, tlog.length() <= maxSizeBound);
- tlogInfo.put(tlog.getName(), tlog.length());
- }
- }
- return tlogInfo;
- }
-
- /**
* Convert the given list of strings into a list of streams, for Solr update requests
* @param strs strings to convert into streams
* @return list of streams