You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ep...@apache.org on 2023/05/15 15:55:09 UTC
[solr] branch main updated: SOLR-16765: Export what the limit param specifies, and don't be off by one in the counter. (#1636)
This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new be68ec9404c SOLR-16765: Export what the limit param specifies, and don't be off by one in the counter. (#1636)
be68ec9404c is described below
commit be68ec9404c160c8e5369ffb930193defcf7a02b
Author: Eric Pugh <ep...@opensourceconnections.com>
AuthorDate: Mon May 15 11:55:02 2023 -0400
SOLR-16765: Export what the limit param specifies, and don't be off by one in the counter. (#1636)
---
solr/CHANGES.txt | 2 ++
.../src/java/org/apache/solr/cli/ExportTool.java | 29 ++++++++++++----------
.../test/org/apache/solr/cli/TestExportTool.java | 2 +-
solr/packaging/test/test_export.bats | 3 ++-
4 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3b68d3f2135..0cff2c95355 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -173,6 +173,8 @@ Bug Fixes
* SOLR-16760: bin/solr package tool supports -h and -help command like other tools. (Eric Pugh)
+* SOLR-16765: bin/solr export tool limit property was off by 1. Now limit results exported to the exact number. (Eric Pugh)
+
Dependency Upgrades
---------------------
* PR#1494: Upgrade forbiddenapis to 3.5 (Uwe Schindler)
diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.java
index a262018206a..a50cfb9bbaa 100644
--- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java
@@ -518,7 +518,7 @@ public class ExportTool extends ToolBase {
addConsumer(consumerlatch);
addProducers(m);
if (output != null) {
- output.println("NO: of shards : " + corehandlers.size());
+ output.println("Number of shards : " + corehandlers.size());
}
CountDownLatch producerLatch = new CountDownLatch(corehandlers.size());
corehandlers.forEach(
@@ -550,10 +550,10 @@ public class ExportTool extends ToolBase {
}
System.out.println(
"\nTotal Docs exported: "
- + (docsWritten.get() - 1)
- + ". Time taken: "
+ + docsWritten.get()
+ + ". Time elapsed: "
+ ((System.currentTimeMillis() - startTime) / 1000)
- + "secs");
+ + "seconds");
}
}
@@ -582,7 +582,9 @@ public class ExportTool extends ToolBase {
}
if (doc == EOFDOC) break;
try {
- if (docsWritten.get() > limit) continue;
+ if (docsWritten.get() >= limit) {
+ continue;
+ }
sink.accept(doc);
} catch (Exception e) {
if (output != null) output.println("Failed to write to file " + e.getMessage());
@@ -605,7 +607,7 @@ public class ExportTool extends ToolBase {
boolean exportDocsFromCore() throws IOException, SolrServerException {
try (SolrClient client = new Http2SolrClient.Builder(baseurl).build()) {
- expectedDocs = getDocCount(replica.getCoreName(), client);
+ expectedDocs = getDocCount(replica.getCoreName(), client, query);
GenericSolrRequest request;
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(Q, query);
@@ -638,17 +640,18 @@ public class ExportTool extends ToolBase {
NamedList<Object> rsp = client.request(request);
String nextCursorMark = (String) rsp.get(CursorMarkParams.CURSOR_MARK_NEXT);
if (nextCursorMark == null || Objects.equals(cursorMark, nextCursorMark)) {
- if (output != null)
+ if (output != null) {
output.println(
StrUtils.formatString(
- "\nExport complete for : {0}, docs : {1}",
- replica.getCoreName(), receivedDocs.get()));
+ "\nExport complete from shard {0}, core {1}, docs received: {2}",
+ replica.getShard(), replica.getCoreName(), receivedDocs.get()));
+ }
if (expectedDocs != receivedDocs.get()) {
if (output != null) {
output.println(
StrUtils.formatString(
- "Could not download all docs for core {0} , expected: {1} , actual",
- replica.getCoreName(), expectedDocs, receivedDocs));
+ "Could not download all docs from core {0}, docs expected: {1}, received: {2}",
+ replica.getCoreName(), expectedDocs, receivedDocs.get()));
return false;
}
}
@@ -672,9 +675,9 @@ public class ExportTool extends ToolBase {
}
}
- static long getDocCount(String coreName, SolrClient client)
+ static long getDocCount(String coreName, SolrClient client, String query)
throws SolrServerException, IOException {
- SolrQuery q = new SolrQuery("*:*");
+ SolrQuery q = new SolrQuery(query);
q.setRows(0);
q.add("distrib", "false");
GenericSolrRequest request =
diff --git a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java
index 56c6e9b8656..a59dbafe7fb 100644
--- a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java
+++ b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java
@@ -180,7 +180,7 @@ public class TestExportTool extends SolrCloudTestCase {
for (Slice slice : coll.getSlices()) {
Replica replica = slice.getLeader();
try (SolrClient client = new Http2SolrClient.Builder(replica.getBaseUrl()).build()) {
- long count = ExportTool.getDocCount(replica.getCoreName(), client);
+ long count = ExportTool.getDocCount(replica.getCoreName(), client, "*:*");
docCounts.put(replica.getCoreName(), count);
totalDocsFromCores += count;
}
diff --git a/solr/packaging/test/test_export.bats b/solr/packaging/test/test_export.bats
index 0e9aba9aef5..699eba33911 100644
--- a/solr/packaging/test/test_export.bats
+++ b/solr/packaging/test/test_export.bats
@@ -55,8 +55,9 @@ teardown() {
run solr export -url "http://localhost:8983/solr/techproducts" -query "*:* -id:test" -format jsonl -out "${BATS_TEST_TMPDIR}/output"
assert [ -e ${BATS_TEST_TMPDIR}/output.jsonl ]
- run solr export -url "http://localhost:8983/solr/techproducts" -query "*:* -id:test" -compress -format jsonl -out "${BATS_TEST_TMPDIR}/output"
+ run solr export -url "http://localhost:8983/solr/techproducts" -query "*:* -id:test" -limit 10 -compress -format jsonl -out "${BATS_TEST_TMPDIR}/output"
assert [ -e ${BATS_TEST_TMPDIR}/output.jsonl.gz ]
+ assert_output --partial 'Total Docs exported: 10'
}