You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ep...@apache.org on 2023/08/03 12:16:12 UTC
[solr] 01/02: SOLR-6994: Integrate post tool into bin/solr and bin/solr.cmd (#1634)
This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit dcc3ee1b9e3a0f8fb0dfb928b6b26e755f3cf938
Author: Eric Pugh <ep...@opensourceconnections.com>
AuthorDate: Fri Jul 7 08:07:51 2023 -0400
SOLR-6994: Integrate post tool into bin/solr and bin/solr.cmd (#1634)
We now support the Post tool on windows by reusing the SolrCLI infrastructure, instead of having a separate post.cmd. This commit does NOT remove the post.jar compilation, or the bin/post shell script. It does update the Ref Guide to refer to bin/solr post. In the future we will want to come to a resolution on the post.jar, and maybe rename bin/solr post to something else like bin/solr indexer? We have very basic bats and unit test now.
---
solr/CHANGES.txt | 2 +
solr/bin/post | 1 +
solr/bin/solr | 9 +-
solr/bin/solr.cmd | 1 +
.../src/java/org/apache/solr/cli/DeleteTool.java | 2 +-
.../src/java/org/apache/solr/cli/PostTool.java | 143 +++++++++++++++++
.../java/org/apache/solr/cli/RunExampleTool.java | 8 +
.../java/org/apache/solr/cli/SimplePostTool.java | 174 ++++++++++++++-------
.../core/src/java/org/apache/solr/cli/SolrCLI.java | 10 +-
.../src/test/org/apache/solr/cli/PostToolTest.java | 55 +++++++
solr/packaging/test/test_extraction.bats | 110 +++++++++++++
solr/packaging/test/test_help.bats | 12 +-
solr/packaging/test/test_post.bats | 173 ++++++++++++++++++++
.../modules/indexing-guide/pages/post-tool.adoc | 107 ++++++-------
14 files changed, 675 insertions(+), 132 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a588e42fd91..2974bf88f94 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -51,6 +51,8 @@ Other Changes
* SOLR-16915: Lower the AffinityPlacementPlugin's default minimalFreeDiskGB to 5 GB (Houston Putman)
+* SOLR-6994: Implement Windows version of bin/post via implementing bin/solr post command that works on Windows and Unix. Deprecate bin/post. (Eric Pugh, Will White)
+
================== 9.3.0 ==================
Upgrade Notes
diff --git a/solr/bin/post b/solr/bin/post
index c9ad8510802..e7ef443afd2 100755
--- a/solr/bin/post
+++ b/solr/bin/post
@@ -49,6 +49,7 @@ fi
# test that Java exists and is executable on this server
"$JAVA" -version >/dev/null 2>&1 || { echo >&2 "Java is required to run this tool! Please install Java 8 or greater before running this script."; exit 1; }
+echo "The bin/post script is deprecated in favour of the bin/solr post command. Please update your scripts."
# ===== post specific code
diff --git a/solr/bin/solr b/solr/bin/solr
index 79b1c6c8046..7aeae89b8af 100644
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -690,7 +690,6 @@ function print_usage() {
elif [ "$CMD" == "package" ]; then
echo ""
run_tool package -solrUrl "http://fake.to.allow.help.to.run" "help"
-
fi
} # end print_usage
@@ -1477,12 +1476,6 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
exit $?
fi
-
-if [[ "$SCRIPT_CMD" == "export" ]]; then
- run_tool export "$@"
- exit $?
-fi
-
if [[ "$SCRIPT_CMD" == "package" ]]; then
if [ $# -gt 0 ]; then
while true; do
@@ -1644,7 +1637,7 @@ fi
# verify the command given is supported
-if [ "$SCRIPT_CMD" != "stop" ] && [ "$SCRIPT_CMD" != "start" ] && [ "$SCRIPT_CMD" != "restart" ] && [ "$SCRIPT_CMD" != "status" ] && [ "$SCRIPT_CMD" != "assert" ]; then
+if [ "$SCRIPT_CMD" != "stop" ] && [ "$SCRIPT_CMD" != "start" ] && [ "$SCRIPT_CMD" != "restart" ] && [ "$SCRIPT_CMD" != "status" ]; then
# handoff this command to the SolrCLI and let it handle the option parsing and validation
run_tool "$SCRIPT_CMD" "$@"
exit $?
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index bfdeab06c92..7b3ecdd9281 100755
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -237,6 +237,7 @@ IF "%1"=="export" goto run_solrcli
IF "%1"=="package" goto run_solrcli
IF "%1"=="api" goto run_solrcli
IF "%1"=="postlogs" goto run_solrcli
+IF "%1"=="post" goto run_solrcli
REM Only allow the command to be the first argument, assume start if not supplied
IF "%1"=="start" goto set_script_cmd
diff --git a/solr/core/src/java/org/apache/solr/cli/DeleteTool.java b/solr/core/src/java/org/apache/solr/cli/DeleteTool.java
index d4d3d2e1943..78e36416d05 100644
--- a/solr/core/src/java/org/apache/solr/cli/DeleteTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/DeleteTool.java
@@ -234,4 +234,4 @@ public class DeleteTool extends ToolBase {
echoIfVerbose("\n", cli);
}
}
-} // end DeleteTool class
+}
diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java b/solr/core/src/java/org/apache/solr/cli/PostTool.java
new file mode 100644
index 00000000000..5e6b5efeef0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cli;
+
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.List;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+
+public class PostTool extends ToolBase {
+
+ public PostTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public PostTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ @Override
+ public String getName() {
+ return "post";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ Option.builder("url")
+ .argName("url")
+ .hasArg()
+ .required(true)
+ .desc("<base Solr update URL>")
+ .build(),
+ Option.builder("commit").required(false).desc("Issue a commit at end of post").build(),
+ Option.builder("optimize").required(false).desc("Issue an optimize at end of post").build(),
+ Option.builder("mode")
+ .argName("mode")
+ .hasArg(true)
+ .required(false)
+ .desc("Files crawls files, web crawls website. default: files.")
+ .build(),
+ Option.builder("recursive")
+ .argName("recursive")
+ .hasArg(true)
+ .required(false)
+ .desc("For web crawl, how deep to go. default: 1")
+ .build(),
+ Option.builder("delay")
+ .argName("delay")
+ .hasArg(true)
+ .required(false)
+ .desc(
+ "If recursive then delay will be the wait time between posts. default: 10 for web, 0 for files")
+ .build(),
+ Option.builder("type")
+ .argName("content-type")
+ .hasArg(true)
+ .required(false)
+ .desc("default: application/json")
+ .build(),
+ Option.builder("filetypes")
+ .argName("<type>[,<type>,...]")
+ .hasArg(true)
+ .required(false)
+ .desc("default: " + SimplePostTool.DEFAULT_FILE_TYPES)
+ .build(),
+ Option.builder("params")
+ .argName("<key>=<value>[&<key>=<value>...]")
+ .hasArg(true)
+ .required(false)
+ .desc("values must be URL-encoded; these pass through to Solr update request")
+ .build(),
+ Option.builder("out")
+ .required(false)
+ .desc("sends Solr response outputs to console")
+ .build(),
+ Option.builder("format")
+ .required(false)
+ .desc(
+ "sends application/json content as Solr commands to /update instead of /update/json/docs")
+ .build());
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ SolrCLI.raiseLogLevelUnlessVerbose(cli);
+
+ String url = cli.getOptionValue("url");
+ URL solrUrl = new URL(url);
+
+ String mode = SimplePostTool.DEFAULT_DATA_MODE;
+ if (cli.hasOption("mode")) {
+ mode = cli.getOptionValue("mode");
+ }
+ boolean auto = true;
+ String type = null;
+ if (cli.hasOption("type")) {
+ type = cli.getOptionValue("type");
+ }
+ String format =
+ cli.hasOption("format")
+ ? SimplePostTool.FORMAT_SOLR
+ : ""; // i.e not solr formatted json commands
+
+ String fileTypes = SimplePostTool.DEFAULT_FILE_TYPES;
+ if (cli.hasOption("filetypes")) {
+ fileTypes = cli.getOptionValue("filetypes");
+ }
+
+ int defaultDelay = (mode.equals((SimplePostTool.DATA_MODE_WEB)) ? 10 : 0);
+ int delay = Integer.parseInt(cli.getOptionValue("delay", String.valueOf(defaultDelay)));
+ int recursive = Integer.parseInt(cli.getOptionValue("recursive", "1"));
+
+ OutputStream out = cli.hasOption("out") ? CLIO.getOutStream() : null;
+ boolean commit = cli.hasOption("commit");
+ boolean optimize = cli.hasOption("optimize");
+
+ String[] args = cli.getArgs();
+
+ SimplePostTool spt =
+ new SimplePostTool(
+ mode, solrUrl, auto, type, format, recursive, delay, fileTypes, out, commit, optimize,
+ args);
+
+ spt.execute();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
index a3615ef92c4..47dfaf30d6f 100644
--- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
@@ -315,12 +315,20 @@ public class RunExampleTool extends ToolBase {
String currentPropVal = System.getProperty("url");
System.setProperty("url", updateUrl);
+ String currentTypeVal = System.getProperty("type");
+ // We assume that example docs are always in XML.
+ System.setProperty("type", "application/xml");
SimplePostTool.main(new String[] {exampledocsDir.getAbsolutePath() + "/*.xml"});
if (currentPropVal != null) {
System.setProperty("url", currentPropVal); // reset
} else {
System.clearProperty("url");
}
+ if (currentTypeVal != null) {
+ System.setProperty("type", currentTypeVal); // reset
+ } else {
+ System.clearProperty("type");
+ }
} else {
echo(
"exampledocs directory not found, skipping indexing step for the techproducts example");
diff --git a/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java b/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
index 4ccbc871f33..11478366711 100644
--- a/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java
@@ -67,6 +67,7 @@ import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.RTimer;
+import org.apache.solr.util.SolrVersion;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@@ -76,12 +77,14 @@ import org.xml.sax.SAXException;
* A simple utility class for posting raw updates to a Solr server. It has a main method, so it can
* be run on the command line. View this not as a best-practice code example, but as a standalone
* example built with an explicit purpose of not having external jar dependencies.
+ *
+ * @deprecated This class is being replaced with bin/solr tool. Please use {@link PostTool} instead.
*/
+@Deprecated(since = "9.2")
public class SimplePostTool {
private static final String DEFAULT_POST_HOST = "localhost";
private static final String DEFAULT_POST_PORT = "8983";
- private static final String VERSION_OF_THIS_TOOL =
- "5.0.0"; // TODO: hardcoded for now, but eventually to sync with actual Solr version
+ private static final String VERSION_OF_THIS_TOOL = SolrVersion.LATEST_STRING;
private static final String DEFAULT_COMMIT = "yes";
private static final String DEFAULT_OPTIMIZE = "no";
@@ -90,8 +93,8 @@ public class SimplePostTool {
private static final String DEFAULT_RECURSIVE = "0";
private static final int DEFAULT_WEB_DELAY = 10;
private static final int MAX_WEB_DEPTH = 10;
- private static final String DEFAULT_CONTENT_TYPE = "application/xml";
- private static final String DEFAULT_FILE_TYPES =
+ public static final String DEFAULT_CONTENT_TYPE = "application/json";
+ public static final String DEFAULT_FILE_TYPES =
"xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log";
private static final String BASIC_AUTH = "basicauth";
@@ -101,6 +104,8 @@ public class SimplePostTool {
static final String DATA_MODE_WEB = "web";
static final String DEFAULT_DATA_MODE = DATA_MODE_FILES;
+ static final String FORMAT_SOLR = "solr";
+
// Input args
boolean auto = false;
int recursive = 0;
@@ -141,7 +146,7 @@ public class SimplePostTool {
mimeMap.put("xml", "application/xml");
mimeMap.put("csv", "text/csv");
mimeMap.put("json", "application/json");
- mimeMap.put("jsonl", "application/json");
+ mimeMap.put("jsonl", "application/jsonl");
mimeMap.put("pdf", "application/pdf");
mimeMap.put("rtf", "text/rtf");
mimeMap.put("html", "text/html");
@@ -198,8 +203,12 @@ public class SimplePostTool {
return;
}
- if (commit) commit();
- if (optimize) optimize();
+ if (commit) {
+ commit();
+ }
+ if (optimize) {
+ optimize();
+ }
displayTiming((long) timer.getTime());
}
@@ -230,7 +239,6 @@ public class SimplePostTool {
}
String params = System.getProperty("params", "");
-
String host = System.getProperty("host", DEFAULT_POST_HOST);
String port = System.getProperty("port", DEFAULT_POST_PORT);
String core = System.getProperty("c");
@@ -244,6 +252,7 @@ public class SimplePostTool {
if (urlStr == null) {
urlStr = String.format(Locale.ROOT, "http://%s:%s/solr/%s/update", host, port, core);
}
+
urlStr = SimplePostTool.appendParam(urlStr, params);
URL url = new URL(urlStr);
String user = null;
@@ -252,7 +261,9 @@ public class SimplePostTool {
} else if (System.getProperty(BASIC_AUTH) != null) {
user = System.getProperty(BASIC_AUTH).trim().split(":")[0];
}
- if (user != null) info("Basic Authentication enabled, user=" + user);
+ if (user != null) {
+ info("Basic Authentication enabled, user=" + user);
+ }
boolean auto = isOn(System.getProperty("auto", DEFAULT_AUTO));
String type = System.getProperty("type");
@@ -263,7 +274,9 @@ public class SimplePostTool {
try {
recursive = Integer.parseInt(r);
} catch (Exception e) {
- if (isOn(r)) recursive = DATA_MODE_WEB.equals(mode) ? 1 : 999;
+ if (isOn(r)) {
+ recursive = DATA_MODE_WEB.equals(mode) ? 1 : 999;
+ }
}
// Delay
int delay = DATA_MODE_WEB.equals(mode) ? DEFAULT_WEB_DELAY : 0;
@@ -343,9 +356,12 @@ public class SimplePostTool {
+ solrUrl
+ (!auto ? " using content-type " + (type == null ? DEFAULT_CONTENT_TYPE : type) : "")
+ "...");
- if (auto) info("Entering auto mode. File endings considered are " + fileTypes);
- if (recursive > 0)
+ if (auto) {
+ info("Entering auto mode. File endings considered are " + fileTypes);
+ }
+ if (recursive > 0) {
info("Entering recursive mode, max depth=" + recursive + ", delay=" + delay + "s");
+ }
int numFilesPosted = postFiles(args, 0, out, type);
info(numFilesPosted + " files indexed.");
}
@@ -382,9 +398,10 @@ public class SimplePostTool {
recursive = MAX_WEB_DEPTH;
warn("Too large recursion depth for web mode, limiting to " + MAX_WEB_DEPTH + "...");
}
- if (delay < DEFAULT_WEB_DELAY)
+ if (delay < DEFAULT_WEB_DELAY) {
warn(
"Never crawl an external web site faster than every 10 seconds, your IP will probably be blocked");
+ }
info("Entering recursive mode, depth=" + recursive + ", delay=" + delay + "s");
}
numPagesPosted = postWebPages(args, 0, out);
@@ -425,8 +442,8 @@ public class SimplePostTool {
+ ")\n"
+ " -Dtype=<content-type> (default="
+ DEFAULT_CONTENT_TYPE
- + ")\n"
+ " -Dhost=<host> (default: "
+ + ")\n"
+ DEFAULT_POST_HOST
+ ")\n"
+ " -Dport=<port> (default: "
@@ -545,7 +562,9 @@ public class SimplePostTool {
* @return number of files posted total
*/
private int postDirectory(File dir, OutputStream out, String type) {
- if (dir.isHidden() && !dir.getName().equals(".")) return (0);
+ if (dir.isHidden() && !dir.getName().equals(".")) {
+ return (0);
+ }
info(
"Indexing directory "
+ dir.getPath()
@@ -577,7 +596,9 @@ public class SimplePostTool {
int filesPosted = 0;
for (File srcFile : files) {
try {
- if (!srcFile.isFile() || srcFile.isHidden()) continue;
+ if (!srcFile.isFile() || srcFile.isHidden()) {
+ continue;
+ }
postFile(srcFile, out, type);
Thread.sleep(delay * 1000L);
filesPosted++;
@@ -599,7 +620,9 @@ public class SimplePostTool {
int handleGlob(File globFile, OutputStream out, String type) {
int filesPosted = 0;
File parent = globFile.getParentFile();
- if (parent == null) parent = new File(".");
+ if (parent == null) {
+ parent = new File(".");
+ }
String fileGlob = globFile.getName();
GlobFileFilter ff = new GlobFileFilter(fileGlob, false);
File[] fileList = parent.listFiles(ff);
@@ -642,9 +665,15 @@ public class SimplePostTool {
* @return the normalized URL string
*/
protected static String normalizeUrlEnding(String link) {
- if (link.contains("#")) link = link.substring(0, link.indexOf('#'));
- if (link.endsWith("?")) link = link.substring(0, link.length() - 1);
- if (link.endsWith("/")) link = link.substring(0, link.length() - 1);
+ if (link.contains("#")) {
+ link = link.substring(0, link.indexOf('#'));
+ }
+ if (link.endsWith("?")) {
+ link = link.substring(0, link.length() - 1);
+ }
+ if (link.endsWith("/")) {
+ link = link.substring(0, link.length() - 1);
+ }
return link;
}
@@ -754,7 +783,9 @@ public class SimplePostTool {
if (!path.endsWith("/")) {
int sep = path.lastIndexOf('/');
String file = path.substring(sep + 1);
- if (file.contains(".") || file.contains("?")) path = path.substring(0, sep);
+ if (file.contains(".") || file.contains("?")) {
+ path = path.substring(0, sep);
+ }
}
link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + path + "/" + link;
}
@@ -778,7 +809,9 @@ public class SimplePostTool {
protected boolean typeSupported(String type) {
for (Map.Entry<String, String> entry : mimeMap.entrySet()) {
if (entry.getValue().equals(type)) {
- if (fileTypes.contains(entry.getKey())) return true;
+ if (fileTypes.contains(entry.getKey())) {
+ return true;
+ }
}
}
return false;
@@ -829,7 +862,9 @@ public class SimplePostTool {
public static String appendParam(String url, String param) {
String[] pa = param.split("&");
for (String p : pa) {
- if (p.trim().length() == 0) continue;
+ if (p.trim().length() == 0) {
+ continue;
+ }
String[] kv = p.split("=");
if (kv.length == 2) {
url = url + (url.contains("?") ? "&" : "?") + kv[0] + "=" + kv[1];
@@ -853,7 +888,7 @@ public class SimplePostTool {
// TODO: Add a flag that disables /update and sends all to /update/extract, to avoid CSV,
// JSON, and XML files
// TODO: from being interpreted as Solr documents internally
- if (type.equals("application/json") && !"solr".equals(format)) {
+ if (type.equals("application/json") && !FORMAT_SOLR.equals(format)) {
suffix = "/json/docs";
String urlStr = appendUrlPath(solrUrl, suffix).toString();
url = new URL(urlStr);
@@ -865,18 +900,22 @@ public class SimplePostTool {
// SolrCell
suffix = "/extract";
String urlStr = appendUrlPath(solrUrl, suffix).toString();
- if (!urlStr.contains("resource.name"))
+ if (!urlStr.contains("resource.name")) {
urlStr =
appendParam(
urlStr, "resource.name=" + URLEncoder.encode(file.getAbsolutePath(), UTF_8));
- if (!urlStr.contains("literal.id"))
+ }
+ if (!urlStr.contains("literal.id")) {
urlStr =
appendParam(
urlStr, "literal.id=" + URLEncoder.encode(file.getAbsolutePath(), UTF_8));
+ }
url = new URL(urlStr);
}
} else {
- if (type == null) type = DEFAULT_CONTENT_TYPE;
+ if (type == null) {
+ type = DEFAULT_CONTENT_TYPE;
+ }
}
info(
"POSTing file "
@@ -891,7 +930,9 @@ public class SimplePostTool {
warn("Can't open/read file: " + file);
} finally {
try {
- if (is != null) is.close();
+ if (is != null) {
+ is.close();
+ }
} catch (IOException e) {
fatal("IOException while closing file: " + e);
}
@@ -941,7 +982,9 @@ public class SimplePostTool {
/** Performs a simple get on the given URL */
public void doGet(URL url) {
try {
- if (mockMode) return;
+ if (mockMode) {
+ return;
+ }
HttpURLConnection urlc = (HttpURLConnection) url.openConnection();
basicAuth(urlc);
urlc.connect();
@@ -960,30 +1003,35 @@ public class SimplePostTool {
*/
public boolean postData(
InputStream data, Long length, OutputStream output, String type, URL url) {
- if (mockMode) return true;
+ if (mockMode) {
+ return true;
+ }
+
boolean success = true;
- if (type == null) type = DEFAULT_CONTENT_TYPE;
- HttpURLConnection urlc = null;
+ if (type == null) {
+ type = DEFAULT_CONTENT_TYPE;
+ }
+ HttpURLConnection urlConnection = null;
try {
try {
- urlc = (HttpURLConnection) url.openConnection();
+ urlConnection = (HttpURLConnection) url.openConnection();
try {
- urlc.setRequestMethod("POST");
+ urlConnection.setRequestMethod("POST");
} catch (ProtocolException e) {
fatal("Shouldn't happen: HttpURLConnection doesn't support POST??" + e);
}
- urlc.setDoOutput(true);
- urlc.setDoInput(true);
- urlc.setUseCaches(false);
- urlc.setAllowUserInteraction(false);
- urlc.setRequestProperty("Content-type", type);
- basicAuth(urlc);
+ urlConnection.setDoOutput(true);
+ urlConnection.setDoInput(true);
+ urlConnection.setUseCaches(false);
+ urlConnection.setAllowUserInteraction(false);
+ urlConnection.setRequestProperty("Content-type", type);
+ basicAuth(urlConnection);
if (null != length) {
- urlc.setFixedLengthStreamingMode(length);
+ urlConnection.setFixedLengthStreamingMode(length);
} else {
- urlc.setChunkedStreamingMode(-1); // use JDK default chunkLen, 4k in Java 8.
+ urlConnection.setChunkedStreamingMode(-1); // use JDK default chunkLen, 4k in Java 8.
}
- urlc.connect();
+ urlConnection.connect();
} catch (IOException e) {
fatal("Connection error (is Solr running at " + solrUrl + " ?): " + e);
success = false;
@@ -991,15 +1039,15 @@ public class SimplePostTool {
fatal("POST failed with error " + e.getMessage());
}
- try (final OutputStream out = urlc.getOutputStream()) {
+ try (final OutputStream out = urlConnection.getOutputStream()) {
pipe(data, out);
} catch (IOException e) {
fatal("IOException while posting data: " + e);
}
try {
- success &= checkResponseCode(urlc);
- try (final InputStream in = urlc.getInputStream()) {
+ success &= checkResponseCode(urlConnection);
+ try (final InputStream in = urlConnection.getInputStream()) {
pipe(in, output);
}
} catch (IOException e) {
@@ -1010,7 +1058,9 @@ public class SimplePostTool {
"Looks like Solr is secured and would not let us in. Try with another user in '-u' parameter");
}
} finally {
- if (urlc != null) urlc.disconnect();
+ if (urlConnection != null) {
+ urlConnection.disconnect();
+ }
}
return success;
}
@@ -1093,15 +1143,22 @@ public class SimplePostTool {
byte[] buf = new byte[1024];
int read = 0;
while ((read = source.read(buf)) >= 0) {
- if (null != dest) dest.write(buf, 0, read);
+ if (null != dest) {
+ dest.write(buf, 0, read);
+ }
+ }
+ if (null != dest) {
+ dest.flush();
}
- if (null != dest) dest.flush();
}
public FileFilter getFileFilterFromFileTypes(String fileTypes) {
String glob;
- if (fileTypes.equals("*")) glob = ".*";
- else glob = "^.*\\.(" + fileTypes.replace(",", "|") + ")$";
+ if (fileTypes.equals("*")) {
+ glob = ".*";
+ } else {
+ glob = "^.*\\.(" + fileTypes.replace(",", "|") + ")$";
+ }
return new GlobFileFilter(glob, true);
}
@@ -1132,7 +1189,9 @@ public class SimplePostTool {
if (nodes.getLength() > 0) {
for (int i = 0; i < nodes.getLength(); i++) {
sb.append(nodes.item(i).getNodeValue()).append(' ');
- if (!concatAll) break;
+ if (!concatAll) {
+ break;
+ }
}
return sb.toString().trim();
} else return "";
@@ -1308,11 +1367,12 @@ public class SimplePostTool {
* @return a set of URIs parsed from the page
*/
protected Set<URI> getLinksFromWebPage(URL u, InputStream is, String type, URL postUrl) {
- Set<URI> l = new HashSet<>();
+ Set<URI> linksFromPage = new HashSet<>();
URL url = null;
try {
ByteArrayOutputStream os = new ByteArrayOutputStream();
URL extractUrl = new URL(appendParam(postUrl.toString(), "extractOnly=true"));
+ extractUrl = new URL(appendParam(extractUrl.toString(), "wt=xml"));
boolean success = postData(is, null, os, type, extractUrl);
if (success) {
Document d = makeDom(os.toByteArray());
@@ -1322,10 +1382,12 @@ public class SimplePostTool {
for (int i = 0; i < links.getLength(); i++) {
String link = links.item(i).getTextContent();
link = computeFullUrl(u, link);
- if (link == null) continue;
+ if (link == null) {
+ continue;
+ }
URI newUri = new URI(link);
if (newUri.getAuthority() == null || !newUri.getAuthority().equals(u.getAuthority())) {
- l.add(newUri);
+ linksFromPage.add(newUri);
}
}
}
@@ -1337,7 +1399,7 @@ public class SimplePostTool {
throw new RuntimeException(e);
}
- return l;
+ return linksFromPage;
}
}
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
index ce1ea16d564..a11fc7bdfa6 100755
--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
@@ -242,6 +242,7 @@ public class SolrCLI implements CLIO {
else if ("package".equals(toolType)) return new PackageTool();
else if ("postlogs".equals(toolType)) return new PostLogsTool();
else if ("version".equals(toolType)) return new VersionTool();
+ else if ("post".equals(toolType)) return new PostTool();
// If you add a built-in tool to this class, add it here to avoid
// classpath scanning
@@ -322,6 +323,7 @@ public class SolrCLI implements CLIO {
try {
cli = (new GnuParser()).parse(options, args);
} catch (ParseException exp) {
+ // Check if we passed in a help argument with a non parsing set of arguments.
boolean hasHelpArg = false;
if (args != null) {
for (String arg : args) {
@@ -333,10 +335,12 @@ public class SolrCLI implements CLIO {
}
if (!hasHelpArg) {
CLIO.err("Failed to parse command-line arguments due to: " + exp.getMessage());
+ exit(1);
+ } else {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(toolName, options);
+ exit(0);
}
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(toolName, options);
- exit(1);
}
if (cli.hasOption("help")) {
diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java
new file mode 100644
index 00000000000..681d788138c
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cli;
+
+import static org.apache.solr.cli.SolrCLI.findTool;
+import static org.apache.solr.cli.SolrCLI.parseCmdLine;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class PostToolTest extends SolrCloudTestCase {
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(1)
+ .addConfig(
+ "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .configure();
+ }
+
+ @Test
+ public void testBasicRun() throws Exception {
+ final String collection = "aliasedCollection";
+ CollectionAdminRequest.createCollection(collection, "config", 1, 1)
+ .process(cluster.getSolrClient());
+
+ String[] args = {"post", "-url", "http://localhost:8983/solr/aliasedCollection", "blah.json"};
+ assertEquals(0, runTool(args));
+ }
+
+ private int runTool(String[] args) throws Exception {
+ Tool tool = findTool(args);
+ assertTrue(tool instanceof PostTool);
+ CommandLine cli = parseCmdLine(tool.getName(), args, tool.getOptions());
+ return tool.runTool(cli);
+ }
+}
diff --git a/solr/packaging/test/test_extraction.bats b/solr/packaging/test/test_extraction.bats
new file mode 100644
index 00000000000..875f4ca2e59
--- /dev/null
+++ b/solr/packaging/test/test_extraction.bats
@@ -0,0 +1,110 @@
+#!/usr/bin/env bats
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+load bats_helper
+
+setup() {
+ common_clean_setup
+}
+
+teardown() {
+ # save a snapshot of SOLR_HOME for failed tests
+ save_home_on_failure
+
+ delete_all_collections
+ SOLR_STOP_WAIT=1 solr stop -all >/dev/null 2>&1
+}
+
+@test "using curl to extract a single pdf file" {
+
+ # Disable security manager to allow extraction
+ # This appears to be a bug.
+ export SOLR_SECURITY_MANAGER_ENABLED=false
+ solr start -c -Dsolr.modules=extraction
+
+
+ solr create_collection -c gettingstarted -d _default
+
+
+ curl -X POST -H 'Content-type:application/json' -d '{
+ "add-requesthandler": {
+ "name": "/update/extract",
+ "class": "solr.extraction.ExtractingRequestHandler",
+ "defaults":{ "lowernames": "true", "captureAttr":"true"}
+ }
+ }' 'http://localhost:8983/solr/gettingstarted/config'
+
+ curl 'http://localhost:8983/solr/gettingstarted/update/extract?literal.id=doc1&commit=true' -F "myfile=@${SOLR_TIP}/example/exampledocs/solr-word.pdf"
+
+ run curl 'http://localhost:8983/solr/gettingstarted/select?q=id:doc1'
+ assert_output --partial '"numFound":1'
+}
+
+@test "using the bin/solr post tool to extract content from pdf" {
+
+ # Disable security manager to allow extraction
+ # This appears to be a bug.
+ export SOLR_SECURITY_MANAGER_ENABLED=false
+ solr start -c -Dsolr.modules=extraction
+
+ solr create_collection -c content_extraction -d _default
+
+ curl -X POST -H 'Content-type:application/json' -d '{
+ "add-requesthandler": {
+ "name": "/update/extract",
+ "class": "solr.extraction.ExtractingRequestHandler",
+ "defaults":{ "lowernames": "true", "captureAttr":"true"}
+ }
+ }' 'http://localhost:8983/solr/content_extraction/config'
+
+ # We filter to pdf to invoke the Extract handler.
+ run solr post -filetypes pdf -commit -url http://localhost:8983/solr/content_extraction/update ${SOLR_TIP}/example/exampledocs
+
+ assert_output --partial '1 files indexed.'
+ refute_output --partial 'ERROR'
+
+ run curl 'http://localhost:8983/solr/content_extraction/select?q=*:*'
+ assert_output --partial '"numFound":1'
+}
+
+@test "using the bin/solr post tool to crawl web site" {
+
+ # Disable security manager to allow extraction
+ # This appears to be a bug.
+ export SOLR_SECURITY_MANAGER_ENABLED=false
+ solr start -c -Dsolr.modules=extraction
+
+ solr create_collection -c website_extraction -d _default
+
+ curl -X POST -H 'Content-type:application/json' -d '{
+ "add-requesthandler": {
+ "name": "/update/extract",
+ "class": "solr.extraction.ExtractingRequestHandler",
+ "defaults":{ "lowernames": "true", "captureAttr":"true"}
+ }
+ }' 'http://localhost:8983/solr/website_extraction/config'
+
+ # Change to -recursive 1 to crawl multiple pages, but may be too slow.
+ run solr post -mode web -commit -url http://localhost:8983/solr/website_extraction/update -recursive 0 -delay 1 https://solr.apache.org/
+
+ assert_output --partial 'POSTed web resource https://solr.apache.org (depth: 0)'
+ refute_output --partial 'ERROR'
+
+ run curl 'http://localhost:8983/solr/website_extraction/select?q=*:*'
+ assert_output --partial '"numFound":1'
+}
diff --git a/solr/packaging/test/test_help.bats b/solr/packaging/test/test_help.bats
index 0bdcb17566f..ed7a91b78b4 100644
--- a/solr/packaging/test/test_help.bats
+++ b/solr/packaging/test/test_help.bats
@@ -90,5 +90,13 @@ setup() {
}
@test "assert help flag prints help" {
- skip "Currently the assert -help flag doesn't return nice help text!"
-}
\ No newline at end of file
+ run solr assert -help
+ assert_output --partial 'usage: assert'
+ refute_output --partial 'ERROR'
+}
+
+@test "post help flag prints help" {
+ run solr post -help
+ assert_output --partial 'usage: post'
+ refute_output --partial 'ERROR'
+}
diff --git a/solr/packaging/test/test_post.bats b/solr/packaging/test/test_post.bats
new file mode 100644
index 00000000000..2b8c18bfe46
--- /dev/null
+++ b/solr/packaging/test/test_post.bats
@@ -0,0 +1,173 @@
+#!/usr/bin/env bats
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load bats_helper
+
+setup_file() {
+ common_clean_setup
+ solr start -c -Dsolr.modules=extraction
+}
+
+teardown_file() {
+ common_setup
+ solr stop -all
+}
+
+setup() {
+ common_setup
+}
+
+teardown() {
+ # save a snapshot of SOLR_HOME for failed tests
+ save_home_on_failure
+}
+
+@test "Check help command" {
+
+ run solr post
+ assert_output --partial 'Failed to parse command-line arguments due to: Missing required option: url'
+
+ run solr post -h
+ assert_output --partial 'usage: post'
+ refute_output --partial 'ERROR'
+
+ run solr post -help
+ assert_output --partial 'usage: post'
+ refute_output --partial 'ERROR'
+
+}
+
+
+@test "basic post with a type specified" {
+
+ run solr create_collection -c monitors -d _default
+ assert_output --partial "Created collection 'monitors'"
+
+ run solr post -type application/xml -url http://localhost:8983/solr/monitors/update ${SOLR_TIP}/example/exampledocs/monitor.xml
+
+ assert_output --partial '1 files indexed.'
+ refute_output --partial 'ERROR'
+}
+
+@test "basic post WITHOUT a type specified" {
+
+ solr create_collection -c monitors_no_type -d _default
+
+ run solr post -url http://localhost:8983/solr/monitors_no_type/update -commit ${SOLR_TIP}/example/exampledocs/monitor.xml
+
+ assert_output --partial '1 files indexed.'
+ refute_output --partial 'ERROR'
+ run curl 'http://localhost:8983/solr/monitors_no_type/select?q=*:*'
+ assert_output --partial '"numFound":1'
+
+ solr create_collection -c books_no_type -d _default
+
+ run solr post -url http://localhost:8983/solr/books_no_type/update -commit ${SOLR_TIP}/example/exampledocs/books.json
+
+ assert_output --partial '1 files indexed.'
+ refute_output --partial 'ERROR'
+ run curl 'http://localhost:8983/solr/books_no_type/select?q=*:*'
+ assert_output --partial '"numFound":4'
+
+ solr create_collection -c books_csv_no_type -d _default
+
+ run solr post -url http://localhost:8983/solr/books_csv_no_type/update -commit ${SOLR_TIP}/example/exampledocs/books.csv
+
+ assert_output --partial '1 files indexed.'
+ refute_output --partial 'ERROR'
+ run curl 'http://localhost:8983/solr/books_csv_no_type/select?q=*:*'
+ assert_output --partial '"numFound":10'
+}
+
+@test "crawling a directory" {
+
+ solr create_collection -c mixed_content -d _default
+
+ # We filter to xml,json,and csv as we don't want to invoke the Extract handler.
+ run solr post -filetypes xml,json,csv -url http://localhost:8983/solr/mixed_content/update -commit ${SOLR_TIP}/example/exampledocs
+
+ assert_output --partial '16 files indexed.'
+ refute_output --partial 'ERROR'
+ run curl 'http://localhost:8983/solr/mixed_content/select?q=*:*'
+ assert_output --partial '"numFound":46'
+}
+
+# this test doesn't complete due to issues in posting to the /extract handler
+@test "crawling a web site" {
+ solr create_collection -c webcrawl -d _default
+
+ curl -X POST -H 'Content-type:application/json' -d '{
+ "add-requesthandler": {
+ "name": "/update/extract",
+ "class": "solr.extraction.ExtractingRequestHandler",
+ "defaults":{ "lowernames": "true", "captureAttr":"true"}
+ }
+ }' 'http://localhost:8983/solr/webcrawl/config'
+
+ run solr post -mode web -url http://localhost:8983/webcrawl/update -recursive 1 -delay 1 https://solr.apache.org
+ assert_output --partial 'Entering crawl at level 0'
+}
+
+@test "commit and optimize and delete" {
+
+ run solr create_collection -c monitors2 -d _default
+ assert_output --partial "Created collection 'monitors2'"
+
+ run solr post -url http://localhost:8983/solr/monitors2/update -type application/xml -commit -optimize ${SOLR_TIP}/example/exampledocs/monitor.xml
+
+ assert_output --partial '1 files indexed.'
+ assert_output --partial 'COMMITting Solr index'
+ assert_output --partial 'Performing an OPTIMIZE'
+ refute_output --partial 'ERROR'
+
+}
+
+@test "args mode" {
+
+ run solr create_collection -c test_args -d _default
+ assert_output --partial "Created collection 'test_args'"
+
+ run solr post -url http://localhost:8983/solr/test_args/update -mode args -type application/xml -out -commit "<delete><query>*:*</query></delete>"
+ assert_output --partial '<int name="status">0</int>'
+
+ # confirm default type
+ run solr post -url http://localhost:8983/solr/test_args/update -mode args -out -commit "{'delete': {'query': '*:*'}}"
+ assert_output --partial '"status":0'
+
+ # confirm we don't get back output without -out
+ run solr post -url http://localhost:8983/solr/test_args/update -mode args -commit "{'delete': {'query': '*:*'}}"
+ refute_output --partial '"status":0'
+
+ run solr post -url http://localhost:8983/solr/test_args/update -mode args -commit -type text/csv -out $'id,value\nROW1,0.47'
+ assert_output --partial '"status":0'
+ run curl 'http://localhost:8983/solr/test_args/select?q=id:ROW1'
+ assert_output --partial '"numFound":1'
+}
+
+# function used because run echo | solr ends up being (run echo) | solr and we loose the output capture.
+capture_echo_to_solr() {
+ echo "{'commit': {}}" | solr post -url http://localhost:8983/solr/test_stdin/update -mode stdin -type application/json -out
+}
+
+@test "stdin mode" {
+
+ run solr create_collection -c test_stdin -d _default
+ assert_output --partial "Created collection 'test_stdin'"
+
+ run capture_echo_to_solr
+ assert_output --partial '"status":0'
+}
diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/post-tool.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/post-tool.adoc
index f2d303b5abc..28a47d28bbc 100644
--- a/solr/solr-ref-guide/modules/indexing-guide/pages/post-tool.adoc
+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/post-tool.adoc
@@ -18,106 +18,114 @@
Solr includes a simple command line tool for POSTing various types of content to a Solr server.
-The tool is `bin/post`.
-The bin/post tool is a Unix shell script; for Windows (non-Cygwin) usage, see the section <<Post Tool Windows Support>> below.
+The tool is `bin/solr post`.
+The bin/solr post tool is a Unix shell script; for Windows (non-Cygwin) usage, see the section <<Post Tool Windows Support>> below.
NOTE: This tool is meant for use by new users exploring Solr's capabilities, and is not intended as a robust solution to be used for indexing documents into production systems.
+NOTE: You may be familiar with SimplePostTool and the bin/post Unix shell script. While this is still available, it is deprecated and will be removed in Solr 10.
+
+
To run it, open a window and enter:
-[source,bash]
+[,console]
----
-bin/post -c gettingstarted example/films/films.json
+$ bin/solr post -url http://localhost:8983/gettingstarted/update example/films/films.json
----
This will contact the server at `localhost:8983`.
-Specifying the `collection/core name` is *mandatory*.
-The `-help` (or simply `-h`) option will output information on its usage (i.e., `bin/post -help)`.
+The `-help` (or simply `-h`) option will output information on its usage (i.e., `bin/solr post -help)`.
-== Using the bin/post Tool
+== Using the bin/solr post Tool
-Specifying either the `collection/core name` or the full update `url` is *mandatory* when using `bin/post`.
+Specifying the full update `url` is *mandatory* when using `bin/solr post`.
-The basic usage of `bin/post` is:
+The basic usage of `bin/solr post` is:
[source,plain]
----
-$ bin/post -h
-Usage: post -c <collection> [OPTIONS] <files|directories|urls|-d ["...",...]>
+$ bin/solr post -h
+Usage: post -url http://localhost:8983/gettingstarted/update [OPTIONS] <files|directories|urls|-d ["...",...]>
or post -help
- collection name defaults to DEFAULT_SOLR_COLLECTION if not specified
-
OPTIONS
=======
Solr options:
- -url <base Solr update URL> (overrides collection, host, and port)
- -host <host> (default: localhost)
- -p or -port <port> (default: 8983)
- -commit yes|no (default: yes)
+ -url <base Solr update URL>
+ -commit issue a commit
-u or -user <user:pass> (sets BasicAuth credentials)
Web crawl options:
-recursive <depth> (default: 1)
-delay <seconds> (default: 10)
-
Directory crawl options:
-delay <seconds> (default: 0)
stdin/args options:
- -type <content/type> (default: application/xml)
+ -type <content/type> (default: application/json)
Other options:
-filetypes <type>[,<type>,...] (default: xml,json,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log)
-params "<key>=<value>[&<key>=<value>...]" (values must be URL-encoded; these pass through to Solr update request)
- -out yes|no (default: no; yes outputs Solr response to console)
-...
+ -out output the Solr responses to console
+ -format solr (sends application/json content as Solr commands to /update instead of /update/json/docs
+
+Examples:
+
+* JSON file: bin/solr post -url http://localhost:8983/wizbang/update events.json
+* XML files: bin/solr post -url http://localhost:8983/records/update article*.xml
+* CSV file: bin/solr post -url http://localhost:8983/signals/update LATEST-signals.csv
+* Directory of files: bin/solr post -filetypes xml,json,csv -url http://localhost:8983/myfiles/update ~/Documents
+* Web crawl: bin/solr post -mode web -url http://localhost:8983/gettingstarted/update -recursive 1 -delay 1 https://solr.apache.org/
+* Standard input (stdin): echo '{commit: {}}' | bin/solr post -mode stdin -url http://localhost:8983/my_collection/update -out
+* Data as string: bin/solr post -url http://localhost:8983/signals/update -mode args -type text/csv -out $'id,value\n1,0.47'
+
----
-== Examples Using bin/post
+== Examples Using bin/solr post
-There are several ways to use `bin/post`.
+There are several ways to use `bin/solr post`.
This section presents several examples.
=== Indexing XML
-Add all documents with file extension `.xml` to collection or core named `gettingstarted`.
+Add all documents with file extension `.xml` to the collection named `gettingstarted`.
[source,bash]
----
-bin/post -c gettingstarted *.xml
+bin/solr post -url http://localhost:8983/solr/gettingstarted/update *.xml
----
-Add all documents with file extension `.xml` to the `gettingstarted` collection/core on Solr running on port `8984`.
+Add all documents with file extension `.xml` to the `gettingstarted` collection on Solr running on port `8984`.
[source,bash]
----
-bin/post -c gettingstarted -p 8984 *.xml
+bin/solr post -url http://localhost:8984/solr/gettingstarted/update *.xml
----
Send XML arguments to delete a document from `gettingstarted`.
[source,bash]
----
-bin/post -c gettingstarted -d '<delete><id>42</id></delete>'
+bin/solr post -url http://localhost:8983/solr/gettingstarted/update -mode args -type application/xml '<delete><id>42</id></delete>'
----
-=== Indexing CSV
+=== Indexing CSV and JSON
-Index all CSV files into `gettingstarted`:
+Index all CSV and JSON files into `gettingstarted` from current directory:
[source,bash]
----
-bin/post -c gettingstarted *.csv
+bin/solr post -url http://localhost:8983/solr/gettingstarted/update -filetypes json,csv .
----
Index a tab-separated file into `gettingstarted`:
[source,bash]
----
-bin/post -c signals -params "separator=%09" -type text/csv data.tsv
+bin/solr post -url http://localhost:8984/solr/signals/update -params "separator=%09" -type text/csv data.tsv
----
The content type (`-type`) parameter is required to treat the file as the proper type, otherwise it will be ignored and a WARNING logged as it does not know what type of content a .tsv file is.
@@ -129,7 +137,7 @@ Index all JSON files into `gettingstarted`.
[source,bash]
----
-bin/post -c gettingstarted *.json
+bin/solr post -url http://localhost:8983/solr/gettingstarted/update *.json
----
=== Indexing Rich Documents (PDF, Word, HTML, etc.)
@@ -138,21 +146,21 @@ Index a PDF file into `gettingstarted`.
[source,bash]
----
-bin/post -c gettingstarted a.pdf
+bin/solr post -url http://localhost:8983/solr/gettingstarted/update a.pdf
----
Automatically detect content types in a folder, and recursively scan it for documents for indexing into `gettingstarted`.
[source,bash]
----
-bin/post -c gettingstarted afolder/
+bin/solr post -url http://localhost:8983/solr/gettingstarted/update afolder/
----
Automatically detect content types in a folder, but limit it to PPT and HTML files and index into `gettingstarted`.
[source,bash]
----
-bin/post -c gettingstarted -filetypes ppt,html afolder/
+bin/solr post -url http://localhost:8983/solr/gettingstarted/update -filetypes ppt,html afolder/
----
=== Indexing to a Password Protected Solr (Basic Auth)
@@ -161,30 +169,5 @@ Index a PDF as the user "solr" with password "SolrRocks":
[source,bash]
----
-bin/post -u solr:SolrRocks -c gettingstarted a.pdf
+bin/solr post -u solr:SolrRocks -url http://localhost:8983/solr/gettingstarted/update a.pdf
----
-
-== Post Tool Windows Support
-
-`bin/post` is a Unix shell script and as such cannot be used directly on Windows.
-However it delegates its work to a cross-platform capable Java program called "SimplePostTool" or `post.jar`, that can be used in Windows environments.
-
-The argument syntax differs significantly from `bin/post`, so your first step should be to print the SimplePostTool help text.
-
-[source,plain]
-----
-$ java -jar example\exampledocs\post.jar -h
-----
-
-This command prints information about all the arguments and System properties available to SimplePostTool users.
-There are also examples showing how to post files, crawl a website or file system folder, and send update commands (deletes, etc.) directly to Solr.
-
-Most usage involves passing both Java System properties and program arguments on the command line. Consider the example below:
-
-[source,plain]
-----
-$ java -jar -Dc=gettingstarted -Dauto example\exampledocs\post.jar example\exampledocs\*
-----
-
-This indexes the contents of the `exampledocs` directory into a collection called `gettingstarted`.
-The `-Dauto` System property governs whether or not Solr sends the document type to Solr during extraction.