You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2021/11/02 23:22:27 UTC
[solr] branch main updated: SOLR-15728: Remove dead,
unused log rotation code from SolrCLI (#375)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new ab9bec6 SOLR-15728: Remove dead, unused log rotation code from SolrCLI (#375)
ab9bec6 is described below
commit ab9bec6aed303e1fb08ac54975e8f50712e6e86c
Author: Jan Høydahl <ja...@users.noreply.github.com>
AuthorDate: Wed Nov 3 00:22:21 2021 +0100
SOLR-15728: Remove dead, unused log rotation code from SolrCLI (#375)
---
solr/CHANGES.txt | 2 +
solr/bin/solr | 8 -
solr/bin/solr.cmd | 12 --
solr/bin/solr.in.cmd | 5 -
solr/bin/solr.in.sh | 5 -
.../src/java/org/apache/solr/util/SolrCLI.java | 237 ++++-----------------
solr/solr-ref-guide/src/configuring-logging.adoc | 1 -
7 files changed, 41 insertions(+), 229 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 92b907e..118b51a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -384,6 +384,8 @@ Other Changes
* SOLR-15699: Removal of the (unused since 5.0) "updateHandler/indexWriter/closeWaitsForMerges" solrconfig.xml element. (Christine Poerschke)
+* SOLR-15728: Remove dead, unused log rotation code from SolrCLI (janhoy)
+
Bug Fixes
---------------------
* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution
diff --git a/solr/bin/solr b/solr/bin/solr
index 52a84c1..0c46a26 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -1996,14 +1996,6 @@ if $verbose ; then
else
q="-q"
fi
-# Rotate and clean on startup. Default to false since 7.4 as log4j2 handles startup rotation
-if [ "${SOLR_LOG_PRESTART_ROTATION:=false}" == "true" ]; then
- # Enable any of these if you require old remove/archive behavior
- #run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -remove_old_solr_logs 7 || echo "Failed removing old solr logs"
- #run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -archive_gc_logs $q || echo "Failed archiving old GC logs"
- #run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -archive_console_logs || echo "Failed archiving old console logs"
- run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -rotate_solr_logs 10 || echo "Failed rotating old solr logs"
-fi
# Establish default GC logging opts if no env var set (otherwise init to sensible default)
if [ -z ${GC_LOG_OPTS+x} ]; then
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index e4f623d..652595a 100755
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -1136,18 +1136,6 @@ if !JAVA_MAJOR_VERSION! LSS 9 (
set IS_64bit=true
)
-REM Clean up and rotate logs. Default to false since 7.4 as log4j2 handles startup rotation
-IF [%SOLR_LOG_PRESTART_ROTATION%] == [] (
- set SOLR_LOG_PRESTART_ROTATION=false
-)
-IF [%SOLR_LOG_PRESTART_ROTATION%] == [true] (
- REM Enable any of these if you require old remove/archive behavior
- REM call :run_utils "-remove_old_solr_logs 7" || echo "Failed removing old solr logs"
- REM call :run_utils "-archive_gc_logs" || echo "Failed archiving old GC logs"
- REM call :run_utils "-archive_console_logs" || echo "Failed archiving old console logs"
- call :run_utils "-rotate_solr_logs 9" || echo "Failed rotating old solr logs"
-)
-
IF NOT "%ZK_HOST%"=="" set SOLR_MODE=solrcloud
IF "%SOLR_MODE%"=="solrcloud" (
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index 28d1bfe..b344879 100755
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -101,11 +101,6 @@ REM set SOLR_LOG_LEVEL=INFO
REM Location where Solr should write logs to. Absolute or relative to solr start dir
REM set SOLR_LOGS_DIR=logs
-REM Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre
-REM start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log
-REM framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.
-REM set SOLR_LOG_PRESTART_ROTATION=false
-
REM Enables jetty request log for all requests
REM set SOLR_REQUESTLOG_ENABLED=false
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index 2d22ce5..e8ef93c 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -125,11 +125,6 @@
# Location where Solr should write logs to. Absolute or relative to solr start dir
#SOLR_LOGS_DIR=logs
-# Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre
-# start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log
-# framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.
-#SOLR_LOG_PRESTART_ROTATION=false
-
# Enables jetty request log for all requests
#SOLR_REQUESTLOG_ENABLED=false
diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
index ef91d7a..ddaa7b5 100755
--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
@@ -16,48 +16,6 @@
*/
package org.apache.solr.util;
-import javax.net.ssl.SSLPeerUnverifiedException;
-import java.io.Console;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.lang.invoke.MethodHandles;
-import java.net.ConnectException;
-import java.net.Socket;
-import java.net.SocketException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.attribute.FileOwnerAttributeView;
-import java.time.Instant;
-import java.time.Period;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@@ -103,13 +61,13 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.UrlScheme;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.UrlScheme;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -125,6 +83,43 @@ import org.noggit.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import java.io.Console;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.lang.invoke.MethodHandles;
+import java.net.ConnectException;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileOwnerAttributeView;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
import static org.apache.solr.common.SolrException.ErrorCode.FORBIDDEN;
import static org.apache.solr.common.SolrException.ErrorCode.UNAUTHORIZED;
import static org.apache.solr.common.params.CommonParams.DISTRIB;
@@ -4201,24 +4196,6 @@ public class SolrCLI implements CLIO {
.build(),
Option.builder("q")
.desc("Be quiet, don't print to stdout, only return exit codes.")
- .build(),
- Option.builder("remove_old_solr_logs")
- .argName("daysToKeep")
- .hasArg()
- .type(Integer.class)
- .desc("Path to logs directory.")
- .build(),
- Option.builder("rotate_solr_logs")
- .argName("generations")
- .hasArg()
- .type(Integer.class)
- .desc("Rotate solr.log to solr.log.1 etc.")
- .build(),
- Option.builder("archive_gc_logs")
- .desc("Archive old garbage collection logs into archive/.")
- .build(),
- Option.builder("archive_console_logs")
- .desc("Archive old console logs into archive/.")
.build()
};
}
@@ -4238,145 +4215,9 @@ public class SolrCLI implements CLIO {
if (cli.hasOption("q")) {
beQuiet = cli.hasOption("q");
}
- if (cli.hasOption("remove_old_solr_logs")) {
- if (removeOldSolrLogs(Integer.parseInt(cli.getOptionValue("remove_old_solr_logs"))) > 0) return 1;
- }
- if (cli.hasOption("rotate_solr_logs")) {
- if (rotateSolrLogs(Integer.parseInt(cli.getOptionValue("rotate_solr_logs"))) > 0) return 1;
- }
- if (cli.hasOption("archive_gc_logs")) {
- if (archiveGcLogs() > 0) return 1;
- }
- if (cli.hasOption("archive_console_logs")) {
- if (archiveConsoleLogs() > 0) return 1;
- }
return 0;
}
- /**
- * Moves gc logs into archived/
- * @return 0 on success
- * @throws Exception on failure
- */
- public int archiveGcLogs() throws Exception {
- prepareLogsPath();
- Path archivePath = logsPath.resolve("archived");
- if (!archivePath.toFile().exists()) {
- Files.createDirectories(archivePath);
- }
- List<Path> archived = Files.find(archivePath, 1, (f, a)
- -> a.isRegularFile() && String.valueOf(f.getFileName()).matches("^solr_gc[_.].+"))
- .collect(Collectors.toList());
- for (Path p : archived) {
- Files.delete(p);
- }
- List<Path> files = Files.find(logsPath, 1, (f, a)
- -> a.isRegularFile() && String.valueOf(f.getFileName()).matches("^solr_gc[_.].+"))
- .collect(Collectors.toList());
- if (files.size() > 0) {
- out("Archiving " + files.size() + " old GC log files to " + archivePath);
- for (Path p : files) {
- Files.move(p, archivePath.resolve(p.getFileName()), StandardCopyOption.REPLACE_EXISTING);
- }
- }
- return 0;
- }
-
- /**
- * Moves console log(s) into archiced/
- * @return 0 on success
- * @throws Exception on failure
- */
- public int archiveConsoleLogs() throws Exception {
- prepareLogsPath();
- Path archivePath = logsPath.resolve("archived");
- if (!archivePath.toFile().exists()) {
- Files.createDirectories(archivePath);
- }
- List<Path> archived = Files.find(archivePath, 1, (f, a)
- -> a.isRegularFile() && String.valueOf(f.getFileName()).endsWith("-console.log"))
- .collect(Collectors.toList());
- for (Path p : archived) {
- Files.delete(p);
- }
- List<Path> files = Files.find(logsPath, 1, (f, a)
- -> a.isRegularFile() && String.valueOf(f.getFileName()).endsWith("-console.log"))
- .collect(Collectors.toList());
- if (files.size() > 0) {
- out("Archiving " + files.size() + " console log files to " + archivePath);
- for (Path p : files) {
- Files.move(p, archivePath.resolve(p.getFileName()), StandardCopyOption.REPLACE_EXISTING);
- }
- }
- return 0;
- }
-
- /**
- * Rotates solr.log before starting Solr. Mimics log4j2 behavior, i.e. with generations=9:
- * <pre>
- * solr.log.9 (and higher) are deleted
- * solr.log.8 -> solr.log.9
- * solr.log.7 -> solr.log.8
- * ...
- * solr.log -> solr.log.1
- * </pre>
- * @param generations number of generations to keep. Should agree with setting in log4j2.xml
- * @return 0 if success
- * @throws Exception if problems
- */
- public int rotateSolrLogs(int generations) throws Exception {
- prepareLogsPath();
- if (logsPath.toFile().exists() && logsPath.resolve("solr.log").toFile().exists()) {
- out("Rotating solr logs, keeping a max of "+generations+" generations.");
- try (Stream<Path> files = Files.find(logsPath, 1,
- (f, a) -> a.isRegularFile() && String.valueOf(f.getFileName()).startsWith("solr.log."))
- .sorted((b,a) -> Integer.valueOf(a.getFileName().toString().substring(9))
- .compareTo(Integer.valueOf(b.getFileName().toString().substring(9))))) {
- files.forEach(p -> {
- try {
- int number = Integer.parseInt(p.getFileName().toString().substring(9));
- if (number >= generations) {
- Files.delete(p);
- } else {
- Path renamed = p.getParent().resolve("solr.log." + (number + 1));
- Files.move(p, renamed);
- }
- } catch (IOException e) {
- out("Problem during rotation of log files: " + e.getMessage());
- }
- });
- } catch (NumberFormatException nfe) {
- throw new Exception("Do not know how to rotate solr.log.<ext> with non-numeric extension. Rotate aborted.", nfe);
- }
- Files.move(logsPath.resolve("solr.log"), logsPath.resolve("solr.log.1"));
- }
-
- return 0;
- }
-
- /**
- * Deletes time-stamped old solr logs, if older than n days
- * @param daysToKeep number of days logs to keep before deleting
- * @return 0 on success
- * @throws Exception on failure
- */
- public int removeOldSolrLogs(int daysToKeep) throws Exception {
- prepareLogsPath();
- if (logsPath.toFile().exists()) {
- try (Stream<Path> stream = Files.find(logsPath, 2, (f, a) -> a.isRegularFile()
- && Instant.now().minus(Period.ofDays(daysToKeep)).isAfter(a.lastModifiedTime().toInstant())
- && String.valueOf(f.getFileName()).startsWith("solr_log_"))) {
- List<Path> files = stream.collect(Collectors.toList());
- if (files.size() > 0) {
- out("Deleting "+files.size() + " solr_log_* files older than " + daysToKeep + " days.");
- for (Path p : files) {
- Files.delete(p);
- }
- }
- }
- }
- return 0;
- }
// Private methods to follow
diff --git a/solr/solr-ref-guide/src/configuring-logging.adoc b/solr/solr-ref-guide/src/configuring-logging.adoc
index 1acaa39..21c658e 100644
--- a/solr/solr-ref-guide/src/configuring-logging.adoc
+++ b/solr/solr-ref-guide/src/configuring-logging.adoc
@@ -126,7 +126,6 @@ Also, in `log4j2.xml` if the default log rotation size threshold of 32MB is too
Java Garbage Collection logs are rotated by the JVM when size hits 20M, for a max of 9 generations.
On every startup or restart of Solr, log4j2 performs log rotation.
-If you choose to use another log framework that does not support rotation on startup, you may enable `SOLR_LOG_PRESTART_ROTATION` in `bin/solr.in.sh` or `bin/solr.in.cmd` to let the start script rotate the logs on startup.
== Logging Slow Queries