You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by at...@apache.org on 2025/05/07 02:33:29 UTC

(druid) branch master updated: Support flexible rotation periods for file-based request logging (#17976)

This is an automated email from the ASF dual-hosted git repository.

atul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git


The following commit(s) were added to refs/heads/master by this push:
     new bb771b30521 Support flexible rotation periods for file-based request logging (#17976)
bb771b30521 is described below

commit bb771b3052167ea4ceb49573ab8a5d24fd521ee9
Author: Atul Mohan <at...@gmail.com>
AuthorDate: Tue May 6 19:33:22 2025 -0700

    Support flexible rotation periods for file-based request logging (#17976)
    
    * Support flexible rotation periods for file-based request logging
    
    * Fix docs
    
    * Fix docs
    
    * Fix docs
    
    * Address comments
    
    * Remove unused variable
---
 docs/configuration/index.md                        |  7 +--
 .../apache/druid/server/log/FileRequestLogger.java | 36 +++++++++-----
 .../server/log/FileRequestLoggerProvider.java      |  7 ++-
 .../druid/server/log/FileRequestLoggerTest.java    | 55 ++++++++++++++++++++--
 4 files changed, 86 insertions(+), 19 deletions(-)

diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index c1f78012347..c7ddc73efa0 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -275,9 +275,10 @@ The `file` request logger stores daily request logs on disk.
 
 |Property|Description|Default|
 |--------|-----------|-------|
-|`druid.request.logging.dir`|Historical, Realtime, and Broker services maintain request logs of all of the requests they get (interaction is via POST, so normal request logs don’t generally capture information about the actual query), this specifies the directory to store the request logs in|none|
-|`druid.request.logging.filePattern`|[Joda datetime format](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html) for each file|"yyyy-MM-dd'.log'"|
-| `druid.request.logging.durationToRetain`| Period to retain the request logs on disk. The period should be at least longer than `P1D`.| none|
+|`druid.request.logging.dir`| Historical, Realtime, and Broker services maintain request logs of all of the requests they get (interaction is via POST, so normal request logs don’t generally capture information about the actual query), this specifies the directory to store the request logs in. | none|
+|`druid.request.logging.filePattern`| [Joda datetime format](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html) for each file.| "yyyy-MM-dd'.log'"|
+|`druid.request.logging.durationToRetain`| Period to retain the request logs on disk. The period should be at least as long as roll period.| none|
+|`druid.request.logging.rollPeriod`| Defines the log rotation period for request logs. The period should be at least `PT1H`. For periods smaller than 1 day, it is recommended to use `"yyyy-MM-dd-HH'.log'"` as the file pattern.| P1D|
 
 The format of request logs is TSV, one line per requests, with five fields: timestamp, remote\_addr, native\_query, query\_context, sql\_query.
 
diff --git a/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java b/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
index a4fe842f3cb..b1bf22061ba 100644
--- a/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
+++ b/server/src/main/java/org/apache/druid/server/log/FileRequestLogger.java
@@ -57,9 +57,10 @@ public class FileRequestLogger implements RequestLogger
   private final File baseDir;
   private final DateTimeFormatter filePattern;
   private final Duration durationToRetain;
+  private final Duration rollPeriod;
   private final Object lock = new Object();
 
-  private DateTime currentDay;
+  private DateTime currentPeriodStart;
   private OutputStreamWriter fileWriter;
 
   public FileRequestLogger(
@@ -67,7 +68,8 @@ public class FileRequestLogger implements RequestLogger
       ScheduledExecutorService exec,
       File baseDir,
       String filePattern,
-      Duration durationToRetain
+      Duration durationToRetain,
+      Duration rollPeriod
   )
   {
     this.exec = exec;
@@ -75,9 +77,14 @@ public class FileRequestLogger implements RequestLogger
     this.baseDir = baseDir;
     this.filePattern = DateTimeFormat.forPattern(filePattern);
     this.durationToRetain = durationToRetain;
+    this.rollPeriod = rollPeriod;
     Preconditions.checkArgument(
-        this.durationToRetain == null || this.durationToRetain.compareTo(Duration.standardDays(1)) >= 0,
-        "request logs retention period must be atleast P1D"
+        this.rollPeriod == null || this.rollPeriod.compareTo(Duration.standardHours(1)) >= 0,
+        "request logs roll period must be atleast PT1H"
+    );
+    Preconditions.checkArgument(
+        this.durationToRetain == null || this.durationToRetain.compareTo(this.rollPeriod) >= 0,
+        "request logs retention period must be atleast as long as roll period"
     );
   }
 
@@ -89,19 +96,24 @@ public class FileRequestLogger implements RequestLogger
       FileUtils.mkdirp(baseDir);
 
       MutableDateTime mutableDateTime = DateTimes.nowUtc().toMutableDateTime(ISOChronology.getInstanceUTC());
-      mutableDateTime.setMillisOfDay(0);
+      mutableDateTime.setMinuteOfHour(0);
+      mutableDateTime.setSecondOfMinute(0);
+      mutableDateTime.setMillisOfSecond(0);
+      if (rollPeriod.compareTo(Duration.standardDays(1)) >= 0) {
+        mutableDateTime.setHourOfDay(0);
+      }
       synchronized (lock) {
-        currentDay = mutableDateTime.toDateTime(ISOChronology.getInstanceUTC());
+        currentPeriodStart = mutableDateTime.toDateTime(ISOChronology.getInstanceUTC());
 
         fileWriter = getFileWriter();
       }
-      long nextDay = currentDay.plusDays(1).getMillis();
-      Duration initialDelay = new Duration(nextDay - System.currentTimeMillis());
+      long nextPeriodMillis = currentPeriodStart.plus(rollPeriod).getMillis();
+      Duration initialDelay = new Duration(nextPeriodMillis - System.currentTimeMillis());
 
       ScheduledExecutors.scheduleWithFixedDelay(
           exec,
           initialDelay,
-          Duration.standardDays(1),
+          rollPeriod,
           new Callable<>()
           {
             @Override
@@ -109,11 +121,11 @@ public class FileRequestLogger implements RequestLogger
             {
               try {
                 synchronized (lock) {
-                  currentDay = currentDay.plusDays(1);
+                  currentPeriodStart = currentPeriodStart.plus(rollPeriod);
 
                   CloseableUtils.closeAndSuppressExceptions(
                       fileWriter,
-                      e -> log.warn("Could not close log file for %s. Creating new log file anyway.", currentDay)
+                      e -> log.warn("Could not close log file for %s. Creating new log file anyway.", currentPeriodStart)
                   );
 
                   fileWriter = getFileWriter();
@@ -173,7 +185,7 @@ public class FileRequestLogger implements RequestLogger
   private OutputStreamWriter getFileWriter() throws FileNotFoundException
   {
     return new OutputStreamWriter(
-        new FileOutputStream(new File(baseDir, filePattern.print(currentDay)), true),
+        new FileOutputStream(new File(baseDir, filePattern.print(currentPeriodStart)), true),
         StandardCharsets.UTF_8
     );
   }
diff --git a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
index 9765929fd3b..cb5f8bb1c26 100644
--- a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
+++ b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
@@ -27,6 +27,7 @@ import org.apache.druid.guice.annotations.Json;
 import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory;
 import org.apache.druid.java.util.common.logger.Logger;
 import org.joda.time.Duration;
+import org.joda.time.Period;
 
 import javax.validation.constraints.NotNull;
 import java.io.File;
@@ -58,6 +59,9 @@ public class FileRequestLoggerProvider implements RequestLoggerProvider
   @JsonProperty
   private Duration durationToRetain;
 
+  @JsonProperty
+  private Duration rollPeriod = new Period("P1D").toStandardDuration();
+
   @Override
   public RequestLogger get()
   {
@@ -66,7 +70,8 @@ public class FileRequestLoggerProvider implements RequestLoggerProvider
         factory.create(1, "RequestLogger-%s"),
         dir,
         filePattern,
-        durationToRetain
+        durationToRetain,
+        rollPeriod
     );
     log.debug(new Exception("Stack trace"), "Creating %s at", logger);
     return logger;
diff --git a/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java b/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
index abbb7b95600..0ceb539adab 100644
--- a/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
+++ b/server/src/test/java/org/apache/druid/server/log/FileRequestLoggerTest.java
@@ -64,7 +64,8 @@ public class FileRequestLoggerTest
         scheduler,
         logDir,
         "yyyy-MM-dd'.log'",
-        null
+        null,
+        Duration.standardDays(1)
     );
     fileRequestLogger.start();
 
@@ -101,6 +102,7 @@ public class FileRequestLoggerTest
         scheduler,
         logDir,
         "yyyy-MM-dd'.log'",
+        Duration.standardDays(1),
         Duration.standardDays(1)
     );
     fileRequestLogger.start();
@@ -119,7 +121,7 @@ public class FileRequestLoggerTest
   public void testLogRemoveWithInvalidDuration() throws Exception
   {
     expectedException.expect(IllegalArgumentException.class);
-    expectedException.expectMessage("request logs retention period must be atleast P1D");
+    expectedException.expectMessage("request logs retention period must be atleast as long as roll period");
     ObjectMapper objectMapper = new ObjectMapper();
     File logDir = temporaryFolder.newFolder();
     FileRequestLogger fileRequestLogger = new FileRequestLogger(
@@ -127,7 +129,54 @@ public class FileRequestLoggerTest
         scheduler,
         logDir,
         "yyyy-MM-dd'.log'",
-        Duration.standardHours(12)
+        Duration.standardMinutes(30),
+        Duration.standardDays(1)
     );
   }
+
+  @Test
+  public void testRollPeriodForLog() throws Exception
+  {
+    ObjectMapper objectMapper = new ObjectMapper();
+    DateTime dateTime = DateTimes.nowUtc();
+    File logDir = temporaryFolder.newFolder();
+    String sqlQueryLogString = dateTime + "\t" + HOST + "\t" + "sql";
+
+    FileRequestLogger hourlyLogger = new FileRequestLogger(
+        objectMapper,
+        scheduler,
+        logDir,
+        "yyyy-MM-dd-HH'.log'",
+        null,
+        Duration.standardHours(1)
+    );
+    FileRequestLogger dailyLoggerWithHourPattern = new FileRequestLogger(
+        objectMapper,
+        scheduler,
+        logDir,
+        "yyyy-MM-dd-HH'.log'",
+        null,
+        Duration.standardHours(26)
+    );
+    hourlyLogger.start();
+    dailyLoggerWithHourPattern.start();
+
+    RequestLogLine sqlRequestLogLine = EasyMock.createMock(RequestLogLine.class);
+    EasyMock.expect(sqlRequestLogLine.getSqlQueryLine(EasyMock.anyObject())).andReturn(sqlQueryLogString).anyTimes();
+    EasyMock.replay(sqlRequestLogLine);
+
+    hourlyLogger.logSqlQuery(sqlRequestLogLine);
+    dailyLoggerWithHourPattern.logSqlQuery(sqlRequestLogLine);
+
+    File hourlyLogFile = new File(logDir, dateTime.toString("yyyy-MM-dd-HH'.log'"));
+    // The hour is zeroed out since the roll period is more than 1 day
+    File dailyLogFile = new File(logDir, dateTime.toString("yyyy-MM-dd-00'.log'"));
+    String hourlyLogString = CharStreams.toString(Files.newBufferedReader(hourlyLogFile.toPath(), StandardCharsets.UTF_8));
+    String dailyLogString = CharStreams.toString(Files.newBufferedReader(dailyLogFile.toPath(), StandardCharsets.UTF_8));
+    Assert.assertTrue(hourlyLogString.contains(sqlQueryLogString + "\n"));
+    Assert.assertTrue(dailyLogString.contains(sqlQueryLogString + "\n"));
+
+    hourlyLogger.stop();
+    dailyLoggerWithHourPattern.stop();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org