[GitHub] [nifi-minifi-cpp] szaszm commented on a change in pull request #955: MINIFICPP-1414 Create in-memory compressed logs

[GitBox <gi...@apache.org>]

szaszm commented on a change in pull request #955:
URL: https://github.com/apache/nifi-minifi-cpp/pull/955#discussion_r603272527



##########
File path: libminifi/test/unit/LoggerTests.cpp
##########
@@ -107,3 +110,95 @@ TEST_CASE("Test ShortenNames", "[ttl6]") {
   LogTestController::getInstance(props)->reset();
   LogTestController::getInstance().reset();
 }
+
+using namespace minifi::io;
+
+std::string decompress(const std::shared_ptr<InputStream>& input) {
+  auto output = utils::make_unique<BufferStream>();
+  auto decompressor = std::make_shared<ZlibDecompressStream>(gsl::make_not_null(output.get()));
+  minifi::internal::pipe(input, decompressor);
+  decompressor->close();
+  return std::string{reinterpret_cast<const char*>(output->getBuffer()), output->size()};
+}
+
+TEST_CASE("Test Compression", "[ttl7]") {
+  auto& log_config = logging::LoggerConfiguration::getConfiguration();
+  auto properties = std::make_shared<logging::LoggerProperties>();
+  std::string className;
+  SECTION("Using root logger") {
+    className = "CompressionTestClassUsingRoot";
+    // by default the root logger is OFF
+    properties->set("logger.root", "INFO");
+  }
+  SECTION("Inherit compression sink") {
+    className = "CompressionTestClassInheriting";
+    properties->set("appender.null", "null");
+    properties->set("logger." + className, "INFO,null");
+  }
+  log_config.initialize(properties);
+  auto logger = log_config.getLogger(className);
+  logger->log_error("Hi there");
+  std::shared_ptr<InputStream> compressed_log{logging::LoggerConfiguration::getCompressedLog(true)};
+  REQUIRE(compressed_log);
+  auto logs = decompress(compressed_log);
+  REQUIRE(logs == "Hi there");
+}
+
+class LoggerTestAccessor {
+ public:
+  static void setCompressionCacheSegmentSize(logging::LoggerConfiguration& log_config, size_t value) {
+    log_config.compression_manager_.cache_segment_size = value;
+  }
+  static void setCompressionCompressedSegmentSize(logging::LoggerConfiguration& log_config, size_t value) {
+    log_config.compression_manager_.compressed_segment_size = value;
+  }
+  static size_t getUncompressedSize(logging::LoggerConfiguration& log_config) {
+    return log_config.compression_manager_.getSink()->cached_logs_.size();
+  }
+  static size_t getCompressedSize(logging::LoggerConfiguration& log_config) {
+    return log_config.compression_manager_.getSink()->compressed_logs_.size();
+  }
+};
+
+TEST_CASE("Test Compression cache overflow is discarded intermittently", "[ttl8]") {
+  auto& log_config = logging::LoggerConfiguration::getConfiguration();
+  auto properties = std::make_shared<logging::LoggerProperties>();
+  properties->set(logging::internal::CompressionManager::compression_cached_log_max_size_, "10 KB");
+  LoggerTestAccessor::setCompressionCacheSegmentSize(log_config, 1_KiB);
+  std::string className = "CompressionTestCacheCleaned";
+  // by default the root logger is OFF
+  properties->set("logger.root", "INFO");
+  log_config.initialize(properties);
+  auto logger = log_config.getLogger(className);
+  for (size_t idx = 0; idx < 10000; ++idx) {
+    logger->log_error("Hi there");
+  }
+  bool cache_shrunk = utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, [&] {
+    return LoggerTestAccessor::getUncompressedSize(log_config) <= 10_KiB;
+  });
+  REQUIRE(cache_shrunk);
+}
+
+TEST_CASE("Setting either properties to 0 disables the compression", "[ttl9]") {

Review comment:
       ```suggestion
   TEST_CASE("Setting either properties to 0 disables in-memory compressed logs", "[ttl9]") {
   ```
   Disabling compression would mean uncompressed storage, but this disables storage completely if I understand correctly.




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org