You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@nifi.apache.org by GitBox <gi...@apache.org> on 2021/07/08 15:53:36 UTC
[GitHub] [nifi] tpalfy commented on a change in pull request #5195: NIFI-8752: Automatic diagnostic at NiFi restart/stop
tpalfy commented on a change in pull request #5195:
URL: https://github.com/apache/nifi/pull/5195#discussion_r666317706
##########
File path: nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DiagnosticProperties.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.util;
+
+import org.apache.nifi.properties.BootstrapProperties;
+import org.apache.nifi.util.NiFiBootstrapUtils;
+import org.apache.nifi.util.file.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class DiagnosticProperties {
+
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosticProperties.class);
+
+ private static final String ALLOWED_PROP_NAME = "nifi.diag.allowed";
+ private static final boolean ALLOWED_DEFAULT_VALUE = true;
+
+ private static final String DIR_PROP_NAME = "nifi.diag.dir";
+ private static final String DIR_DEFAULT_VALUE = "./diagnostics";
+
+ private static final String MAX_FILE_COUNT_PROP_NAME = "nifi.diag.filecount.max";
+ private static final int MAX_FILE_COUNT_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String MAX_SIZE_PROP_NAME = "nifi.diag.size.max.byte";
+ private static final int MAX_SIZE_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String VERBOSE_PROP_NAME = "nifi.diag.verbose";
+ private static final boolean VERBOSE_DEFAULT_VALUE = false;
+
+ private final String dirPath;
+ private final int maxFileCount;
+ private final int maxSizeInBytes;
+ private final boolean verbose;
+ private final boolean allowed;
+
+ public DiagnosticProperties() throws IOException {
+ BootstrapProperties properties = NiFiBootstrapUtils.loadBootstrapProperties();
+ this.dirPath = properties.getProperty(DIR_PROP_NAME, DIR_DEFAULT_VALUE);
+ this.maxFileCount = getPropertyAsInt(properties.getProperty(MAX_FILE_COUNT_PROP_NAME), MAX_FILE_COUNT_DEFAULT_VALUE);
+ this.maxSizeInBytes = getPropertyAsInt(properties.getProperty(MAX_SIZE_PROP_NAME), MAX_SIZE_DEFAULT_VALUE);
+ this.verbose = getPropertyAsBoolean(properties.getProperty(VERBOSE_PROP_NAME), VERBOSE_DEFAULT_VALUE);
+ this.allowed = getPropertyAsBoolean(properties.getProperty(ALLOWED_PROP_NAME), ALLOWED_DEFAULT_VALUE);
+ createDiagDir();
+ }
+
+ public Path getOldestFile() throws IOException {
+ Comparator<? super Path> lastModifiedComparator = Comparator.comparingLong(p -> p.toFile().lastModified());
+
+ final List<Path> oldestFiles;
+
+ try (Stream<Path> paths = Files.walk(Paths.get(dirPath))) {
+ oldestFiles = paths.filter(Files::isRegularFile)
+ .sorted(lastModifiedComparator)
+ .limit(1)
+ .collect(Collectors.toList());
+ }
+
+ return oldestFiles.get(0);
+ }
+
+ public boolean isFileCountExceeded() {
+ return new File(dirPath).list().length >= maxFileCount;
+ }
+
+ public boolean isSizeExceeded() {
+ return FileUtils.getDirectorySize(Paths.get(dirPath), logger) >= maxSizeInBytes;
+ }
+
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ public String getDirPath() {
+ return dirPath;
+ }
+
+ public boolean isAllowed() {
+ return allowed;
+ }
+
+ private static int getPropertyAsInt(final String property, final int defaultValue) {
+ try {
+ return Integer.parseInt(property);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ private static boolean getPropertyAsBoolean(final String property, final boolean defaultValue) {
+ try {
+ return Boolean.parseBoolean(property);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ private void createDiagDir() {
Review comment:
Is an abbreviation that only save a couple characters worth it?
```suggestion
private void createDiagnosticsDir() {
```
##########
File path: nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DiagnosticProperties.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.util;
+
+import org.apache.nifi.properties.BootstrapProperties;
+import org.apache.nifi.util.NiFiBootstrapUtils;
+import org.apache.nifi.util.file.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class DiagnosticProperties {
+
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosticProperties.class);
+
+ private static final String ALLOWED_PROP_NAME = "nifi.diag.allowed";
+ private static final boolean ALLOWED_DEFAULT_VALUE = true;
+
+ private static final String DIR_PROP_NAME = "nifi.diag.dir";
+ private static final String DIR_DEFAULT_VALUE = "./diagnostics";
+
+ private static final String MAX_FILE_COUNT_PROP_NAME = "nifi.diag.filecount.max";
+ private static final int MAX_FILE_COUNT_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String MAX_SIZE_PROP_NAME = "nifi.diag.size.max.byte";
+ private static final int MAX_SIZE_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String VERBOSE_PROP_NAME = "nifi.diag.verbose";
+ private static final boolean VERBOSE_DEFAULT_VALUE = false;
+
+ private final String dirPath;
+ private final int maxFileCount;
+ private final int maxSizeInBytes;
+ private final boolean verbose;
+ private final boolean allowed;
+
+ public DiagnosticProperties() throws IOException {
+ BootstrapProperties properties = NiFiBootstrapUtils.loadBootstrapProperties();
+ this.dirPath = properties.getProperty(DIR_PROP_NAME, DIR_DEFAULT_VALUE);
+ this.maxFileCount = getPropertyAsInt(properties.getProperty(MAX_FILE_COUNT_PROP_NAME), MAX_FILE_COUNT_DEFAULT_VALUE);
+ this.maxSizeInBytes = getPropertyAsInt(properties.getProperty(MAX_SIZE_PROP_NAME), MAX_SIZE_DEFAULT_VALUE);
+ this.verbose = getPropertyAsBoolean(properties.getProperty(VERBOSE_PROP_NAME), VERBOSE_DEFAULT_VALUE);
+ this.allowed = getPropertyAsBoolean(properties.getProperty(ALLOWED_PROP_NAME), ALLOWED_DEFAULT_VALUE);
+ createDiagDir();
+ }
+
+ public Path getOldestFile() throws IOException {
+ Comparator<? super Path> lastModifiedComparator = Comparator.comparingLong(p -> p.toFile().lastModified());
+
+ final List<Path> oldestFiles;
+
+ try (Stream<Path> paths = Files.walk(Paths.get(dirPath))) {
+ oldestFiles = paths.filter(Files::isRegularFile)
+ .sorted(lastModifiedComparator)
+ .limit(1)
+ .collect(Collectors.toList());
+ }
+
+ return oldestFiles.get(0);
+ }
+
+ public boolean isFileCountExceeded() {
+ return new File(dirPath).list().length >= maxFileCount;
+ }
+
+ public boolean isSizeExceeded() {
+ return FileUtils.getDirectorySize(Paths.get(dirPath), logger) >= maxSizeInBytes;
+ }
+
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ public String getDirPath() {
+ return dirPath;
+ }
+
+ public boolean isAllowed() {
+ return allowed;
+ }
+
+ private static int getPropertyAsInt(final String property, final int defaultValue) {
+ try {
+ return Integer.parseInt(property);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ private static boolean getPropertyAsBoolean(final String property, final boolean defaultValue) {
+ try {
+ return Boolean.parseBoolean(property);
Review comment:
This doesn't throw any exception so it would be okay to inline it and remove the `getPropertyAsBoolean` altogether.
##########
File path: nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DiagnosticProperties.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.util;
+
+import org.apache.nifi.properties.BootstrapProperties;
+import org.apache.nifi.util.NiFiBootstrapUtils;
+import org.apache.nifi.util.file.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class DiagnosticProperties {
+
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosticProperties.class);
+
+ private static final String ALLOWED_PROP_NAME = "nifi.diag.allowed";
+ private static final boolean ALLOWED_DEFAULT_VALUE = true;
+
+ private static final String DIR_PROP_NAME = "nifi.diag.dir";
+ private static final String DIR_DEFAULT_VALUE = "./diagnostics";
+
+ private static final String MAX_FILE_COUNT_PROP_NAME = "nifi.diag.filecount.max";
+ private static final int MAX_FILE_COUNT_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String MAX_SIZE_PROP_NAME = "nifi.diag.size.max.byte";
+ private static final int MAX_SIZE_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String VERBOSE_PROP_NAME = "nifi.diag.verbose";
+ private static final boolean VERBOSE_DEFAULT_VALUE = false;
+
+ private final String dirPath;
+ private final int maxFileCount;
+ private final int maxSizeInBytes;
+ private final boolean verbose;
+ private final boolean allowed;
+
+ public DiagnosticProperties() throws IOException {
+ BootstrapProperties properties = NiFiBootstrapUtils.loadBootstrapProperties();
+ this.dirPath = properties.getProperty(DIR_PROP_NAME, DIR_DEFAULT_VALUE);
+ this.maxFileCount = getPropertyAsInt(properties.getProperty(MAX_FILE_COUNT_PROP_NAME), MAX_FILE_COUNT_DEFAULT_VALUE);
+ this.maxSizeInBytes = getPropertyAsInt(properties.getProperty(MAX_SIZE_PROP_NAME), MAX_SIZE_DEFAULT_VALUE);
+ this.verbose = getPropertyAsBoolean(properties.getProperty(VERBOSE_PROP_NAME), VERBOSE_DEFAULT_VALUE);
+ this.allowed = getPropertyAsBoolean(properties.getProperty(ALLOWED_PROP_NAME), ALLOWED_DEFAULT_VALUE);
+ createDiagDir();
+ }
+
+ public Path getOldestFile() throws IOException {
+ Comparator<? super Path> lastModifiedComparator = Comparator.comparingLong(p -> p.toFile().lastModified());
+
+ final List<Path> oldestFiles;
+
+ try (Stream<Path> paths = Files.walk(Paths.get(dirPath))) {
+ oldestFiles = paths.filter(Files::isRegularFile)
+ .sorted(lastModifiedComparator)
+ .limit(1)
+ .collect(Collectors.toList());
+ }
+
+ return oldestFiles.get(0);
Review comment:
Although it shouldn't happen when we actually call this, `oldestFiles` can be empty.
Instead of
```java
.limit(1)
.collect(Collectors.toList());
```
we could use
```java
.findFirst();
```
which returns an `Optional`
##########
File path: nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DiagnosticProperties.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.bootstrap.util;
+
+import org.apache.nifi.properties.BootstrapProperties;
+import org.apache.nifi.util.NiFiBootstrapUtils;
+import org.apache.nifi.util.file.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class DiagnosticProperties {
+
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosticProperties.class);
+
+ private static final String ALLOWED_PROP_NAME = "nifi.diag.allowed";
+ private static final boolean ALLOWED_DEFAULT_VALUE = true;
+
+ private static final String DIR_PROP_NAME = "nifi.diag.dir";
+ private static final String DIR_DEFAULT_VALUE = "./diagnostics";
+
+ private static final String MAX_FILE_COUNT_PROP_NAME = "nifi.diag.filecount.max";
+ private static final int MAX_FILE_COUNT_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String MAX_SIZE_PROP_NAME = "nifi.diag.size.max.byte";
+ private static final int MAX_SIZE_DEFAULT_VALUE = Integer.MAX_VALUE;
+
+ private static final String VERBOSE_PROP_NAME = "nifi.diag.verbose";
+ private static final boolean VERBOSE_DEFAULT_VALUE = false;
+
+ private final String dirPath;
+ private final int maxFileCount;
+ private final int maxSizeInBytes;
+ private final boolean verbose;
+ private final boolean allowed;
+
+ public DiagnosticProperties() throws IOException {
+ BootstrapProperties properties = NiFiBootstrapUtils.loadBootstrapProperties();
+ this.dirPath = properties.getProperty(DIR_PROP_NAME, DIR_DEFAULT_VALUE);
+ this.maxFileCount = getPropertyAsInt(properties.getProperty(MAX_FILE_COUNT_PROP_NAME), MAX_FILE_COUNT_DEFAULT_VALUE);
+ this.maxSizeInBytes = getPropertyAsInt(properties.getProperty(MAX_SIZE_PROP_NAME), MAX_SIZE_DEFAULT_VALUE);
+ this.verbose = getPropertyAsBoolean(properties.getProperty(VERBOSE_PROP_NAME), VERBOSE_DEFAULT_VALUE);
+ this.allowed = getPropertyAsBoolean(properties.getProperty(ALLOWED_PROP_NAME), ALLOWED_DEFAULT_VALUE);
Review comment:
```suggestion
this.verbose = Boolean.parseBoolean(properties.getProperty(VERBOSE_PROP_NAME));
this.allowed = Boolean.parseBoolean(properties.getProperty(ALLOWED_PROP_NAME));
```
##########
File path: nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java
##########
@@ -575,11 +577,43 @@ public static long getContainerCapacity(final Path path) {
/**
* Returns the free capacity for a given path
+ *
* @param path path
* @return usable space
*/
public static long getContainerUsableSpace(final Path path) {
return path.toFile().getUsableSpace();
}
+ /**
+ * Returns the size for a given directory.
+ * @param path directory path
+ * @param logger logger
+ * @return the size in bytes
+ */
+ public static long getDirectorySize(Path path, final Logger logger) {
+ long size = 0;
+ try (Stream<Path> walk = Files.walk(path)) {
+ size = walk
+ .filter(Files::isRegularFile)
+ .mapToLong(getFileSizeByPathFunction(logger))
+ .sum();
+
+ } catch (IOException e) {
+ logger.error("IO exception occured while tried to get the size of the directory {}", path, e);
+ }
+ return size;
+ }
+
+ private static ToLongFunction<Path> getFileSizeByPathFunction(Logger logger) {
+ return p -> {
+ try {
+ return Files.size(p);
+ } catch (IOException e) {
+ logger.error("Failed to get size of directory {}", p, e);
+ return 0L;
+ }
+ };
+ }
Review comment:
Unnecessarily confusing abbreviation. Also this is called with regular files, not directories.
```suggestion
return path -> {
try {
return Files.size(path);
} catch (IOException e) {
logger.error("Failed to get size of file {}", path, e);
return 0L;
}
};
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org