You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2023/09/30 20:29:02 UTC
[nifi] branch main updated: NIFI-12143: This closes #7820. Introduced a new nifi-calcite-utils module and refactored the nifi-sql-reporting-tasks to mamke use of it. This uses the newer calcite api instead of deprecated classes/methods - Refactored QueryRecord / JoinEnrichment to use new nifi-calcite-utils
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 342fb0d876 NIFI-12143: This closes #7820. Introduced a new nifi-calcite-utils module and refactored the nifi-sql-reporting-tasks to mamke use of it. This uses the newer calcite api instead of deprecated classes/methods - Refactored QueryRecord / JoinEnrichment to use new nifi-calcite-utils
342fb0d876 is described below
commit 342fb0d876217b5eefa7571ffa0627f53273676d
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Fri Sep 29 14:11:04 2023 -0400
NIFI-12143: This closes #7820. Introduced a new nifi-calcite-utils module and refactored the nifi-sql-reporting-tasks to mamke use of it. This uses the newer calcite api instead of deprecated classes/methods
- Refactored QueryRecord / JoinEnrichment to use new nifi-calcite-utils
Signed-off-by: Joseph Witt <jo...@apache.org>
---
nifi-commons/nifi-calcite-utils/pom.xml | 47 ++++
.../main/java/org/apache/nifi/sql/ArrayType.java | 16 +-
.../java/org/apache/nifi/sql/CalciteDatabase.java | 219 +++++++++++++++
.../java/org/apache/nifi/sql/ColumnSchema.java | 100 +++++++
.../main/java/org/apache/nifi/sql/ColumnType.java | 20 +-
.../org/apache/nifi/sql/IterableRowStream.java | 56 ++++
.../src/main/java/org/apache/nifi/sql/MapType.java | 26 +-
.../main/java/org/apache/nifi/sql/NiFiTable.java | 173 ++++++++++++
.../java/org/apache/nifi/sql/NiFiTableSchema.java | 15 +-
.../org/apache/nifi/sql/ResettableDataSource.java | 33 +--
.../main/java/org/apache/nifi/sql/RowStream.java | 30 +-
.../main/java/org/apache/nifi/sql/ScalarType.java | 52 ++++
.../org/apache/nifi/sql/internal/CalciteTable.java | 90 ++++++
.../sql/internal/NiFiProjectTableScanRule.java | 122 +++++++++
.../nifi/sql/internal/NiFiTableEnumerator.java | 96 +++----
.../apache/nifi/sql/internal/NiFiTableScan.java | 97 +++++++
.../org/apache/nifi/sql/TestCalciteDatabase.java | 136 ++++++++++
nifi-commons/pom.xml | 1 +
.../serialization/record/MockRecordParser.java | 5 +-
.../nifi-sql-reporting-tasks/pom.xml | 5 +
.../apache/nifi/reporting/sql/CachedStatement.java | 14 +-
.../nifi/reporting/sql/MetricsSqlQueryService.java | 131 ++++-----
.../org/apache/nifi/reporting/sql/QueryResult.java | 2 +-
.../sql/bulletins/BulletinEnumerator.java | 153 -----------
.../bulletins/BulletinProjectTableScanRule.java | 76 ------
.../reporting/sql/bulletins/BulletinTable.java | 195 -------------
.../reporting/sql/bulletins/BulletinTableScan.java | 91 -------
.../ConnectionStatusEnumerator.java | 140 ----------
.../ConnectionStatusProjectTableScanRule.java | 76 ------
.../connectionstatus/ConnectionStatusTable.java | 205 --------------
.../ConnectionStatusTableScan.java | 91 -------
.../ConnectionStatusPredictionsEnumerator.java | 133 ---------
...ctionStatusPredictionsProjectTableScanRule.java | 76 ------
.../ConnectionStatusPredictionsTable.java | 182 -------------
.../ConnectionStatusPredictionsTableScan.java | 91 -------
.../sql/datasources/BulletinDataSource.java | 96 +++++++
.../datasources/ConnectionStatusDataSource.java | 123 +++++++++
.../ConnectionStatusPredictionDataSource.java | 102 +++++++
.../datasources/FlowConfigHistoryDataSource.java | 106 ++++++++
.../sql/datasources/GroupStatusCache.java | 49 ++++
.../sql/datasources/JvmMetricsDataSource.java | 113 ++++++++
.../datasources/ProcessGroupStatusDataSource.java | 132 +++++++++
.../sql/datasources/ProcessorStatusDataSource.java | 129 +++++++++
.../sql/datasources/ProvenanceDataSource.java | 161 +++++++++++
.../FlowConfigHistoryEnumerator.java | 151 -----------
.../FlowConfigHistoryProjectTableScanRule.java | 76 ------
.../flowconfighistory/FlowConfigHistoryTable.java | 211 ---------------
.../FlowConfigHistoryTableScan.java | 91 -------
.../sql/metrics/JvmMetricsEnumerator.java | 149 ----------
.../metrics/JvmMetricsProjectTableScanRule.java | 76 ------
.../reporting/sql/metrics/JvmMetricsTable.java | 208 --------------
.../reporting/sql/metrics/JvmMetricsTableScan.java | 91 -------
.../ProcessGroupStatusEnumerator.java | 168 ------------
.../ProcessGroupStatusProjectTableScanRule.java | 76 ------
.../ProcessGroupStatusTable.java | 209 --------------
.../ProcessGroupStatusTableScan.java | 91 -------
.../processorstatus/ProcessorStatusEnumerator.java | 175 ------------
.../ProcessorStatusProjectTableScanRule.java | 76 ------
.../sql/processorstatus/ProcessorStatusTable.java | 211 ---------------
.../processorstatus/ProcessorStatusTableScan.java | 91 -------
.../sql/provenance/ProvenanceEnumerator.java | 191 -------------
.../provenance/ProvenanceProjectTableScanRule.java | 76 ------
.../reporting/sql/provenance/ProvenanceTable.java | 214 ---------------
.../sql/provenance/ProvenanceTableScan.java | 91 -------
.../util/ConnectionStatusRecursiveIterator.java | 91 -------
.../reporting/sql/TestQueryNiFiReportingTask.java | 49 +++-
.../nifi-standard-processors/pom.xml | 5 +
.../nifi/processors/standard/JoinEnrichment.java | 13 -
.../nifi/processors/standard/QueryRecord.java | 170 +++++-------
.../standard/calcite/RecordPathFunctions.java | 24 +-
.../standard/enrichment/SqlJoinCache.java | 55 ++--
.../enrichment/SqlJoinCalciteParameters.java | 29 +-
.../standard/enrichment/SqlJoinStrategy.java | 15 +-
.../queryrecord/FlowFileProjectTableScanRule.java | 76 ------
.../org/apache/nifi/queryrecord/FlowFileTable.java | 301 ---------------------
.../apache/nifi/queryrecord/FlowFileTableScan.java | 91 -------
.../apache/nifi/queryrecord/RecordDataSource.java | 188 +++++++++++++
77 files changed, 2641 insertions(+), 5194 deletions(-)
diff --git a/nifi-commons/nifi-calcite-utils/pom.xml b/nifi-commons/nifi-calcite-utils/pom.xml
new file mode 100644
index 0000000000..7fe8191ebc
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/pom.xml
@@ -0,0 +1,47 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-calcite-utils</artifactId>
+ <name>nifi-calcite-utils</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ <version>1.35.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ArrayType.java
similarity index 63%
copy from nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
copy to nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ArrayType.java
index f8a1fccbd7..ba45f5963b 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ArrayType.java
@@ -14,12 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.reporting.sql;
-import java.io.Closeable;
-import java.sql.ResultSet;
+package org.apache.nifi.sql;
-interface QueryResult extends Closeable {
- ResultSet getResultSet();
- int getRecordsRead();
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.rel.type.RelDataType;
+
+public record ArrayType(ColumnType elementType) implements ColumnType {
+ @Override
+ public RelDataType getRelationalDataType(final JavaTypeFactory typeFactory) {
+ final RelDataType elementType = elementType().getRelationalDataType(typeFactory);
+ return typeFactory.createArrayType(elementType, -1);
+ }
}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/CalciteDatabase.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/CalciteDatabase.java
new file mode 100644
index 0000000000..f5d9574400
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/CalciteDatabase.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql;
+
+import org.apache.calcite.config.CalciteConnectionProperty;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.schema.FunctionContext;
+import org.apache.calcite.schema.impl.ScalarFunctionImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * <p>
+ * A simple abstraction for creating a database facade over data.
+ * </p>
+ *
+ * <p>
+ * While this abstraction does not provide all of the power that apache Calcite provides,
+ * it does provide the capabilities that are necessary for the uses cases where NiFi has made use of Calcite.
+ * This abstraction, however, greatly simplifies the API so that we can easily expose a SQL interface to data.
+ * </p>
+ *
+ * <p>
+ * The typical usage pattern for this class is as follows:
+ * </p>
+ *
+ * <pre>
+ * <code>
+ * try (final CalciteDatabase database = new CalciteDatabase()) {
+ * final ResettableDataSource dataSource = getDataSource();
+ * final NiFiTable firstTable = new NiFiTable("MY_TABLE", dataSource, getLogger());
+ * database.addTable(firstTable);
+ *
+ * final ResettableDataSource secondDataSource = getDataSource();
+ * final NiFiTable secondTable = new NiFiTable("YOUR_TABLE", secondDataSource, getLogger());
+ * database.addTable(secondTable);
+ *
+ * try (final PreparedStatement stmt = database.getConnection().prepareStatement("SELECT * FROM MY_TABLE")) {
+ * ...
+ * }
+ * }
+ * </code>
+ * </pre>
+ */
+public class CalciteDatabase implements Closeable {
+ private static final Logger logger = LoggerFactory.getLogger(CalciteDatabase.class);
+
+ private final CalciteConnection connection;
+ private final Map<String, NiFiTable> tables = new HashMap<>();
+
+ static {
+ try {
+ DriverManager.registerDriver(new org.apache.calcite.jdbc.Driver());
+ } catch (final Exception e) {
+ throw new RuntimeException("Could not initialize Calcite JDBC Driver");
+ }
+ }
+
+ /**
+ * Creates a new database using the default connection properties
+ *
+ * @throws SQLException if unable to create the database/connection
+ */
+ public CalciteDatabase() throws SQLException {
+ this(null);
+ }
+
+ /**
+ * Creates a new database with the given calcite connection properties
+ *
+ * @param calciteConnectionProperties the connection properties to provide to calcite
+ * @throws SQLException if unable to create the database/connection
+ */
+ public CalciteDatabase(final Properties calciteConnectionProperties) throws SQLException {
+ this.connection = createConnection(calciteConnectionProperties);
+ }
+
+ /**
+ * Adds the given table to the database
+ *
+ * @param table the table to add
+ */
+ public void addTable(final NiFiTable table) {
+ if (tables.containsKey(table.getName())) {
+ throw new IllegalStateException("Database already contains a table named " + table.getName());
+ }
+
+ connection.getRootSchema().add(table.getName(), table.createCalciteTable());
+ tables.put(table.getName(), table);
+ }
+
+ /**
+ * Retrieves the table with the given name
+ *
+ * @param tableName the name of the table
+ * @return the NiFiTable with the given name, or <code>null</code> if no table exists with the given name
+ */
+ public NiFiTable getTable(final String tableName) {
+ return tables.get(tableName);
+ }
+
+ /**
+ * @return all NiFiTables in the database
+ */
+ public Collection<NiFiTable> getTables() {
+ return Collections.unmodifiableCollection(tables.values());
+ }
+
+ /**
+ * Removes the table with the given name from the database
+ *
+ * @param tableName the name of the table
+ * @return the NiFiTable that was removed, or <code>null</code> if no table was removed
+ */
+ public NiFiTable removeTable(final String tableName) {
+ final NiFiTable table = tables.remove(tableName);
+ if (table == null) {
+ return null;
+ }
+
+ connection.getRootSchema().removeTable(tableName);
+ return table;
+ }
+
+ /**
+ * Adds a user-defined function (UDF) to the database with the given name.
+ * Note that Calcite requires that the Class have a default, zero-arg constructor,
+ * or a constructor that takes exactly one argument of type {@link FunctionContext}.
+ *
+ * @param functionName the name of the function
+ * @param functionClass the class that implements the function
+ * @param methodName the name of the method that implements the function
+ */
+ public void addUserDefinedFunction(final String functionName, final Class<?> functionClass, final String methodName) {
+ connection.getRootSchema().add(functionName, ScalarFunctionImpl.create(functionClass, methodName));
+ }
+
+ /**
+ * Adds a user-defined function (UDF) to the database with the given name.
+ * Note that Calcite requires that the Class that the given method belongs to have a default, zero-arg constructor,
+ * or a constructor that takes exactly one argument of type {@link FunctionContext}.
+ *
+ * @param functionName the name of the function
+ * @param method the method to invoke in order to call the function
+ */
+ public void addUserDefinedFunction(final String functionName, final Method method) {
+ connection.getRootSchema().add(functionName, ScalarFunctionImpl.create(method));
+ }
+
+ /**
+ * The database connection that can be used for creating Statements, etc.
+ *
+ * @return the database connection
+ */
+ public Connection getConnection() {
+ return connection;
+ }
+
+ private CalciteConnection createConnection(final Properties properties) throws SQLException {
+ final Properties calciteProperties;
+ if (properties == null) {
+ calciteProperties = new Properties();
+ calciteProperties.put(CalciteConnectionProperty.LEX.camelName(), Lex.MYSQL_ANSI.name());
+ } else {
+ calciteProperties = properties;
+ }
+
+ final Connection sqlConnection = DriverManager.getConnection("jdbc:calcite:", calciteProperties);
+ final CalciteConnection connection = sqlConnection.unwrap(CalciteConnection.class);
+ connection.getRootSchema().setCacheEnabled(false);
+
+ return connection;
+ }
+
+ @Override
+ public void close() throws IOException {
+ for (final NiFiTable table : getTables()) {
+ try {
+ table.close();
+ } catch (final Exception e) {
+ logger.warn("Encountered failure when attempting to close {}", table, e);
+ }
+ }
+
+ try {
+ connection.close();
+ } catch (final SQLException e) {
+ throw new IOException(e);
+ }
+ }
+}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ColumnSchema.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ColumnSchema.java
new file mode 100644
index 0000000000..3584e02849
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ColumnSchema.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql;
+
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.rel.type.RelDataType;
+
+import java.util.Objects;
+
+/**
+ * A schema that describes a single column in a {@link NiFiTable}
+ */
+public class ColumnSchema {
+ private final String name;
+ private final ColumnType columnType;
+ private final boolean nullable;
+
+ /**
+ * <p>
+ * Convenience method to create a schema for a column with a scalar value.
+ * </p>
+ *
+ * <p>
+ * This is equivalent to calling <code>new ColumnSchema(name, new ScalarType(type), nullable);</code>
+ * </p>
+ *
+ * @param name the name of the column
+ * @param type the type of the column
+ * @param nullable whether or not the column supports null values
+ */
+ public ColumnSchema(final String name, final Class<?> type, final boolean nullable) {
+ this(name, new ScalarType(type), nullable);
+ }
+
+ /**
+ * Creates a column schema with the given name and type that is optionally nullable
+ *
+ * @param name the name of the column
+ * @param type the type of the column
+ * @param nullable whether or not the column supports null values
+ */
+ public ColumnSchema(final String name, final ColumnType type, final boolean nullable) {
+ this.name = Objects.requireNonNull(name);
+ this.columnType = type;
+ this.nullable = nullable;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ RelDataType toRelationalDataType(final JavaTypeFactory typeFactory) {
+ final RelDataType javaType = columnType.getRelationalDataType(typeFactory);
+ return typeFactory.createTypeWithNullability(javaType, nullable);
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ final ColumnSchema that = (ColumnSchema) other;
+ return nullable == that.nullable
+ && Objects.equals(name, that.name)
+ && Objects.equals(columnType, that.columnType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, columnType, nullable);
+ }
+
+ @Override
+ public String toString() {
+ return "ColumnSchema[name='" + name + "', columnType=" + columnType + ", nullable=" + nullable + "]";
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ColumnType.java
similarity index 63%
copy from nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
copy to nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ColumnType.java
index f8a1fccbd7..aebef6b90f 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ColumnType.java
@@ -14,12 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.reporting.sql;
-import java.io.Closeable;
-import java.sql.ResultSet;
+package org.apache.nifi.sql;
+
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.rel.type.RelDataType;
+
+public interface ColumnType {
+
+ /**
+ * Converts the column into the {@link RelDataType} that Calcite requires
+ *
+ * @param typeFactory the type factory to use for creating the RelDataType
+ * @return the appropriate RelDataType for Calcite
+ */
+ RelDataType getRelationalDataType(JavaTypeFactory typeFactory);
-interface QueryResult extends Closeable {
- ResultSet getResultSet();
- int getRecordsRead();
}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/IterableRowStream.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/IterableRowStream.java
new file mode 100644
index 0000000000..172b55d46a
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/IterableRowStream.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql;
+
+import java.util.Iterator;
+import java.util.function.Function;
+
+/**
+ * A convenience implementation of RowStream that can be used to convert any {@link Iterable} to a RowStream.
+ *
+ * @param <T> the type object in the Iterable
+ */
+public class IterableRowStream<T> implements RowStream {
+ private final Iterator<T> iterator;
+ private final Function<T, Object[]> transform;
+
+ /**
+ * Creates an IterableRowStream that is capable of iterating over the objects in the given {@link Iterable}
+ *
+ * @param iterable an Iterable that is capable of iterating over objects
+ * @param transform a transformation function that is capable of converting the Objects in the Iterable to the <code>Object[]</code> necessary for Calcite.
+ */
+ public IterableRowStream(final Iterable<T> iterable, final Function<T, Object[]> transform) {
+ this.iterator = iterable.iterator();
+ this.transform = transform;
+ }
+
+ @Override
+ public Object[] nextRow() {
+ if (!iterator.hasNext()) {
+ return null;
+ }
+
+ final T value = iterator.next();
+ return transform.apply(value);
+ }
+
+ @Override
+ public void close() {
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/MapType.java
similarity index 59%
copy from nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
copy to nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/MapType.java
index 345de7582d..efbc1600e4 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/MapType.java
@@ -14,25 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.reporting.sql;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
+package org.apache.nifi.sql;
-class CachedStatement {
- private final PreparedStatement statement;
- private final Connection connection;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.rel.type.RelDataType;
- CachedStatement(final PreparedStatement statement, final Connection connection) {
- this.statement = statement;
- this.connection = connection;
- }
-
- PreparedStatement getStatement() {
- return statement;
- }
-
- Connection getConnection() {
- return connection;
+public record MapType(ColumnType keyType, ColumnType valueType) implements ColumnType {
+ @Override
+ public RelDataType getRelationalDataType(final JavaTypeFactory typeFactory) {
+ final RelDataType keyType = keyType().getRelationalDataType(typeFactory);
+ final RelDataType valueType = valueType().getRelationalDataType(typeFactory);
+ return typeFactory.createMapType(keyType, valueType);
}
}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/NiFiTable.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/NiFiTable.java
new file mode 100644
index 0000000000..ad4ce3c891
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/NiFiTable.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql;
+
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.linq4j.AbstractEnumerable;
+import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.util.Pair;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.sql.internal.CalciteTable;
+import org.apache.nifi.sql.internal.NiFiTableEnumerator;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A table that can be queried in a {@link CalciteDatabase}
+ */
+public class NiFiTable implements Closeable {
+ private final String name;
+ private final NiFiTableSchema tableSchema;
+ private final ComponentLog logger;
+
+ private final Set<NiFiTableEnumerator> enumerators = Collections.synchronizedSet(new HashSet<>());
+
+ private volatile ResettableDataSource dataSource;
+ private volatile int maxRecordsRead;
+
+
+ /**
+ * Created a NiFi table with a datasource already established. The table's schema will be established as the schema
+ * of the datasource.
+ *
+ * @param name the name of the table. This is the name that should be referenced in SQL statements.
+ * @param dataSource the source of data
+ * @param logger the logger to use
+ */
+ public NiFiTable(final String name, final ResettableDataSource dataSource, final ComponentLog logger) {
+ this(name, dataSource.getSchema(), logger);
+ this.dataSource = dataSource;
+ }
+
+ /**
+ * Created a NiFi table without yet establishing the datasource. This table cannot be queried until the datasource
+ * is established by calling {@link #setDataSource(ResettableDataSource)}.
+ *
+ * @param name the name of the table. This is the name that should be referenced in SQL statements.
+ * @param schema the table's schema
+ * @param logger the logger to use
+ */
+ public NiFiTable(final String name, final NiFiTableSchema schema, final ComponentLog logger) {
+ this.name = name;
+ this.tableSchema = schema;
+ this.logger = logger;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Updates the datasource to use for retrieving rows of data. The given data source must have the same schema as was provided when the
+ * table was created.
+ *
+ * @param dataSource the source of data
+ */
+ public void setDataSource(final ResettableDataSource dataSource) {
+ if (!tableSchema.equals(dataSource.getSchema())) {
+ throw new IllegalArgumentException("Cannot update data source because the newly provided data source [%s] has a different schema than the current data source [%s]".formatted(
+ dataSource, this.dataSource));
+ }
+
+ this.dataSource = dataSource;
+ maxRecordsRead = 0;
+ }
+
+ /**
+ * @return the number of records that were read from the datasource
+ */
+ public int getRecordsRead() {
+ return maxRecordsRead;
+ }
+
+ CalciteTable createCalciteTable() {
+ return new CalciteTable(this::createTableDefinition, NiFiTableEnumerable::new);
+ }
+
+ private RelDataType createTableDefinition(final RelDataTypeFactory typeFactory) {
+ final List<String> names = new ArrayList<>();
+ final List<RelDataType> types = new ArrayList<>();
+
+ final JavaTypeFactory javaTypeFactory = (JavaTypeFactory) typeFactory;
+ for (final ColumnSchema column : tableSchema.columns()) {
+ names.add(column.getName());
+ types.add(column.toRelationalDataType(javaTypeFactory));
+ }
+
+ return typeFactory.createStructType(Pair.zip(names, types));
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ final NiFiTable nifiTable = (NiFiTable) o;
+ return Objects.equals(name, nifiTable.name) && Objects.equals(tableSchema, nifiTable.tableSchema);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableSchema);
+ }
+
+ @Override
+ public String toString() {
+ return "NiFiTable[name=" + name + "]";
+ }
+
+ public void close() {
+ for (final NiFiTableEnumerator enumerator : enumerators) {
+ enumerator.close();
+ }
+ }
+
+
+ private class NiFiTableEnumerable extends AbstractEnumerable<Object> {
+ private final int[] fields;
+
+ public NiFiTableEnumerable(final int[] fields) {
+ this.fields = fields;
+ }
+
+ public Enumerator<Object> enumerator() {
+ final NiFiTableEnumerator flowFileEnumerator = new NiFiTableEnumerator(dataSource, logger, fields, this::onFinish, enumerators::remove);
+ enumerators.add(flowFileEnumerator);
+ return flowFileEnumerator;
+ }
+
+ private void onFinish() {
+ final int recordCount = maxRecordsRead;
+ if (recordCount > maxRecordsRead) {
+ maxRecordsRead = recordCount;
+ }
+ }
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/NiFiTableSchema.java
similarity index 79%
copy from nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
copy to nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/NiFiTableSchema.java
index f8a1fccbd7..816b54001a 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/NiFiTableSchema.java
@@ -14,12 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.reporting.sql;
-import java.io.Closeable;
-import java.sql.ResultSet;
+package org.apache.nifi.sql;
-interface QueryResult extends Closeable {
- ResultSet getResultSet();
- int getRecordsRead();
+import java.util.List;
+
+/**
+ * The schema for a {@link NiFiTable}
+ *
+ * @param columns a List of columns in the table
+ */
+public record NiFiTableSchema(List<ColumnSchema> columns) {
}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ResettableDataSource.java
similarity index 58%
copy from nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
copy to nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ResettableDataSource.java
index 345de7582d..e1075b95a1 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ResettableDataSource.java
@@ -14,25 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.reporting.sql;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
+package org.apache.nifi.sql;
-class CachedStatement {
- private final PreparedStatement statement;
- private final Connection connection;
+import java.io.IOException;
- CachedStatement(final PreparedStatement statement, final Connection connection) {
- this.statement = statement;
- this.connection = connection;
- }
+/**
+ * A Source of data for a database
+ */
+public interface ResettableDataSource {
- PreparedStatement getStatement() {
- return statement;
- }
+ /**
+ * @return the schema that the rows returned by the {@link RowStream} will adhere to
+ */
+ NiFiTableSchema getSchema();
- Connection getConnection() {
- return connection;
- }
+ /**
+ * Resets the data source and establishes a {@link RowStream} that can be used for iterating over data
+ *
+ * @return the newly established RowStream
+ * @throws IOException if unable to obtain a RowStream due to IO failures
+ */
+ RowStream reset() throws IOException;
}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/RowStream.java
similarity index 55%
copy from nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
copy to nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/RowStream.java
index 345de7582d..3eafc7d358 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/RowStream.java
@@ -14,25 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.reporting.sql;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
+package org.apache.nifi.sql;
-class CachedStatement {
- private final PreparedStatement statement;
- private final Connection connection;
+import java.io.IOException;
- CachedStatement(final PreparedStatement statement, final Connection connection) {
- this.statement = statement;
- this.connection = connection;
- }
+/**
+ * An interface that is responsible for returning rows of data
+ */
+public interface RowStream extends AutoCloseable {
- PreparedStatement getStatement() {
- return statement;
- }
+ /**
+ * Returns the next row of data as an object array, or <code>null</code> if the stream is out of data.
+ * Note that the objects returned MUST adhere to the schema of the {@link ResettableDataSource} that created this RowStream.
+ *
+ * @return the next row of data, or <code>null</code> if there is no more data
+ * @throws IOException if unable to obtain the next row of data due to I/O failure
+ */
+ Object[] nextRow() throws IOException;
- Connection getConnection() {
- return connection;
- }
}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ScalarType.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ScalarType.java
new file mode 100644
index 0000000000..3ebffad5ae
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/ScalarType.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql;
+
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.rel.type.RelDataType;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+
+public record ScalarType(Class<?> scalarType) implements ColumnType {
+
+ public static final ScalarType STRING = new ScalarType(String.class);
+ public static final ScalarType INTEGER = new ScalarType(Integer.class);
+ public static final ScalarType LONG = new ScalarType(Long.class);
+ public static final ScalarType BOOLEAN = new ScalarType(Boolean.class);
+ public static final ScalarType SHORT = new ScalarType(Short.class);
+ public static final ScalarType BYTE = new ScalarType(Byte.class);
+ public static final ScalarType CHARACTER = new ScalarType(Character.class);
+ public static final ScalarType DOUBLE = new ScalarType(Double.class);
+ public static final ScalarType FLOAT = new ScalarType(Float.class);
+ public static final ScalarType DECIMAL = new ScalarType(BigDecimal.class);
+ public static final ScalarType BIGINT = new ScalarType(BigInteger.class);
+ public static final ScalarType DATE = new ScalarType(Date.class);
+ public static final ScalarType TIME = new ScalarType(Time.class);
+ public static final ScalarType TIMESTAMP = new ScalarType(Timestamp.class);
+ public static final ScalarType UUID = new ScalarType(java.util.UUID.class);
+ public static final ScalarType OBJECT = new ScalarType(Object.class);
+
+ @Override
+ public RelDataType getRelationalDataType(final JavaTypeFactory typeFactory) {
+ return typeFactory.createJavaType(scalarType());
+ }
+}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/CalciteTable.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/CalciteTable.java
new file mode 100644
index 0000000000..5bbb452a3a
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/CalciteTable.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql.internal;
+
+import org.apache.calcite.DataContext;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.QueryProvider;
+import org.apache.calcite.linq4j.Queryable;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.QueryableTable;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Schemas;
+import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.schema.impl.AbstractTable;
+
+import java.lang.reflect.Type;
+import java.util.function.Function;
+
+/**
+ * <p>
+ * An internal representation of a NiFiTable that is necessary for Apache Calcite.
+ * This class never be referenced outside of the module in which it is defined.
+ * </p>
+ */
+public class CalciteTable extends AbstractTable implements QueryableTable, TranslatableTable {
+ private final Function<RelDataTypeFactory, RelDataType> tableDefinitionFactory;
+ private final Function<int[], Enumerable<Object>> projectionFactory;
+
+ public CalciteTable(final Function<RelDataTypeFactory, RelDataType> tableDefinitionFactory, final Function<int[], Enumerable<Object>> projectionFactory) {
+ this.tableDefinitionFactory = tableDefinitionFactory;
+ this.projectionFactory = projectionFactory;
+ }
+
+ // Returns an enumerable over a given projection of the fields.
+ // Called from generated code. While the DataContent is not used, it is provided by the calling code and must be present.
+ public Enumerable<Object> project(final DataContext dataContext, final int[] fields) {
+ return projectionFactory.apply(fields);
+ }
+
+ @Override
+ public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
+ return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
+ }
+
+ @Override
+ public Type getElementType() {
+ return Object[].class;
+ }
+
+ @Override
+ public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
+ // Request all fields.
+ final int fieldCount = relOptTable.getRowType().getFieldCount();
+ final int[] fields = new int[fieldCount];
+ for (int i = 0; i < fieldCount; i++) {
+ fields[i] = i;
+ }
+
+ return new NiFiTableScan(context.getCluster(), relOptTable, fields);
+ }
+
+ @Override
+ public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
+ return tableDefinitionFactory.apply(typeFactory);
+ }
+}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiProjectTableScanRule.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiProjectTableScanRule.java
new file mode 100644
index 0000000000..0bc0d74526
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiProjectTableScanRule.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql.internal;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelRule;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+import java.util.List;
+
+class NiFiProjectTableScanRule extends RelRule<NiFiProjectTableScanRule.Config> {
+
+ NiFiProjectTableScanRule(final Config config) {
+ super(config);
+ }
+
+ @Override
+ public void onMatch(final RelOptRuleCall call) {
+ final LogicalProject project = call.rel(0);
+ final NiFiTableScan scan = call.rel(1);
+
+ final int[] fields = getProjectionFields(project.getProjects());
+ if (fields == null) {
+ // Project contains expressions more complex than just field references.
+ return;
+ }
+
+ final NiFiTableScan tableScan = new NiFiTableScan(scan.getCluster(), scan.getTable(), fields);
+ call.transformTo(tableScan);
+ }
+
+
+ private static int[] getProjectionFields(final List<RexNode> expressions) {
+ final int[] fields = new int[expressions.size()];
+ for (int i = 0; i < expressions.size(); i++) {
+ final RexNode exp = expressions.get(i);
+
+ if (exp instanceof RexInputRef) {
+ fields[i] = ((RexInputRef) exp).getIndex();
+ } else {
+ // not a simple projection
+ return null;
+ }
+ }
+
+ return fields;
+ }
+
+ public interface Config extends RelRule.Config {
+ // This impl comes directly from the Calcite documentation.
+ Config DEFAULT = new StandardConfig()
+ .withOperandSupplier(b0 -> b0.operand(LogicalProject.class).oneInput(b1 ->
+ b1.operand(NiFiTableScan.class).noInputs()));
+
+
+ @Override
+ default NiFiProjectTableScanRule toRule() {
+ return new NiFiProjectTableScanRule(this);
+ }
+ }
+
+ private static class StandardConfig implements Config {
+ private RelBuilderFactory relBuilderFactory = RelFactories.LOGICAL_BUILDER;
+ private String description;
+ private OperandTransform operandTransform;
+
+
+ @Override
+ public StandardConfig withRelBuilderFactory(final RelBuilderFactory factory) {
+ this.relBuilderFactory = factory;
+ return this;
+ }
+
+ @Override
+ public StandardConfig withDescription(final String description) {
+ this.description = description;
+ return this;
+ }
+
+ @Override
+ public StandardConfig withOperandSupplier(final OperandTransform transform) {
+ this.operandTransform = transform;
+ return this;
+ }
+
+ @Override
+ public RelBuilderFactory relBuilderFactory() {
+ return relBuilderFactory;
+ }
+
+ @Override
+ public String description() {
+ return description;
+ }
+
+
+ @Override
+ public OperandTransform operandSupplier() {
+ return this.operandTransform;
+ }
+ }
+
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileEnumerator.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiTableEnumerator.java
similarity index 62%
rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileEnumerator.java
rename to nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiTableEnumerator.java
index 1a0d2ba2ec..1ab1a62dbd 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileEnumerator.java
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiTableEnumerator.java
@@ -15,57 +15,59 @@
* limitations under the License.
*/
-package org.apache.nifi.queryrecord;
+package org.apache.nifi.sql.internal;
import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.serialization.RecordReader;
-import org.apache.nifi.serialization.RecordReaderFactory;
-import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
-import java.io.InputStream;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.List;
+import java.util.function.Consumer;
-public class FlowFileEnumerator implements Enumerator<Object> {
- private final ProcessSession session;
- private final FlowFile flowFile;
+public class NiFiTableEnumerator implements Enumerator<Object> {
+ private final ResettableDataSource dataSource;
private final ComponentLog logger;
- private final RecordReaderFactory recordParserFactory;
private final int[] fields;
+ private final Runnable onFinishCallback;
+ private final Consumer<NiFiTableEnumerator> onCloseCallback;
- private InputStream rawIn;
+
+ private RowStream rowStream;
private Object currentRow;
- private RecordReader recordParser;
private int recordsRead = 0;
- public FlowFileEnumerator(final ProcessSession session, final FlowFile flowFile, final ComponentLog logger, final RecordReaderFactory parserFactory, final int[] fields) {
- this.session = session;
- this.flowFile = flowFile;
- this.recordParserFactory = parserFactory;
+
+ public NiFiTableEnumerator(final ResettableDataSource dataSource, final ComponentLog logger, final int[] fields, final Runnable onFinishCallback,
+ final Consumer<NiFiTableEnumerator> onCloseCallback) {
+ this.dataSource = dataSource;
this.logger = logger;
this.fields = fields;
+ this.onFinishCallback = onFinishCallback;
+ this.onCloseCallback = onCloseCallback;
reset();
}
+
@Override
public Object current() {
return currentRow;
}
+
@Override
public boolean moveNext() {
currentRow = null;
try {
- final Record record = recordParser.nextRecord();
- if (record == null) {
+ final Object[] row = rowStream.nextRow();
+ if (row == null) {
// If we are out of data, close the InputStream. We do this because
// Calcite does not necessarily call our close() method.
close();
+
try {
onFinish();
} catch (final Exception e) {
@@ -75,31 +77,32 @@ public class FlowFileEnumerator implements Enumerator<Object> {
return false;
}
- currentRow = filterColumns(record);
+ currentRow = filterColumns(row);
} catch (final Exception e) {
- throw new ProcessException("Failed to read next record in stream for " + flowFile + " due to " + e.getMessage(), e);
+ throw new ProcessException("Failed to read next row in stream", e);
}
recordsRead++;
return true;
}
- protected int getRecordsRead() {
+ public int getRecordsRead() {
return recordsRead;
}
- protected void onFinish() {
+ private void onFinish() {
+ if (onFinishCallback != null) {
+ onFinishCallback.run();
+ }
}
- private Object filterColumns(final Record record) {
- if (record == null) {
+ private Object filterColumns(final Object[] row) {
+ if (row == null) {
return null;
}
- final Object[] row = record.getValues();
-
// If we want no fields or if the row is null, just return null
- if (fields == null || row == null) {
+ if (fields == null) {
return row;
}
@@ -136,37 +139,36 @@ public class FlowFileEnumerator implements Enumerator<Object> {
@Override
public void reset() {
- if (rawIn != null) {
+ if (rowStream != null) {
try {
- rawIn.close();
+ rowStream.close();
} catch (final Exception e) {
- logger.warn("Could not close FlowFile's input due to " + e, e);
+ logger.warn("Could not close data stream {}", rowStream, e);
}
}
- rawIn = session.read(flowFile);
-
try {
- recordParser = recordParserFactory.createRecordReader(flowFile, rawIn, logger);
+ rowStream = dataSource.reset();
} catch (final Exception e) {
- throw new ProcessException("Failed to reset stream", e);
+ throw new RuntimeException("Failed to data stream from " + dataSource, e);
}
}
- @Override
- public void close() {
- if (recordParser != null) {
- try {
- recordParser.close();
- } catch (final Exception e) {
- logger.warn("Failed to close decorated source for " + flowFile, e);
- }
- }
+ @Override
+ public final void close() {
try {
- rawIn.close();
- } catch (final Exception e) {
- logger.warn("Failed to close InputStream for " + flowFile, e);
+ if (onCloseCallback != null) {
+ onCloseCallback.accept(this);
+ }
+ } finally {
+ if (rowStream != null) {
+ try {
+ rowStream.close();
+ } catch (Exception e) {
+ logger.warn("Failed to close {}", rowStream, e);
+ }
+ }
}
}
}
diff --git a/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiTableScan.java b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiTableScan.java
new file mode 100644
index 0000000000..f68ba35bfa
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/main/java/org/apache/nifi/sql/internal/NiFiTableScan.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql.internal;
+
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
+import org.apache.calcite.adapter.enumerable.PhysType;
+import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.linq4j.tree.BlockStatement;
+import org.apache.calcite.linq4j.tree.Blocks;
+import org.apache.calcite.linq4j.tree.ConstantExpression;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MethodCallExpression;
+import org.apache.calcite.linq4j.tree.ParameterExpression;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+
+import java.util.Collections;
+import java.util.List;
+
+class NiFiTableScan extends TableScan implements EnumerableRel {
+ private final ConstantExpression fieldExpression;
+ private final int[] fields;
+
+ protected NiFiTableScan(final RelOptCluster cluster, final RelOptTable table, final int[] fields) {
+ super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), Collections.emptyList(), table);
+ this.fields = fields;
+ fieldExpression = Expressions.constant(fields);
+ }
+
+ @Override
+ public void register(final RelOptPlanner planner) {
+ planner.addRule(new NiFiProjectTableScanRule(NiFiProjectTableScanRule.Config.DEFAULT));
+ }
+
+ @Override
+ public RelDataType deriveRowType() {
+ final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
+ final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
+ for (final int field : fields) {
+ builder.add(fieldList.get(field));
+ }
+ return builder.build();
+ }
+
+ // Logic taken directly from the Calcite CSV Tutorial
+ @Override
+ public RelOptCost computeSelfCost(final RelOptPlanner planner, final RelMetadataQuery mq) {
+ // Multiply the cost by a factor that makes a scan more attractive if it
+ // has significantly fewer fields than the original scan.
+ //
+ // The "+ 2D" on top and bottom keeps the function fairly smooth.
+ //
+ // For example, if table has 3 fields, project has 1 field,
+ // then factor = (1 + 2) / (3 + 2) = 0.6
+ return super.computeSelfCost(planner, mq)
+ .multiplyBy(((double) fields.length + 2D)
+ / ((double) table.getRowType().getFieldCount() + 2D));
+ }
+
+ @Override
+ public Result implement(final EnumerableRelImplementor implementor, final Prefer pref) {
+ final JavaTypeFactory typeFactory = implementor.getTypeFactory();
+ final ParameterExpression rootExpression = implementor.getRootExpression();
+ final PhysType physType = PhysTypeImpl.of(typeFactory, getRowType(), pref.preferArray());
+
+ final Expression tableExpression = table.getExpression(CalciteTable.class);
+ final MethodCallExpression methodCallExpression = Expressions.call(tableExpression, "project", rootExpression, fieldExpression);
+ final BlockStatement blockStatement = Blocks.toBlock(methodCallExpression);
+ return implementor.result(physType, blockStatement);
+ }
+}
diff --git a/nifi-commons/nifi-calcite-utils/src/test/java/org/apache/nifi/sql/TestCalciteDatabase.java b/nifi-commons/nifi-calcite-utils/src/test/java/org/apache/nifi/sql/TestCalciteDatabase.java
new file mode 100644
index 0000000000..019731147a
--- /dev/null
+++ b/nifi-commons/nifi-calcite-utils/src/test/java/org/apache/nifi/sql/TestCalciteDatabase.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.sql;
+
+import org.apache.nifi.logging.ComponentLog;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+public class TestCalciteDatabase {
+
+ @Test
+ public void testSimpleQuery() throws SQLException, IOException {
+ try (final CalciteDatabase database = createNameNumberDatabase();
+ final PreparedStatement stmt = database.getConnection().prepareStatement("SELECT * FROM CANNED_DATA WHERE number > 100");
+ final ResultSet resultSet = stmt.executeQuery()) {
+
+ assertTrue(resultSet.next());
+
+ assertEquals("Mark", resultSet.getString("name"));
+ assertEquals("Mark", resultSet.getString(1));
+ assertEquals(192, resultSet.getInt(2));
+ assertEquals(192, resultSet.getInt("number"));
+
+ assertTrue(resultSet.next());
+ assertEquals("Jane Doe", resultSet.getString("name"));
+ assertEquals("Jane Doe", resultSet.getString(1));
+ assertEquals(2910, resultSet.getInt(2));
+ assertEquals(2910, resultSet.getInt("number"));
+
+ assertFalse(resultSet.next());
+ }
+ }
+
+ @Test
+ public void testAddFunction() throws SQLException, NoSuchMethodException, IOException {
+ try (final CalciteDatabase database = createNameNumberDatabase()) {
+ database.addUserDefinedFunction("UPS", ToUpperCase.class, "invoke");
+ database.addUserDefinedFunction("UPS_METHOD", ToUpperCase.class.getDeclaredMethod("invoke", String.class));
+
+ final String[] queries = new String[]{
+ "SELECT UPS(name) AS UP FROM CANNED_DATA WHERE name NOT LIKE '% %'",
+ "SELECT UPS_METHOD(name) AS UP FROM CANNED_DATA WHERE name NOT LIKE '% %'"
+ };
+
+ for (final String query : queries) {
+ try (final PreparedStatement stmt = database.getConnection().prepareStatement(query);
+ final ResultSet resultSet = stmt.executeQuery()) {
+
+ assertTrue(resultSet.next());
+ assertEquals("MARK", resultSet.getString(1));
+ assertEquals("MARK", resultSet.getString("UP"));
+
+ assertTrue(resultSet.next());
+ assertEquals("OTHER", resultSet.getString(1));
+ assertEquals("OTHER", resultSet.getString("UP"));
+
+ assertFalse(resultSet.next());
+ }
+ }
+ }
+ }
+
+ public static class ToUpperCase {
+ public String invoke(final String value) {
+ return value.toUpperCase();
+ }
+ }
+
+ private CalciteDatabase createNameNumberDatabase() throws SQLException {
+ final CalciteDatabase database = new CalciteDatabase();
+
+ final NiFiTableSchema tableSchema = new NiFiTableSchema(List.of(
+ new ColumnSchema("name", String.class, false),
+ new ColumnSchema("number", int.class, false)
+ ));
+
+ final List<Object[]> rows = List.of(
+ new Object[] { "Mark", 192 },
+ new Object[] { "John Doe", 41 },
+ new Object[] { "Jane Doe", 2910 },
+ new Object[] { "Other", -42 }
+ );
+ final ListDataSource arrayListDataSource = new ListDataSource(tableSchema, rows);
+
+ final NiFiTable table = new NiFiTable("CANNED_DATA", arrayListDataSource, mock(ComponentLog.class));
+ database.addTable(table);
+
+ return database;
+ }
+
+
+ private static class ListDataSource implements ResettableDataSource {
+ private final NiFiTableSchema schema;
+ private final List<Object[]> rows;
+
+ public ListDataSource(final NiFiTableSchema schema, final List<Object[]> rows) {
+ this.schema = schema;
+ this.rows = rows;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return schema;
+ }
+
+ @Override
+ public RowStream reset() {
+ return new IterableRowStream<>(rows, row -> row);
+ }
+ }
+}
diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml
index deb6ff3698..2e26b83334 100644
--- a/nifi-commons/pom.xml
+++ b/nifi-commons/pom.xml
@@ -76,5 +76,6 @@
<module>nifi-web-utils</module>
<module>nifi-write-ahead-log</module>
<module>nifi-xml-processing</module>
+ <module>nifi-calcite-utils</module>
</modules>
</project>
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordParser.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordParser.java
index e52de8c732..7a334b0220 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordParser.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordParser.java
@@ -73,7 +73,9 @@ public class MockRecordParser extends AbstractControllerService implements Recor
}
@Override
- public RecordReader createRecordReader(Map<String, String> variables, InputStream in, long inputLength, ComponentLog logger) throws IOException, SchemaNotFoundException {
+ public RecordReader createRecordReader(final Map<String, String> variables, final InputStream in, final long inputLength, final ComponentLog logger)
+ throws IOException, SchemaNotFoundException {
+
final Iterator<Object[]> itr = records.iterator();
return new RecordReader() {
@@ -81,6 +83,7 @@ public class MockRecordParser extends AbstractControllerService implements Recor
@Override
public void close() throws IOException {
+ in.close();
}
@Override
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml
index f710dab1c5..9ab3742e4d 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml
@@ -59,6 +59,11 @@
<artifactId>nifi-record-sink-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-calcite-utils</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
index 345de7582d..d7072c433e 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/CachedStatement.java
@@ -16,16 +16,18 @@
*/
package org.apache.nifi.reporting.sql;
+import org.apache.nifi.sql.CalciteDatabase;
+
import java.sql.Connection;
import java.sql.PreparedStatement;
class CachedStatement {
private final PreparedStatement statement;
- private final Connection connection;
+ private final CalciteDatabase database;
- CachedStatement(final PreparedStatement statement, final Connection connection) {
+ CachedStatement(final PreparedStatement statement, final CalciteDatabase database) {
this.statement = statement;
- this.connection = connection;
+ this.database = database;
}
PreparedStatement getStatement() {
@@ -33,6 +35,10 @@ class CachedStatement {
}
Connection getConnection() {
- return connection;
+ return database.getConnection();
+ }
+
+ CalciteDatabase getDatabase() {
+ return database;
}
}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsSqlQueryService.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsSqlQueryService.java
index 867b9038bf..46789e1963 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsSqlQueryService.java
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsSqlQueryService.java
@@ -19,33 +19,31 @@ package org.apache.nifi.reporting.sql;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
-import org.apache.calcite.config.CalciteConnectionProperty;
-import org.apache.calcite.config.Lex;
-import org.apache.calcite.jdbc.CalciteConnection;
-import org.apache.calcite.schema.SchemaPlus;
import org.apache.nifi.avro.AvroTypeUtil;
import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.reporting.ReportingContext;
-import org.apache.nifi.reporting.sql.bulletins.BulletinTable;
-import org.apache.nifi.reporting.sql.connectionstatus.ConnectionStatusTable;
-import org.apache.nifi.reporting.sql.connectionstatuspredictions.ConnectionStatusPredictionsTable;
-import org.apache.nifi.reporting.sql.flowconfighistory.FlowConfigHistoryTable;
-import org.apache.nifi.reporting.sql.metrics.JvmMetricsTable;
-import org.apache.nifi.reporting.sql.processgroupstatus.ProcessGroupStatusTable;
-import org.apache.nifi.reporting.sql.processorstatus.ProcessorStatusTable;
-import org.apache.nifi.reporting.sql.provenance.ProvenanceTable;
+import org.apache.nifi.reporting.sql.datasources.BulletinDataSource;
+import org.apache.nifi.reporting.sql.datasources.ConnectionStatusDataSource;
+import org.apache.nifi.reporting.sql.datasources.ConnectionStatusPredictionDataSource;
+import org.apache.nifi.reporting.sql.datasources.FlowConfigHistoryDataSource;
+import org.apache.nifi.reporting.sql.datasources.GroupStatusCache;
+import org.apache.nifi.reporting.sql.datasources.JvmMetricsDataSource;
+import org.apache.nifi.reporting.sql.datasources.ProcessGroupStatusDataSource;
+import org.apache.nifi.reporting.sql.datasources.ProcessorStatusDataSource;
+import org.apache.nifi.reporting.sql.datasources.ProvenanceDataSource;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.ResultSetRecordSet;
+import org.apache.nifi.sql.CalciteDatabase;
+import org.apache.nifi.sql.NiFiTable;
+import org.apache.nifi.sql.ResettableDataSource;
import org.apache.nifi.util.db.JdbcCommon;
import java.io.IOException;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Properties;
+import java.time.Duration;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Supplier;
@@ -64,12 +62,6 @@ public class MetricsSqlQueryService implements MetricsQueryService {
public MetricsSqlQueryService(ComponentLog logger, final int defaultPrecision, final int defaultScale) {
this.defaultPrecision = defaultPrecision;
this.defaultScale = defaultScale;
- try {
- DriverManager.registerDriver(new org.apache.calcite.jdbc.Driver());
- } catch (final SQLException e) {
- throw new ProcessException("Failed to load Calcite JDBC Driver", e);
- }
-
this.logger = logger;
}
@@ -77,9 +69,7 @@ public class MetricsSqlQueryService implements MetricsQueryService {
return logger;
}
- public QueryResult query(final ReportingContext context, final String sql)
- throws Exception {
-
+ public QueryResult query(final ReportingContext context, final String sql) throws Exception {
final Supplier<CachedStatement> statementBuilder = () -> {
try {
return buildCachedStatement(sql, context);
@@ -148,59 +138,55 @@ public class MetricsSqlQueryService implements MetricsQueryService {
}
private CachedStatement buildCachedStatement(final String sql, final ReportingContext context) throws Exception {
+ final CalciteDatabase database = new CalciteDatabase();
+
+ final GroupStatusCache groupStatusCache = new GroupStatusCache(Duration.ofSeconds(60));
- final CalciteConnection connection = createConnection();
- final SchemaPlus rootSchema = createRootSchema(connection);
+ final ResettableDataSource connectionStatusDataSource = new ConnectionStatusDataSource(context, groupStatusCache);
+ final NiFiTable connectionStatusTable = new NiFiTable("CONNECTION_STATUS", connectionStatusDataSource, getLogger());
+ database.addTable(connectionStatusTable);
- final ConnectionStatusTable connectionStatusTable = new ConnectionStatusTable(context, getLogger());
- rootSchema.add("CONNECTION_STATUS", connectionStatusTable);
if (context.isAnalyticsEnabled()) {
- final ConnectionStatusPredictionsTable connectionStatusPredictionsTable = new ConnectionStatusPredictionsTable(context, getLogger());
- rootSchema.add("CONNECTION_STATUS_PREDICTIONS", connectionStatusPredictionsTable);
+ final ResettableDataSource predictionDataSource = new ConnectionStatusPredictionDataSource(context, groupStatusCache);
+ final NiFiTable connectionStatusPredictionsTable = new NiFiTable("CONNECTION_STATUS_PREDICTIONS", predictionDataSource, getLogger());
+ database.addTable(connectionStatusPredictionsTable);
} else {
getLogger().debug("Analytics is not enabled, CONNECTION_STATUS_PREDICTIONS table is not available for querying");
}
- final ProcessorStatusTable processorStatusTable = new ProcessorStatusTable(context, getLogger());
- rootSchema.add("PROCESSOR_STATUS", processorStatusTable);
- final ProcessGroupStatusTable processGroupStatusTable = new ProcessGroupStatusTable(context, getLogger());
- rootSchema.add("PROCESS_GROUP_STATUS", processGroupStatusTable);
- final JvmMetricsTable jvmMetricsTable = new JvmMetricsTable(context, getLogger());
- rootSchema.add("JVM_METRICS", jvmMetricsTable);
- final BulletinTable bulletinTable = new BulletinTable(context, getLogger());
- rootSchema.add("BULLETINS", bulletinTable);
- final ProvenanceTable provenanceTable = new ProvenanceTable(context, getLogger());
- rootSchema.add("PROVENANCE", provenanceTable);
- final FlowConfigHistoryTable flowConfigHistoryTable = new FlowConfigHistoryTable(context, getLogger());
- rootSchema.add("FLOW_CONFIG_HISTORY", flowConfigHistoryTable);
-
- rootSchema.setCacheEnabled(false);
- final PreparedStatement stmt = connection.prepareStatement(sql);
- return new CachedStatement(stmt, connection);
- }
+ final ResettableDataSource processorStatusDataSource = new ProcessorStatusDataSource(context, groupStatusCache);
+ final NiFiTable processorStatusTable = new NiFiTable("PROCESSOR_STATUS", processorStatusDataSource, getLogger());
+ database.addTable(processorStatusTable);
- private SchemaPlus createRootSchema(final CalciteConnection calciteConnection) {
- final SchemaPlus rootSchema = calciteConnection.getRootSchema();
- // Add any custom functions here
- return rootSchema;
- }
+ final ResettableDataSource processGroupStatusDataSource = new ProcessGroupStatusDataSource(context, groupStatusCache);
+ final NiFiTable processGroupStatusTable = new NiFiTable("PROCESS_GROUP_STATUS", processGroupStatusDataSource, getLogger());
+ database.addTable(processGroupStatusTable);
- private CalciteConnection createConnection() {
- final Properties properties = new Properties();
- properties.put(CalciteConnectionProperty.LEX.camelName(), Lex.MYSQL_ANSI.name());
+ final ResettableDataSource jvmMetricsDataSource = new JvmMetricsDataSource();
+ final NiFiTable jvmMetricsTable = new NiFiTable("JVM_METRICS", jvmMetricsDataSource, getLogger());
+ database.addTable(jvmMetricsTable);
- try {
- final Connection connection = DriverManager.getConnection("jdbc:calcite:", properties);
- return connection.unwrap(CalciteConnection.class);
- } catch (final Exception e) {
- throw new ProcessException(e);
- }
+ final ResettableDataSource bulletinDataSource = new BulletinDataSource(context);
+ final NiFiTable bulletinTable = new NiFiTable("BULLETINS", bulletinDataSource, getLogger());
+ database.addTable(bulletinTable);
+
+ final ResettableDataSource provenanceDataSource = new ProvenanceDataSource(context);
+ final NiFiTable provenanceTable = new NiFiTable("PROVENANCE", provenanceDataSource, getLogger());
+ database.addTable(provenanceTable);
+
+ final ResettableDataSource flowConfigHistoryDataSource = new FlowConfigHistoryDataSource(context);
+ final NiFiTable flowConfigHistoryTable = new NiFiTable("FLOW_CONFIG_HISTORY", flowConfigHistoryDataSource, getLogger());
+ database.addTable(flowConfigHistoryTable);
+
+ final Connection connection = database.getConnection();
+ final PreparedStatement stmt = connection.prepareStatement(sql);
+ return new CachedStatement(stmt, database);
}
private void clearQueue(final BlockingQueue<CachedStatement> statementQueue) {
CachedStatement stmt;
while ((stmt = statementQueue.poll()) != null) {
- closeQuietly(stmt.getStatement(), stmt.getConnection());
+ closeQuietly(stmt.getStatement(), stmt.getDatabase());
}
}
@@ -226,26 +212,9 @@ public class MetricsSqlQueryService implements MetricsQueryService {
clearQueue(queue);
}
- private class PreparedStatementException extends RuntimeException {
-
- public PreparedStatementException() {
- super();
- }
-
- public PreparedStatementException(String message) {
- super(message);
- }
-
- public PreparedStatementException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public PreparedStatementException(Throwable cause) {
+ private static class PreparedStatementException extends RuntimeException {
+ public PreparedStatementException(final Throwable cause) {
super(cause);
}
-
- public PreparedStatementException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
}
}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
index f8a1fccbd7..98b7e7ffae 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/QueryResult.java
@@ -19,7 +19,7 @@ package org.apache.nifi.reporting.sql;
import java.io.Closeable;
import java.sql.ResultSet;
-interface QueryResult extends Closeable {
+public interface QueryResult extends Closeable {
ResultSet getResultSet();
int getRecordsRead();
}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinEnumerator.java
deleted file mode 100644
index 7066010be3..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinEnumerator.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.reporting.sql.bulletins;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.Bulletin;
-import org.apache.nifi.reporting.BulletinQuery;
-import org.apache.nifi.reporting.BulletinRepository;
-import org.apache.nifi.reporting.ComponentType;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-public class BulletinEnumerator implements Enumerator<Object> {
- private final ReportingContext context;
- private final ComponentLog logger;
- private final int[] fields;
-
- private Iterator<Bulletin> bulletinIterator;
- private Object currentRow;
- private int recordsRead = 0;
-
- public BulletinEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.context = context;
- this.logger = logger;
- this.fields = fields;
- reset();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- currentRow = null;
-
- if (!bulletinIterator.hasNext()) {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- return false;
- }
-
- final Bulletin bulletin = bulletinIterator.next();
- currentRow = filterColumns(bulletin);
-
- recordsRead++;
- return true;
- }
-
- protected int getRecordsRead() {
- return recordsRead;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final Bulletin bulletin) {
- if (bulletin == null) {
- return null;
- }
-
- final boolean isClustered = context.isClustered();
- String nodeId = context.getClusterNodeIdentifier();
- if (nodeId == null && isClustered) {
- nodeId = "unknown";
- }
-
- final Object[] row = new Object[]{
- bulletin.getId(),
- bulletin.getCategory(),
- bulletin.getGroupId(),
- bulletin.getGroupName(),
- bulletin.getGroupPath(),
- bulletin.getLevel(),
- bulletin.getMessage(),
- bulletin.getNodeAddress(),
- nodeId,
- bulletin.getSourceId(),
- bulletin.getSourceName(),
- bulletin.getSourceType() == null ? null : bulletin.getSourceType().name(),
- bulletin.getTimestamp() == null ? null : bulletin.getTimestamp().getTime(),
- bulletin.getFlowFileUuid()
- };
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- BulletinRepository bulletinRepo = context.getBulletinRepository();
- List<Bulletin> fullBulletinList = new ArrayList<>(bulletinRepo.findBulletinsForController());
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.PROCESSOR).build()));
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.INPUT_PORT).build()));
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.OUTPUT_PORT).build()));
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.REMOTE_PROCESS_GROUP).build()));
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.REPORTING_TASK).build()));
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.PARAMETER_PROVIDER).build()));
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.FLOW_REGISTRY_CLIENT).build()));
- fullBulletinList.addAll(bulletinRepo.findBulletins((new BulletinQuery.Builder()).sourceType(ComponentType.CONTROLLER_SERVICE).build()));
-
- bulletinIterator = fullBulletinList.iterator();
- }
-
- @Override
- public void close() {
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinProjectTableScanRule.java
deleted file mode 100644
index 4ca86722e6..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.bulletins;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link BulletinTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class BulletinProjectTableScanRule extends RelOptRule {
- public static final BulletinProjectTableScanRule INSTANCE = new BulletinProjectTableScanRule();
-
- private BulletinProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(BulletinTableScan.class, none())),
- "BulletinProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final BulletinTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new BulletinTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.bulletinTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinTable.java
deleted file mode 100644
index 42a086461b..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinTable.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.bulletins;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class BulletinTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private volatile ReportingContext context;
- private volatile int maxRecordsRead;
-
- private final Set<BulletinEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a Connection Status table.
- */
- public BulletinTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- }
-
- @Override
- public String toString() {
- return "BulletinTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final BulletinEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- public Enumerator<Object> enumerator() {
- final BulletinEnumerator bulletinEnumerator = new BulletinEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(bulletinEnumerator);
- }
-
- return bulletinEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new BulletinTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = Arrays.asList(
- "bulletinId",
- "bulletinCategory",
- "bulletinGroupId",
- "bulletinGroupName",
- "bulletinGroupPath",
- "bulletinLevel",
- "bulletinMessage",
- "bulletinNodeAddress",
- "bulletinNodeId",
- "bulletinSourceId",
- "bulletinSourceName",
- "bulletinSourceType",
- "bulletinTimestamp",
- "bulletinFlowFileUuid"
- );
- final List<RelDataType> types = Arrays.asList(
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(String.class)
- );
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinTableScan.java
deleted file mode 100644
index 7ba4925933..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/bulletins/BulletinTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.bulletins;
-
-import java.util.Collections;
-import java.util.List;
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-/**
- * Relational expression representing a query for bulletin information.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class BulletinTableScan extends TableScan implements EnumerableRel {
- final BulletinTable bulletinTable;
- final int[] fields;
-
- protected BulletinTableScan(final RelOptCluster cluster, final RelOptTable table, final BulletinTable bulletinTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), Collections.emptyList(), table);
-
- this.bulletinTable = bulletinTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new BulletinTableScan(getCluster(), table, bulletinTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(BulletinProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(BulletinTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusEnumerator.java
deleted file mode 100644
index 7281852ff4..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusEnumerator.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.reporting.sql.connectionstatus;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.controller.status.ConnectionStatus;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-import org.apache.nifi.reporting.sql.util.ConnectionStatusRecursiveIterator;
-
-
-public class ConnectionStatusEnumerator implements Enumerator<Object> {
- private final ReportingContext context;
- private final ComponentLog logger;
- private final int[] fields;
-
- private ConnectionStatusRecursiveIterator connectionStatusIterator;
- private Object currentRow;
- private int recordsRead = 0;
-
- public ConnectionStatusEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.context = context;
- this.connectionStatusIterator = new ConnectionStatusRecursiveIterator(context);
- this.logger = logger;
- this.fields = fields;
- reset();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- currentRow = null;
- final ConnectionStatus connectionStatus = connectionStatusIterator.next();
- if (connectionStatus == null) {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- return false;
- }
-
- currentRow = filterColumns(connectionStatus);
-
- recordsRead++;
- return true;
- }
-
- protected int getRecordsRead() {
- return recordsRead;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final ConnectionStatus status) {
- if (status == null) {
- return null;
- }
-
- final Object[] row = new Object[]{
- status.getId(),
- status.getGroupId(),
- status.getName(),
- status.getSourceId(),
- status.getSourceName(),
- status.getDestinationId(),
- status.getDestinationName(),
- status.getBackPressureDataSizeThreshold(),
- status.getBackPressureBytesThreshold(),
- status.getBackPressureObjectThreshold(),
- // isBackPressureEnabled
- ((status.getBackPressureObjectThreshold() > 0 && status.getBackPressureObjectThreshold() <= status.getQueuedCount())
- || (status.getBackPressureBytesThreshold() > 0 && status.getBackPressureBytesThreshold() <= status.getQueuedBytes())),
- status.getInputCount(),
- status.getInputBytes(),
- status.getQueuedCount(),
- status.getQueuedBytes(),
- status.getOutputCount(),
- status.getOutputBytes(),
- status.getMaxQueuedCount(),
- status.getMaxQueuedBytes()
- };
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- // Clear the root PG status object so it is fetched fresh on the first record
- connectionStatusIterator = null;
- this.connectionStatusIterator = new ConnectionStatusRecursiveIterator(context);
- }
-
- @Override
- public void close() {
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusProjectTableScanRule.java
deleted file mode 100644
index 8c97bf2775..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.connectionstatus;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link ConnectionStatusTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class ConnectionStatusProjectTableScanRule extends RelOptRule {
- public static final ConnectionStatusProjectTableScanRule INSTANCE = new ConnectionStatusProjectTableScanRule();
-
- private ConnectionStatusProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(ConnectionStatusTableScan.class, none())),
- "ConnectionStatusProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final ConnectionStatusTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new ConnectionStatusTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.connectionStatusTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusTable.java
deleted file mode 100644
index 3cf0f316f5..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusTable.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.connectionstatus;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class ConnectionStatusTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private volatile ReportingContext context;
- private volatile int maxRecordsRead;
-
- private final Set<ConnectionStatusEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a Connection Status table.
- */
- public ConnectionStatusTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- }
-
- @Override
- public String toString() {
- return "ConnectionStatusTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final ConnectionStatusEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- public Enumerator<Object> enumerator() {
- final ConnectionStatusEnumerator connectionStatusEnumerator = new ConnectionStatusEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(connectionStatusEnumerator);
- }
-
- return connectionStatusEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new ConnectionStatusTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = Arrays.asList(
- "id",
- "groupId",
- "name",
- "sourceId",
- "sourceName",
- "destinationId",
- "destinationName",
- "backPressureDataSizeThreshold",
- "backPressureBytesThreshold",
- "backPressureObjectThreshold",
- "isBackPressureEnabled",
- "inputCount",
- "inputBytes",
- "queuedCount",
- "queuedBytes",
- "outputCount",
- "outputBytes",
- "maxQueuedCount",
- "maxQueuedBytes"
- );
- final List<RelDataType> types = Arrays.asList(
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(boolean.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class)
- );
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusTableScan.java
deleted file mode 100644
index 9add6e591e..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatus/ConnectionStatusTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.connectionstatus;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a snapshot (at scan time) of status for all connections in the flow.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class ConnectionStatusTableScan extends TableScan implements EnumerableRel {
- final ConnectionStatusTable connectionStatusTable;
- final int[] fields;
-
- protected ConnectionStatusTableScan(final RelOptCluster cluster, final RelOptTable table, final ConnectionStatusTable connectionStatusTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.connectionStatusTable = connectionStatusTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new ConnectionStatusTableScan(getCluster(), table, connectionStatusTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(ConnectionStatusProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(ConnectionStatusTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsEnumerator.java
deleted file mode 100644
index b5013812e2..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsEnumerator.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.connectionstatuspredictions;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.controller.status.ConnectionStatus;
-import org.apache.nifi.controller.status.analytics.ConnectionStatusPredictions;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-import org.apache.nifi.reporting.sql.util.ConnectionStatusRecursiveIterator;
-
-
-public class ConnectionStatusPredictionsEnumerator implements Enumerator<Object> {
- private final ReportingContext context;
- private final ComponentLog logger;
- private final int[] fields;
-
- private ConnectionStatusRecursiveIterator connectionStatusIterator;
- private Object currentRow;
- private int recordsRead = 0;
-
- public ConnectionStatusPredictionsEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.context = context;
- this.connectionStatusIterator = new ConnectionStatusRecursiveIterator(context);
- this.logger = logger;
- this.fields = fields;
- reset();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- currentRow = null;
- final ConnectionStatus connectionStatus = connectionStatusIterator.next();
- if (connectionStatus == null) {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- return false;
- }
-
- currentRow = filterColumns(connectionStatus);
-
- recordsRead++;
- return true;
- }
-
- protected int getRecordsRead() {
- return recordsRead;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final ConnectionStatus status) {
- if (status == null || status.getPredictions() == null) {
- return null;
- }
-
- final ConnectionStatusPredictions predictions = status.getPredictions();
-
- if (predictions == null) {
- return null;
- }
-
- final Object[] row = new Object[]{
- status.getId(),
- predictions.getNextPredictedQueuedBytes(),
- predictions.getNextPredictedQueuedCount(),
- predictions.getPredictedPercentBytes(),
- predictions.getPredictedPercentCount(),
- predictions.getPredictedTimeToBytesBackpressureMillis(),
- predictions.getPredictedTimeToCountBackpressureMillis(),
- predictions.getPredictionIntervalMillis()
- };
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- // Clear the root PG status object so it is fetched fresh on the first record
- connectionStatusIterator = null;
- this.connectionStatusIterator = new ConnectionStatusRecursiveIterator(context);
- }
-
- @Override
- public void close() {
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsProjectTableScanRule.java
deleted file mode 100644
index 8947fffab9..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.connectionstatuspredictions;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link ConnectionStatusPredictionsTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class ConnectionStatusPredictionsProjectTableScanRule extends RelOptRule {
- public static final ConnectionStatusPredictionsProjectTableScanRule INSTANCE = new ConnectionStatusPredictionsProjectTableScanRule();
-
- private ConnectionStatusPredictionsProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(ConnectionStatusPredictionsTableScan.class, none())),
- "ConnectionStatusProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final ConnectionStatusPredictionsTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new ConnectionStatusPredictionsTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.connectionStatusPredictionsTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsTable.java
deleted file mode 100644
index 1d1c943b92..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsTable.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.connectionstatuspredictions;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class ConnectionStatusPredictionsTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private volatile ReportingContext context;
- private volatile int maxRecordsRead;
-
- private final Set<ConnectionStatusPredictionsEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a Connection Status table.
- */
- public ConnectionStatusPredictionsTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- }
-
- @Override
- public String toString() {
- return "ConnectionStatusPredictionsTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final ConnectionStatusPredictionsEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- public Enumerator<Object> enumerator() {
- final ConnectionStatusPredictionsEnumerator connectionStatusPredictionsEnumerator = new ConnectionStatusPredictionsEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(connectionStatusPredictionsEnumerator);
- }
-
- return connectionStatusPredictionsEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new ConnectionStatusPredictionsTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
- final List<String> names = Arrays.asList(
- "connectionId",
- "predictedQueuedBytes",
- "predictedQueuedCount",
- "predictedPercentBytes",
- "predictedPercentCount",
- "predictedTimeToBytesBackpressureMillis",
- "predictedTimeToCountBackpressureMillis",
- "predictionIntervalMillis"
- );
- final List<RelDataType> types = Arrays.asList(
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class)
- );
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsTableScan.java
deleted file mode 100644
index 31b3aab464..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/connectionstatuspredictions/ConnectionStatusPredictionsTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.connectionstatuspredictions;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a snapshot (at scan time) of status for all connections in the flow.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class ConnectionStatusPredictionsTableScan extends TableScan implements EnumerableRel {
- final ConnectionStatusPredictionsTable connectionStatusPredictionsTable;
- final int[] fields;
-
- protected ConnectionStatusPredictionsTableScan(final RelOptCluster cluster, final RelOptTable table, final ConnectionStatusPredictionsTable connectionStatusPredictionsTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.connectionStatusPredictionsTable = connectionStatusPredictionsTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new ConnectionStatusPredictionsTableScan(getCluster(), table, connectionStatusPredictionsTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(ConnectionStatusPredictionsProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(ConnectionStatusPredictionsTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/BulletinDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/BulletinDataSource.java
new file mode 100644
index 0000000000..6fd4c7f47f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/BulletinDataSource.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.reporting.Bulletin;
+import org.apache.nifi.reporting.BulletinQuery;
+import org.apache.nifi.reporting.BulletinRepository;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.IterableRowStream;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+
+import java.util.List;
+
+public class BulletinDataSource implements ResettableDataSource {
+ private static final NiFiTableSchema SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema("bulletinId", long.class, false),
+ new ColumnSchema("bulletinCategory", String.class, false),
+ new ColumnSchema("bulletinGroupId", String.class, true),
+ new ColumnSchema("bulletinGroupName", String.class, true),
+ new ColumnSchema("bulletinGroupPath", String.class, true),
+ new ColumnSchema("bulletinLevel", String.class, false),
+ new ColumnSchema("bulletinMessage", String.class, false),
+ new ColumnSchema("bulletinNodeAddress", String.class, true),
+ new ColumnSchema("bulletinNodeId", String.class, true),
+ new ColumnSchema("bulletinSourceId", String.class, true),
+ new ColumnSchema("bulletinSourceName", String.class, true),
+ new ColumnSchema("bulletinSourceType", String.class, true),
+ new ColumnSchema("bulletinTimestamp", long.class, false),
+ new ColumnSchema("bulletinFlowFileUuid", String.class, true)
+ ));
+
+ private final ReportingContext reportingContext;
+
+ public BulletinDataSource(final ReportingContext reportingContext) {
+ this.reportingContext = reportingContext;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return SCHEMA;
+ }
+
+ @Override
+ public RowStream reset() {
+ final boolean isClustered = reportingContext.isClustered();
+ String nodeId = "unknown";
+ if (isClustered) {
+ final String clusterNodeId = reportingContext.getClusterNodeIdentifier();
+ if (clusterNodeId != null) {
+ nodeId = clusterNodeId;
+ }
+ }
+
+ final String resolvedNodeId = nodeId;
+ final BulletinRepository bulletinRepo = reportingContext.getBulletinRepository();
+ final List<Bulletin> bulletins = bulletinRepo.findBulletins(new BulletinQuery.Builder().build());
+ return new IterableRowStream<>(bulletins, bulletin -> toArray(bulletin, resolvedNodeId));
+ }
+
+ private Object[] toArray(final Bulletin bulletin, final String nodeId) {
+ return new Object[] {
+ bulletin.getId(),
+ bulletin.getCategory(),
+ bulletin.getGroupId(),
+ bulletin.getGroupName(),
+ bulletin.getGroupPath(),
+ bulletin.getLevel(),
+ bulletin.getMessage(),
+ bulletin.getNodeAddress(),
+ nodeId,
+ bulletin.getSourceId(),
+ bulletin.getSourceName(),
+ bulletin.getSourceType() == null ? null : bulletin.getSourceType().name(),
+ bulletin.getTimestamp() == null ? null : bulletin.getTimestamp().getTime(),
+ bulletin.getFlowFileUuid()
+ };
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ConnectionStatusDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ConnectionStatusDataSource.java
new file mode 100644
index 0000000000..3e6775e483
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ConnectionStatusDataSource.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.controller.status.ConnectionStatus;
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.IterableRowStream;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ConnectionStatusDataSource implements ResettableDataSource {
+ private static final NiFiTableSchema SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema("id", String.class, false),
+ new ColumnSchema("groupId", String.class, false),
+ new ColumnSchema("name", String.class, false),
+ new ColumnSchema("sourceId", String.class, false),
+ new ColumnSchema("sourceName", String.class, true),
+ new ColumnSchema("destinationId", String.class, false),
+ new ColumnSchema("destinationName", String.class, true),
+ new ColumnSchema("backPressureDataSizeThreshold", String.class, true),
+ new ColumnSchema("backPressureBytesThreshold", Long.class, true),
+ new ColumnSchema("backPressureObjectThreshold", Long.class, true),
+ new ColumnSchema("isBackPressureEnabled", boolean.class, false),
+ new ColumnSchema("inputCount", long.class, false),
+ new ColumnSchema("inputBytes", long.class, false),
+ new ColumnSchema("queuedCount", int.class, false),
+ new ColumnSchema("queuedBytes", long.class, false),
+ new ColumnSchema("outputCount", long.class, false),
+ new ColumnSchema("outputBytes", long.class, false),
+ new ColumnSchema("maxQueuedCount", Integer.class, true),
+ new ColumnSchema("maxQueuedBytes", Long.class, true)
+ ));
+
+ private final ReportingContext reportingContext;
+ private final GroupStatusCache groupStatusCache;
+ private ProcessGroupStatus lastFetchedStatus = null;
+ private List<ConnectionStatus> lastConnectionStatuses = null;
+
+ public ConnectionStatusDataSource(final ReportingContext reportingContext, final GroupStatusCache groupStatusCache) {
+ this.reportingContext = reportingContext;
+ this.groupStatusCache = groupStatusCache;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return SCHEMA;
+ }
+
+ @Override
+ public RowStream reset() {
+ final ProcessGroupStatus groupStatus = groupStatusCache.getGroupStatus(reportingContext);
+
+ final List<ConnectionStatus> connectionStatuses;
+ if (groupStatus == lastFetchedStatus) {
+ connectionStatuses = lastConnectionStatuses;
+ } else {
+ connectionStatuses = lastConnectionStatuses = gatherConnectionStatuses(groupStatus);
+ }
+
+ lastFetchedStatus = groupStatus;
+ return new IterableRowStream<>(connectionStatuses, this::toArray);
+ }
+
+ private List<ConnectionStatus> gatherConnectionStatuses(final ProcessGroupStatus groupStatus) {
+ final List<ConnectionStatus> allStatuses = new ArrayList<>();
+ gatherConnectionStatuses(groupStatus, allStatuses);
+ return allStatuses;
+ }
+
+ private void gatherConnectionStatuses(final ProcessGroupStatus groupStatus, final List<ConnectionStatus> connectionStatuses) {
+ connectionStatuses.addAll(groupStatus.getConnectionStatus());
+ for (final ProcessGroupStatus childStatus : groupStatus.getProcessGroupStatus()) {
+ gatherConnectionStatuses(childStatus, connectionStatuses);
+ }
+ }
+
+ private Object[] toArray(final ConnectionStatus status) {
+ return new Object[] {
+ status.getId(),
+ status.getGroupId(),
+ status.getName(),
+ status.getSourceId(),
+ status.getSourceName(),
+ status.getDestinationId(),
+ status.getDestinationName(),
+ status.getBackPressureDataSizeThreshold(),
+ status.getBackPressureBytesThreshold(),
+ status.getBackPressureObjectThreshold(),
+ // isBackPressureEnabled
+ ((status.getBackPressureObjectThreshold() > 0 && status.getBackPressureObjectThreshold() <= status.getQueuedCount())
+ || (status.getBackPressureBytesThreshold() > 0 && status.getBackPressureBytesThreshold() <= status.getQueuedBytes())),
+ status.getInputCount(),
+ status.getInputBytes(),
+ status.getQueuedCount(),
+ status.getQueuedBytes(),
+ status.getOutputCount(),
+ status.getOutputBytes(),
+ status.getMaxQueuedCount(),
+ status.getMaxQueuedBytes()
+ };
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ConnectionStatusPredictionDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ConnectionStatusPredictionDataSource.java
new file mode 100644
index 0000000000..97d43f6b24
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ConnectionStatusPredictionDataSource.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.controller.status.ConnectionStatus;
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.controller.status.analytics.ConnectionStatusPredictions;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.IterableRowStream;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ConnectionStatusPredictionDataSource implements ResettableDataSource {
+ private static final NiFiTableSchema SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema("connectionId", String.class, false),
+ new ColumnSchema("predictedQueuedBytes", long.class, true),
+ new ColumnSchema("predictedQueuedCount", int.class, true),
+ new ColumnSchema("predictedPercentBytes", int.class, true),
+ new ColumnSchema("predictedPercentCount", int.class, true),
+ new ColumnSchema("predictedTimeToBytesBackpressureMillis", long.class, true),
+ new ColumnSchema("predictedTimeToCountBackpressureMillis", long.class, true),
+ new ColumnSchema("predictionIntervalMillis", long.class, true)
+ ));
+
+ private final ReportingContext reportingContext;
+ private final GroupStatusCache groupStatusCache;
+ private ProcessGroupStatus lastFetchedStatus = null;
+ private List<ConnectionStatus> lastConnectionStatuses = null;
+
+ public ConnectionStatusPredictionDataSource(final ReportingContext reportingContext, final GroupStatusCache groupStatusCache) {
+ this.reportingContext = reportingContext;
+ this.groupStatusCache = groupStatusCache;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return SCHEMA;
+ }
+
+ @Override
+ public RowStream reset() {
+ final ProcessGroupStatus groupStatus = groupStatusCache.getGroupStatus(reportingContext);
+
+ final List<ConnectionStatus> connectionStatuses;
+ if (groupStatus == lastFetchedStatus) {
+ connectionStatuses = lastConnectionStatuses;
+ } else {
+ connectionStatuses = lastConnectionStatuses = gatherConnectionStatuses(groupStatus);
+ }
+
+ lastFetchedStatus = groupStatus;
+ return new IterableRowStream<>(connectionStatuses, this::toArray);
+ }
+
+ private List<ConnectionStatus> gatherConnectionStatuses(final ProcessGroupStatus groupStatus) {
+ final List<ConnectionStatus> allStatuses = new ArrayList<>();
+ gatherConnectionStatuses(groupStatus, allStatuses);
+ return allStatuses;
+ }
+
+ private void gatherConnectionStatuses(final ProcessGroupStatus groupStatus, final List<ConnectionStatus> connectionStatuses) {
+ connectionStatuses.addAll(groupStatus.getConnectionStatus());
+ for (final ProcessGroupStatus childStatus : groupStatus.getProcessGroupStatus()) {
+ gatherConnectionStatuses(childStatus, connectionStatuses);
+ }
+ }
+
+ private Object[] toArray(final ConnectionStatus status) {
+ final ConnectionStatusPredictions predictions = status.getPredictions();
+
+ return new Object[] {
+ status.getId(),
+ predictions.getNextPredictedQueuedBytes(),
+ predictions.getNextPredictedQueuedCount(),
+ predictions.getPredictedPercentBytes(),
+ predictions.getPredictedPercentCount(),
+ predictions.getPredictedTimeToBytesBackpressureMillis(),
+ predictions.getPredictedTimeToCountBackpressureMillis(),
+ predictions.getPredictionIntervalMillis()
+ };
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/FlowConfigHistoryDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/FlowConfigHistoryDataSource.java
new file mode 100644
index 0000000000..9f84a6faeb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/FlowConfigHistoryDataSource.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.action.Action;
+import org.apache.nifi.action.details.ActionDetails;
+import org.apache.nifi.action.details.ConfigureDetails;
+import org.apache.nifi.action.details.ConnectDetails;
+import org.apache.nifi.action.details.MoveDetails;
+import org.apache.nifi.action.details.PurgeDetails;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.IterableRowStream;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+
+import java.util.List;
+
+public class FlowConfigHistoryDataSource implements ResettableDataSource {
+ public static final NiFiTableSchema SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema("actionId", int.class, false),
+ new ColumnSchema("actionTimestamp", long.class, false),
+ new ColumnSchema("actionUserIdentity", String.class, false),
+ new ColumnSchema("actionSourceId", String.class, false),
+ new ColumnSchema("actionSourceName", String.class, false),
+ new ColumnSchema("actionSourceType", String.class, false),
+ new ColumnSchema("actionOperation", String.class, false),
+ new ColumnSchema("configureDetailsName", String.class, false),
+ new ColumnSchema("configureDetailsPreviousValue", String.class, false),
+ new ColumnSchema("configureDetailsValue", String.class, false),
+ new ColumnSchema("connectionSourceId", String.class, false),
+ new ColumnSchema("connectionSourceName", String.class, false),
+ new ColumnSchema("connectionSourceType", String.class, false),
+ new ColumnSchema("connectionDestinationId", String.class, false),
+ new ColumnSchema("connectionDestinationName", String.class, false),
+ new ColumnSchema("connectionDestinationType", String.class, false),
+ new ColumnSchema("connectionRelationship", String.class, false),
+ new ColumnSchema("moveGroup", String.class, false),
+ new ColumnSchema("moveGroupId", String.class, false),
+ new ColumnSchema("movePreviousGroup", String.class, false),
+ new ColumnSchema("movePreviousGroupId", String.class, false),
+ new ColumnSchema("purgeEndDate", long.class, false)
+ ));
+
+ private final ReportingContext reportingContext;
+
+ public FlowConfigHistoryDataSource(final ReportingContext reportingContext) {
+ this.reportingContext = reportingContext;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return SCHEMA;
+ }
+
+ @Override
+ public RowStream reset() {
+ final List<Action> fullFlowConfigHistoryList = reportingContext.getEventAccess().getFlowChanges(0, Short.MAX_VALUE);
+ return new IterableRowStream<>(fullFlowConfigHistoryList, this::toArray);
+ }
+
+ private Object[] toArray(final Action action) {
+ final ActionDetails details = action.getActionDetails();
+
+ return new Object[] {
+ action.getId(),
+ action.getTimestamp().getTime(),
+ action.getUserIdentity(),
+ action.getSourceId(),
+ action.getSourceName(),
+ action.getSourceType(),
+ action.getOperation().toString(),
+ details instanceof ConfigureDetails ? ((ConfigureDetails) details).getName() : null,
+ details instanceof ConfigureDetails ? ((ConfigureDetails) details).getPreviousValue() : null,
+ details instanceof ConfigureDetails ? ((ConfigureDetails) details).getValue() : null,
+ details instanceof ConnectDetails ? ((ConnectDetails) details).getSourceId() : null,
+ details instanceof ConnectDetails ? ((ConnectDetails) details).getSourceName() : null,
+ details instanceof ConnectDetails ? ((ConnectDetails) details).getSourceType() : null,
+ details instanceof ConnectDetails ? ((ConnectDetails) details).getDestinationId() : null,
+ details instanceof ConnectDetails ? ((ConnectDetails) details).getDestinationName() : null,
+ details instanceof ConnectDetails ? ((ConnectDetails) details).getDestinationType() : null,
+ details instanceof ConnectDetails ? ((ConnectDetails) details).getRelationship() : null,
+ details instanceof MoveDetails ? ((MoveDetails) details).getGroup() : null,
+ details instanceof MoveDetails ? ((MoveDetails) details).getGroupId() : null,
+ details instanceof MoveDetails ? ((MoveDetails) details).getPreviousGroup() : null,
+ details instanceof MoveDetails ? ((MoveDetails) details).getPreviousGroupId() : null,
+ details instanceof PurgeDetails ? ((PurgeDetails) details).getEndDate().getTime() : null
+ };
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/GroupStatusCache.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/GroupStatusCache.java
new file mode 100644
index 0000000000..847dc503b9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/GroupStatusCache.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.reporting.ReportingContext;
+
+import java.time.Duration;
+
+public class GroupStatusCache {
+ private final long refreshMillis;
+
+ private long nextRefreshTime;
+ private ProcessGroupStatus cached;
+
+
+ public GroupStatusCache(final Duration refreshFrequency) {
+ this.refreshMillis = refreshFrequency.toMillis();
+ nextRefreshTime = 0L;
+ }
+
+ public synchronized ProcessGroupStatus getGroupStatus(final ReportingContext context) {
+ if (cached == null || System.currentTimeMillis() > nextRefreshTime) {
+ final ProcessGroupStatus retrieved = context.getEventAccess().getControllerStatus();
+ this.cached = retrieved;
+
+ final long now = System.currentTimeMillis();
+ this.nextRefreshTime = now + refreshMillis;
+ return retrieved;
+ }
+
+ return cached;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/JvmMetricsDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/JvmMetricsDataSource.java
new file mode 100644
index 0000000000..67d8fa9b86
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/JvmMetricsDataSource.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.metrics.jvm.JmxJvmMetrics;
+import org.apache.nifi.metrics.jvm.JvmMetrics;
+import org.apache.nifi.processor.DataUnit;
+import org.apache.nifi.reporting.util.metrics.MetricNames;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.IterableRowStream;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.Thread.State.BLOCKED;
+import static java.lang.Thread.State.RUNNABLE;
+import static java.lang.Thread.State.TERMINATED;
+import static java.lang.Thread.State.TIMED_WAITING;
+
+public class JvmMetricsDataSource implements ResettableDataSource {
+ private static final NiFiTableSchema BASE_SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema(MetricNames.JVM_DAEMON_THREAD_COUNT.replaceAll("[.-]", "_"), int.class, false),
+ new ColumnSchema(MetricNames.JVM_THREAD_COUNT.replaceAll("[.-]", "_"), int.class, false),
+ new ColumnSchema(MetricNames.JVM_THREAD_STATES_BLOCKED.replaceAll("[.-]", "_"), int.class, false),
+ new ColumnSchema(MetricNames.JVM_THREAD_STATES_RUNNABLE.replaceAll("[.-]", "_"), int.class, false),
+ new ColumnSchema(MetricNames.JVM_THREAD_STATES_TERMINATED.replaceAll("[.-]", "_"), int.class, false),
+ new ColumnSchema(MetricNames.JVM_THREAD_STATES_TIMED_WAITING.replaceAll("[.-]", "_"), int.class, false),
+ new ColumnSchema(MetricNames.JVM_UPTIME.replaceAll("[.-]", "_"), long.class, false),
+ new ColumnSchema(MetricNames.JVM_HEAP_USED.replaceAll("[.-]", "_"), double.class, false),
+ new ColumnSchema(MetricNames.JVM_HEAP_USAGE.replaceAll("[.-]", "_"), double.class, false),
+ new ColumnSchema(MetricNames.JVM_NON_HEAP_USAGE.replaceAll("[.-]", "_"), double.class, false),
+ new ColumnSchema(MetricNames.JVM_FILE_DESCRIPTOR_USAGE.replaceAll("[.-]", "_"), double.class, false)
+ ));
+
+ private final JvmMetrics virtualMachineMetrics = JmxJvmMetrics.getInstance();
+ private final NiFiTableSchema schema;
+
+ public JvmMetricsDataSource() {
+ final List<ColumnSchema> columns = new ArrayList<>(BASE_SCHEMA.columns());
+ for (final String gcName : virtualMachineMetrics.garbageCollectors().keySet()) {
+ final String gcRunsName = normalize(MetricNames.JVM_GC_RUNS + "_" + gcName);
+ final String gcTimeName = normalize(MetricNames.JVM_GC_TIME + "_" + gcName);
+
+ columns.add(new ColumnSchema(gcRunsName, long.class, true));
+ columns.add(new ColumnSchema(gcTimeName, long.class, true));
+ }
+
+ schema = new NiFiTableSchema(columns);
+ }
+
+ private String normalize(final String value) {
+ return value.replaceAll("[ -.]", "_");
+ }
+
+ public NiFiTableSchema getSchema() {
+ return schema;
+ }
+
+ @Override
+ public RowStream reset() {
+ final List<Object> values = new ArrayList<>();
+ values.add(virtualMachineMetrics.daemonThreadCount());
+ values.add(virtualMachineMetrics.threadCount());
+
+ // Normalize the thread percentages to an integer 0-100
+ values.add(getThreadStatePercentage(BLOCKED));
+ values.add(getThreadStatePercentage(RUNNABLE));
+ values.add(getThreadStatePercentage(TERMINATED));
+ values.add(getThreadStatePercentage(TIMED_WAITING));
+
+ values.add(virtualMachineMetrics.uptime());
+ values.add(virtualMachineMetrics.heapUsed(DataUnit.B));
+ values.add(virtualMachineMetrics.heapUsage());
+ values.add(virtualMachineMetrics.nonHeapUsage());
+ values.add(virtualMachineMetrics.fileDescriptorUsage());
+
+ virtualMachineMetrics.garbageCollectors().values().forEach((gc) -> {
+ values.add(gc.getRuns());
+ values.add(gc.getTime(TimeUnit.MILLISECONDS));
+ });
+
+ return new IterableRowStream<>(Collections.singleton(values), v -> v.toArray(new Object[0]));
+ }
+
+ private int getThreadStatePercentage(final Thread.State threadState) {
+ final Double ratio = virtualMachineMetrics.threadStatePercentages().get(threadState);
+ if (ratio == null) {
+ return 0;
+ }
+
+ return (int) (ratio * 100D);
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProcessGroupStatusDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProcessGroupStatusDataSource.java
new file mode 100644
index 0000000000..614682ff2f
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProcessGroupStatusDataSource.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.IterableRowStream;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ProcessGroupStatusDataSource implements ResettableDataSource {
+ private static final NiFiTableSchema SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema("id", String.class, false),
+ new ColumnSchema("groupId", String.class, false),
+ new ColumnSchema("name", String.class, false),
+ new ColumnSchema("bytesRead", long.class, false),
+ new ColumnSchema("bytesWritten", long.class, false),
+ new ColumnSchema("bytesReceived", long.class, false),
+ new ColumnSchema("bytesSent", long.class, false),
+ new ColumnSchema("bytesTransferred", long.class, false),
+ new ColumnSchema("flowFilesReceived", int.class, false),
+ new ColumnSchema("flowFilesSent", int.class, false),
+ new ColumnSchema("flowFilesTransferred", int.class, false),
+ new ColumnSchema("inputContentSize", long.class, false),
+ new ColumnSchema("inputCount", int.class, false),
+ new ColumnSchema("outputContentSize", long.class, false),
+ new ColumnSchema("outputCount", int.class, false),
+ new ColumnSchema("queuedCount", int.class, false),
+ new ColumnSchema("queuedContentSize", long.class, false),
+ new ColumnSchema("activeThreadCount", int.class, false),
+ new ColumnSchema("terminatedThreadCount", int.class, false),
+ new ColumnSchema("versionedFlowState", String.class, false),
+ new ColumnSchema("processingNanos", long.class, false)
+ ));
+
+
+ private final ReportingContext reportingContext;
+ private final GroupStatusCache groupStatusCache;
+ private ProcessGroupStatus lastFetchedStatus = null;
+ private List<GroupStatusAndParentId> lastStatuses = null;
+
+ public ProcessGroupStatusDataSource(final ReportingContext reportingContext, final GroupStatusCache groupStatusCache) {
+ this.reportingContext = reportingContext;
+ this.groupStatusCache = groupStatusCache;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return SCHEMA;
+ }
+
+ @Override
+ public RowStream reset() {
+ final ProcessGroupStatus groupStatus = groupStatusCache.getGroupStatus(reportingContext);
+
+ final List<GroupStatusAndParentId> groupStatuses;
+ if (groupStatus == lastFetchedStatus) {
+ groupStatuses = lastStatuses;
+ } else {
+ groupStatuses = lastStatuses = gatherProcessGroupStatuses(groupStatus);
+ }
+
+ lastFetchedStatus = groupStatus;
+ return new IterableRowStream<>(groupStatuses, this::toArray);
+ }
+
+ private List<GroupStatusAndParentId> gatherProcessGroupStatuses(final ProcessGroupStatus groupStatus) {
+ final List<GroupStatusAndParentId> allStatuses = new ArrayList<>();
+ gatherProcessGroupStatuses(null, groupStatus, allStatuses);
+ return allStatuses;
+ }
+
+ private void gatherProcessGroupStatuses(final String parentGroupId, final ProcessGroupStatus groupStatus, final List<GroupStatusAndParentId> groupStatuses) {
+ groupStatuses.add(new GroupStatusAndParentId(parentGroupId, groupStatus));
+
+ for (final ProcessGroupStatus childStatus : groupStatus.getProcessGroupStatus()) {
+ gatherProcessGroupStatuses(groupStatus.getId(), childStatus, groupStatuses);
+ }
+ }
+
+ private Object[] toArray(final GroupStatusAndParentId groupStatusAndParentId) {
+ final ProcessGroupStatus status = groupStatusAndParentId.status();
+ final String parentId = groupStatusAndParentId.parentId();
+
+ return new Object[] {
+ status.getId(),
+ parentId,
+ status.getName(),
+ status.getBytesRead(),
+ status.getBytesWritten(),
+ status.getBytesReceived(),
+ status.getBytesSent(),
+ status.getBytesTransferred(),
+ status.getFlowFilesReceived(),
+ status.getFlowFilesSent(),
+ status.getFlowFilesTransferred(),
+ status.getInputContentSize(),
+ status.getInputCount(),
+ status.getOutputContentSize(),
+ status.getOutputCount(),
+ status.getQueuedContentSize(),
+ status.getActiveThreadCount(),
+ status.getTerminatedThreadCount(),
+ status.getQueuedCount(),
+ status.getVersionedFlowState() == null ? null : status.getVersionedFlowState().name(),
+ status.getProcessingNanos()
+ };
+ }
+
+ private record GroupStatusAndParentId(String parentId, ProcessGroupStatus status) {
+ }
+}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProcessorStatusDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProcessorStatusDataSource.java
new file mode 100644
index 0000000000..e3aade120a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProcessorStatusDataSource.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.controller.status.ProcessorStatus;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.IterableRowStream;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class ProcessorStatusDataSource implements ResettableDataSource {
+ private static final NiFiTableSchema SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema("id", String.class, false),
+ new ColumnSchema("groupId", String.class, false),
+ new ColumnSchema("name", String.class, false),
+ new ColumnSchema("processorType", String.class, false),
+ new ColumnSchema("averageLineageDuration", long.class, false),
+ new ColumnSchema("bytesRead", long.class, false),
+ new ColumnSchema("bytesWritten", long.class, false),
+ new ColumnSchema("bytesReceived", long.class, false),
+ new ColumnSchema("bytesSent", long.class, false),
+ new ColumnSchema("flowFilesRemoved", int.class, false),
+ new ColumnSchema("flowFilesReceived", int.class, false),
+ new ColumnSchema("flowFilesSent", int.class, false),
+ new ColumnSchema("inputCount", int.class, false),
+ new ColumnSchema("inputBytes", long.class, false),
+ new ColumnSchema("outputCount", int.class, false),
+ new ColumnSchema("outputBytes", long.class, false),
+ new ColumnSchema("activeThreadCount", int.class, false),
+ new ColumnSchema("terminatedThreadCount", int.class, false),
+ new ColumnSchema("invocations", int.class, false),
+ new ColumnSchema("processingNanos", long.class, false),
+ new ColumnSchema("runStatus", String.class, false),
+ new ColumnSchema("executionNode", String.class, false)
+ ));
+
+
+ private final ReportingContext reportingContext;
+ private final GroupStatusCache groupStatusCache;
+ private ProcessGroupStatus lastFetchedStatus = null;
+ private List<ProcessorStatus> lastStatuses = null;
+
+ public ProcessorStatusDataSource(final ReportingContext reportingContext, final GroupStatusCache groupStatusCache) {
+ this.reportingContext = reportingContext;
+ this.groupStatusCache = groupStatusCache;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return SCHEMA;
+ }
+
+ @Override
+ public RowStream reset() {
+ final ProcessGroupStatus groupStatus = groupStatusCache.getGroupStatus(reportingContext);
+
+ final List<ProcessorStatus> processorStatuses;
+ if (groupStatus == lastFetchedStatus) {
+ processorStatuses = lastStatuses;
+ } else {
+ processorStatuses = lastStatuses = gatherProcessorStatuses(groupStatus);
+ }
+
+ lastFetchedStatus = groupStatus;
+ return new IterableRowStream<>(processorStatuses, this::toArray);
+ }
+
+ private List<ProcessorStatus> gatherProcessorStatuses(final ProcessGroupStatus groupStatus) {
+ final List<ProcessorStatus> allStatuses = new ArrayList<>();
+ gatherProcessorStatuses(groupStatus, allStatuses);
+ return allStatuses;
+ }
+
+ private void gatherProcessorStatuses(final ProcessGroupStatus groupStatus, final List<ProcessorStatus> processorStatuses) {
+ processorStatuses.addAll(groupStatus.getProcessorStatus());
+ for (final ProcessGroupStatus childStatus : groupStatus.getProcessGroupStatus()) {
+ gatherProcessorStatuses(childStatus, processorStatuses);
+ }
+ }
+
+ private Object[] toArray(final ProcessorStatus status) {
+ return new Object[] {
+ status.getId(),
+ status.getGroupId(),
+ status.getName(),
+ status.getType(),
+ status.getAverageLineageDuration(TimeUnit.MILLISECONDS),
+ status.getBytesRead(),
+ status.getBytesWritten(),
+ status.getBytesReceived(),
+ status.getBytesSent(),
+ status.getFlowFilesRemoved(),
+ status.getFlowFilesReceived(),
+ status.getFlowFilesSent(),
+ status.getInputCount(),
+ status.getInputBytes(),
+ status.getOutputCount(),
+ status.getOutputBytes(),
+ status.getActiveThreadCount(),
+ status.getTerminatedThreadCount(),
+ status.getInvocations(),
+ status.getProcessingNanos(),
+ status.getRunStatus().name(),
+ status.getExecutionNode() == null ? null : status.getExecutionNode().name()
+ };
+ }
+}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProvenanceDataSource.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProvenanceDataSource.java
new file mode 100644
index 0000000000..f9089d47df
--- /dev/null
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/datasources/ProvenanceDataSource.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.reporting.sql.datasources;
+
+import org.apache.nifi.controller.status.ProcessGroupStatus;
+import org.apache.nifi.provenance.ProvenanceEventRecord;
+import org.apache.nifi.reporting.ReportingContext;
+import org.apache.nifi.reporting.util.provenance.ComponentMapHolder;
+import org.apache.nifi.sql.ArrayType;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.MapType;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+import org.apache.nifi.sql.ScalarType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class ProvenanceDataSource implements ResettableDataSource {
+
+ public static final NiFiTableSchema SCHEMA = new NiFiTableSchema(List.of(
+ new ColumnSchema("eventId", long.class, false),
+ new ColumnSchema("eventType", String.class, false),
+ new ColumnSchema("timestampMillis", long.class, true),
+ new ColumnSchema("durationMillis", long.class, true),
+ new ColumnSchema("lineageStart", long.class, true),
+ new ColumnSchema("details", String.class, true),
+ new ColumnSchema("componentId", String.class, true),
+ new ColumnSchema("componentName", String.class, true),
+ new ColumnSchema("componentType", String.class, true),
+ new ColumnSchema("processGroupId", String.class, true),
+ new ColumnSchema("processGroupName", String.class, true),
+ new ColumnSchema("entityId", String.class, true),
+ new ColumnSchema("entityType", String.class, true),
+ new ColumnSchema("entitySize", long.class, true),
+ new ColumnSchema("previousEntitySize", long.class, true),
+ new ColumnSchema("updatedAttributes", new MapType(ScalarType.STRING, ScalarType.STRING), true),
+ new ColumnSchema("previousAttributes", new MapType(ScalarType.STRING, ScalarType.STRING), true),
+ new ColumnSchema("contentPath", String.class, true),
+ new ColumnSchema("previousContentPath", String.class, true),
+ new ColumnSchema("parentIds", new ArrayType(ScalarType.STRING), true),
+ new ColumnSchema("childIds", new ArrayType(ScalarType.STRING), true),
+ new ColumnSchema("transitUri", String.class, true),
+ new ColumnSchema("remoteIdentifier", String.class, true),
+ new ColumnSchema("alternateIdentifier", String.class, true)
+ ));
+
+ private static final String ENTITY_TYPE = "org.apache.nifi.flowfile.FlowFile";
+ private static final int FETCH_SIZE = 1_000;
+ private final ReportingContext reportingContext;
+ private final ComponentMapHolder componentMapHolder;
+
+ private Iterator<ProvenanceEventRecord> iterator;
+ private long lastId = -1L;
+
+
+ public ProvenanceDataSource(final ReportingContext reportingContext) {
+ this.reportingContext = reportingContext;
+
+ final ProcessGroupStatus procGroupStatus = reportingContext.getEventAccess().getControllerStatus();
+ this.componentMapHolder = ComponentMapHolder.createComponentMap(procGroupStatus);
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return SCHEMA;
+ }
+
+ @Override
+ public RowStream reset() throws IOException {
+ lastId = -1L;
+ iterator = null;
+
+ return new RowStream() {
+ @Override
+ public Object[] nextRow() throws IOException {
+ if (iterator == null || !iterator.hasNext()) {
+ iterator = fetchEvents();
+ }
+
+ if (iterator.hasNext()) {
+ final ProvenanceEventRecord event = iterator.next();
+ lastId = event.getEventId();
+ return toArray(event);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void close() {
+ }
+ };
+ }
+
+ private Object[] toArray(final ProvenanceEventRecord provenanceEvent) {
+ final String nodeIdentifier = reportingContext.getClusterNodeIdentifier();
+ final String processGroupId = componentMapHolder.getProcessGroupId(provenanceEvent.getComponentId(), provenanceEvent.getComponentType());
+ final String groupName = componentMapHolder.getComponentName(processGroupId);
+
+ final ArrayList<Object> rowList = new ArrayList<>();
+ rowList.add(provenanceEvent.getEventId());
+ rowList.add(provenanceEvent.getEventType().name());
+ rowList.add(provenanceEvent.getEventTime());
+ rowList.add(provenanceEvent.getEventDuration());
+ rowList.add(provenanceEvent.getLineageStartDate());
+ rowList.add(provenanceEvent.getDetails());
+ rowList.add(provenanceEvent.getComponentId());
+ rowList.add(componentMapHolder.getComponentName(provenanceEvent.getComponentId()));
+ rowList.add(provenanceEvent.getComponentType());
+ rowList.add(processGroupId);
+ rowList.add(groupName);
+ rowList.add(provenanceEvent.getFlowFileUuid());
+ rowList.add(ENTITY_TYPE); // entityType
+ rowList.add(provenanceEvent.getFileSize());
+ rowList.add(provenanceEvent.getPreviousFileSize());
+ rowList.add(provenanceEvent.getUpdatedAttributes());
+ rowList.add(provenanceEvent.getPreviousAttributes());
+
+ if (nodeIdentifier != null) {
+ final String contentPathBase = "/nifi-api/provenance-events/" + provenanceEvent.getEventId() + "/content/";
+ final String nodeIdSuffix = "?clusterNodeId=" + nodeIdentifier;
+ rowList.add(contentPathBase + "output" + nodeIdSuffix);
+ rowList.add(contentPathBase + "input" + nodeIdSuffix);
+ } else {
+ rowList.add(null);
+ rowList.add(null);
+ }
+
+ rowList.add(provenanceEvent.getParentUuids());
+ rowList.add(provenanceEvent.getChildUuids());
+ rowList.add(provenanceEvent.getTransitUri());
+ rowList.add(provenanceEvent.getSourceSystemFlowFileIdentifier());
+ rowList.add(provenanceEvent.getAlternateIdentifierUri());
+
+ return rowList.toArray();
+ }
+
+ private Iterator<ProvenanceEventRecord> fetchEvents() throws IOException {
+ final List<ProvenanceEventRecord> events = reportingContext.getEventAccess().getProvenanceEvents(lastId + 1, FETCH_SIZE);
+ return events.iterator();
+ }
+}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryEnumerator.java
deleted file mode 100644
index 7968a233cc..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryEnumerator.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.reporting.sql.flowconfighistory;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.action.Action;
-import org.apache.nifi.action.details.ConfigureDetails;
-import org.apache.nifi.action.details.ConnectDetails;
-import org.apache.nifi.action.details.MoveDetails;
-import org.apache.nifi.action.details.PurgeDetails;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.util.Iterator;
-import java.util.List;
-
-public class FlowConfigHistoryEnumerator implements Enumerator<Object> {
- private final ReportingContext context;
- private final ComponentLog logger;
- private final int[] fields;
-
- private Iterator<Action> actionIterator;
- private Object currentRow;
- private int recordsRead = 0;
-
- public FlowConfigHistoryEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.context = context;
- this.logger = logger;
- this.fields = fields;
- reset();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- currentRow = null;
-
- if (!actionIterator.hasNext()) {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- return false;
- }
-
- final Action action = actionIterator.next();
- currentRow = filterColumns(action);
-
- recordsRead++;
- return true;
- }
-
- protected int getRecordsRead() {
- return recordsRead;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final Action action) {
- if (action == null) {
- return null;
- }
-
- final boolean isClustered = context.isClustered();
- String nodeId = context.getClusterNodeIdentifier();
- if (nodeId == null && isClustered) {
- nodeId = "unknown";
- }
-
- final Object[] row = new Object[]{
- action.getId(),
- action.getTimestamp().getTime(),
- action.getUserIdentity(),
- action.getSourceId(),
- action.getSourceName(),
- action.getSourceType(),
- action.getOperation().toString(),
- action.getActionDetails() instanceof ConfigureDetails ? ((ConfigureDetails) action.getActionDetails()).getName() : null,
- action.getActionDetails() instanceof ConfigureDetails ? ((ConfigureDetails) action.getActionDetails()).getPreviousValue() : null,
- action.getActionDetails() instanceof ConfigureDetails ? ((ConfigureDetails) action.getActionDetails()).getValue() : null,
- action.getActionDetails() instanceof ConnectDetails ? ((ConnectDetails) action.getActionDetails()).getSourceId() : null,
- action.getActionDetails() instanceof ConnectDetails ? ((ConnectDetails) action.getActionDetails()).getSourceName() : null,
- action.getActionDetails() instanceof ConnectDetails ? ((ConnectDetails) action.getActionDetails()).getSourceType() : null,
- action.getActionDetails() instanceof ConnectDetails ? ((ConnectDetails) action.getActionDetails()).getDestinationId() : null,
- action.getActionDetails() instanceof ConnectDetails ? ((ConnectDetails) action.getActionDetails()).getDestinationName() : null,
- action.getActionDetails() instanceof ConnectDetails ? ((ConnectDetails) action.getActionDetails()).getDestinationType() : null,
- action.getActionDetails() instanceof ConnectDetails ? ((ConnectDetails) action.getActionDetails()).getRelationship() : null,
- action.getActionDetails() instanceof MoveDetails ? ((MoveDetails) action.getActionDetails()).getGroup() : null,
- action.getActionDetails() instanceof MoveDetails ? ((MoveDetails) action.getActionDetails()).getGroupId() : null,
- action.getActionDetails() instanceof MoveDetails ? ((MoveDetails) action.getActionDetails()).getPreviousGroup() : null,
- action.getActionDetails() instanceof MoveDetails ? ((MoveDetails) action.getActionDetails()).getPreviousGroupId() : null,
- action.getActionDetails() instanceof PurgeDetails ? ((PurgeDetails) action.getActionDetails()).getEndDate().getTime() : null
- };
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- List<Action> fullFlowConfigHistoryList = context.getEventAccess().getFlowChanges(0, Short.MAX_VALUE);
- actionIterator = fullFlowConfigHistoryList.iterator();
- }
-
- @Override
- public void close() {
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryProjectTableScanRule.java
deleted file mode 100644
index 9f33e4f00d..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.flowconfighistory;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link FlowConfigHistoryTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class FlowConfigHistoryProjectTableScanRule extends RelOptRule {
- public static final FlowConfigHistoryProjectTableScanRule INSTANCE = new FlowConfigHistoryProjectTableScanRule();
-
- private FlowConfigHistoryProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(FlowConfigHistoryTableScan.class, none())),
- "BulletinProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final FlowConfigHistoryTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new FlowConfigHistoryTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.flowConfigHistoryTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryTable.java
deleted file mode 100644
index c5af557030..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryTable.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.flowconfighistory;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class FlowConfigHistoryTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private final ReportingContext context;
- private volatile int maxRecordsRead;
-
- private final Set<FlowConfigHistoryEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a Flow Configuration History table.
- */
- public FlowConfigHistoryTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- }
-
- @Override
- public String toString() {
- return "FlowConfigHistoryTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final FlowConfigHistoryEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- public Enumerator<Object> enumerator() {
- final FlowConfigHistoryEnumerator flowConfigHistoryEnumerator = new FlowConfigHistoryEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(flowConfigHistoryEnumerator);
- }
-
- return flowConfigHistoryEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new FlowConfigHistoryTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = Arrays.asList(
- "actionId",
- "actionTimestamp",
- "actionUserIdentity",
- "actionSourceId",
- "actionSourceName",
- "actionSourceType",
- "actionOperation",
- "configureDetailsName",
- "configureDetailsPreviousValue",
- "configureDetailsValue",
- "connectionSourceId",
- "connectionSourceName",
- "connectionSourceType",
- "connectionDestinationId",
- "connectionDestinationName",
- "connectionDestinationType",
- "connectionRelationship",
- "moveGroup",
- "moveGroupId",
- "movePreviousGroup",
- "movePreviousGroupId",
- "purgeEndDate"
- );
- final List<RelDataType> types = Arrays.asList(
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(long.class)
- );
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryTableScan.java
deleted file mode 100644
index b5ae01e6f6..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/flowconfighistory/FlowConfigHistoryTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.flowconfighistory;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a query for bulletin information.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class FlowConfigHistoryTableScan extends TableScan implements EnumerableRel {
- final FlowConfigHistoryTable flowConfigHistoryTable;
- final int[] fields;
-
- protected FlowConfigHistoryTableScan(final RelOptCluster cluster, final RelOptTable table, final FlowConfigHistoryTable flowConfigHistoryTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.flowConfigHistoryTable = flowConfigHistoryTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new FlowConfigHistoryTableScan(getCluster(), table, flowConfigHistoryTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(FlowConfigHistoryProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(FlowConfigHistoryTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsEnumerator.java
deleted file mode 100644
index 0da6270c80..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsEnumerator.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.reporting.sql.metrics;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.metrics.jvm.JmxJvmMetrics;
-import org.apache.nifi.metrics.jvm.JvmMetrics;
-import org.apache.nifi.processor.DataUnit;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.util.ArrayList;
-import java.util.concurrent.TimeUnit;
-
-import static java.lang.Thread.State.BLOCKED;
-import static java.lang.Thread.State.RUNNABLE;
-import static java.lang.Thread.State.TERMINATED;
-import static java.lang.Thread.State.TIMED_WAITING;
-
-public class JvmMetricsEnumerator implements Enumerator<Object> {
- private final ReportingContext context;
- private final ComponentLog logger;
- private final int[] fields;
-
- private volatile JvmMetrics virtualMachineMetrics;
- private Object currentRow;
- private boolean fetchMetrics = true;
-
- public JvmMetricsEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.context = context;
- this.logger = logger;
- this.fields = fields;
- reset();
- virtualMachineMetrics = JmxJvmMetrics.getInstance();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- if (fetchMetrics) {
- currentRow = filterColumns(virtualMachineMetrics);
- // Toggle fetchMetrics so only one row is returned
- fetchMetrics = false;
- return true;
- } else {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- // Reset fetchMetrics for the next scan
- fetchMetrics = true;
- return false;
- }
- }
-
- protected int getRecordsRead() {
- return 1;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final JvmMetrics metrics) {
- if (metrics == null) {
- return null;
- }
-
- final ArrayList<Object> rowList = new ArrayList<>();
- rowList.add(virtualMachineMetrics.daemonThreadCount());
- rowList.add(virtualMachineMetrics.threadCount());
-
- // Normalize the thread percentages to an integer 0-100
- Double d = virtualMachineMetrics.threadStatePercentages().get(BLOCKED);
- rowList.add((int) (100 * (d == null ? 0 : d)));
- d = virtualMachineMetrics.threadStatePercentages().get(RUNNABLE);
- rowList.add((int) (100 * (d == null ? 0 : d)));
- d = virtualMachineMetrics.threadStatePercentages().get(TERMINATED);
- rowList.add((int) (100 * (d == null ? 0 : d)));
- d = virtualMachineMetrics.threadStatePercentages().get(TIMED_WAITING);
- rowList.add((int) (100 * (d == null ? 0 : d)));
-
- rowList.add(virtualMachineMetrics.uptime());
- rowList.add(virtualMachineMetrics.heapUsed(DataUnit.B));
- rowList.add(virtualMachineMetrics.heapUsage());
- rowList.add(virtualMachineMetrics.nonHeapUsage());
- rowList.add(virtualMachineMetrics.fileDescriptorUsage());
- virtualMachineMetrics.garbageCollectors().values().forEach((gc) -> {
- rowList.add(gc.getRuns());
- rowList.add(gc.getTime(TimeUnit.MILLISECONDS));
- });
-
- final Object[] row = rowList.toArray();
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- // Reset fetchMetrics for the next scan
- fetchMetrics = true;
- }
-
- @Override
- public void close() {
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsProjectTableScanRule.java
deleted file mode 100644
index dbb3a71bb6..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.metrics;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link JvmMetricsTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class JvmMetricsProjectTableScanRule extends RelOptRule {
- public static final JvmMetricsProjectTableScanRule INSTANCE = new JvmMetricsProjectTableScanRule();
-
- private JvmMetricsProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(JvmMetricsTableScan.class, none())),
- "JvmMetricsProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final JvmMetricsTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new JvmMetricsTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.jvmMetricsTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsTable.java
deleted file mode 100644
index 0056a34ec0..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsTable.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.metrics;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.metrics.jvm.JmxJvmMetrics;
-import org.apache.nifi.metrics.jvm.JvmMetrics;
-import org.apache.nifi.reporting.ReportingContext;
-import org.apache.nifi.reporting.util.metrics.MetricNames;
-import org.apache.nifi.reporting.util.metrics.MetricsService;
-
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-
-public class JvmMetricsTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private volatile ReportingContext context;
- private volatile int maxRecordsRead;
- private volatile MetricsService metricsService = new MetricsService();
- private volatile JvmMetrics virtualMachineMetrics;
-
- private final Set<JvmMetricsEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a JVM Metrics table.
- */
- public JvmMetricsTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- virtualMachineMetrics = JmxJvmMetrics.getInstance();
- }
-
- @Override
- public String toString() {
- return "JvmMetricsTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final JvmMetricsEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- public Enumerator<Object> enumerator() {
- final JvmMetricsEnumerator jvmMetricsEnumerator = new JvmMetricsEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(jvmMetricsEnumerator);
- }
-
- return jvmMetricsEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new JvmMetricsTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = Stream.of(
- MetricNames.JVM_DAEMON_THREAD_COUNT,
- MetricNames.JVM_THREAD_COUNT,
- MetricNames.JVM_THREAD_STATES_BLOCKED,
- MetricNames.JVM_THREAD_STATES_RUNNABLE,
- MetricNames.JVM_THREAD_STATES_TERMINATED,
- MetricNames.JVM_THREAD_STATES_TIMED_WAITING,
- MetricNames.JVM_UPTIME,
- MetricNames.JVM_HEAP_USED,
- MetricNames.JVM_HEAP_USAGE,
- MetricNames.JVM_NON_HEAP_USAGE,
- MetricNames.JVM_FILE_DESCRIPTOR_USAGE
- ).map((name) -> name.replace(".", "_").replace("-","_")).collect(Collectors.toList());
- final List<RelDataType> types = new ArrayList<>();
- types.add(typeFactory.createJavaType(int.class));
- types.add(typeFactory.createJavaType(int.class));
- types.add(typeFactory.createJavaType(int.class));
- types.add(typeFactory.createJavaType(int.class));
- types.add(typeFactory.createJavaType(int.class));
- types.add(typeFactory.createJavaType(int.class));
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createJavaType(double.class));
- types.add(typeFactory.createJavaType(double.class));
- types.add(typeFactory.createJavaType(double.class));
- types.add(typeFactory.createJavaType(double.class));
-
- // Add fields for the garbage collectors
- metricsService.getMetrics(virtualMachineMetrics);
- for (Map.Entry<String, JvmMetrics.GarbageCollectorStats> entry : virtualMachineMetrics.garbageCollectors().entrySet()) {
- final String gcName = entry.getKey().replace(" ", "").replace("-","_");
- names.add((MetricNames.JVM_GC_RUNS + "_" + gcName).replace(".", "_"));
- names.add((MetricNames.JVM_GC_TIME + "_" + gcName).replace(".", "_"));
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createJavaType(long.class));
- }
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsTableScan.java
deleted file mode 100644
index 25279287ba..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/metrics/JvmMetricsTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.metrics;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a snapshot of metrics for NiFi's JVM.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class JvmMetricsTableScan extends TableScan implements EnumerableRel {
- final JvmMetricsTable jvmMetricsTable;
- final int[] fields;
-
- protected JvmMetricsTableScan(final RelOptCluster cluster, final RelOptTable table, final JvmMetricsTable jvmMetricsTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.jvmMetricsTable = jvmMetricsTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new JvmMetricsTableScan(getCluster(), table, jvmMetricsTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(JvmMetricsProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(JvmMetricsTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusEnumerator.java
deleted file mode 100644
index 45e1016ceb..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusEnumerator.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.reporting.sql.processgroupstatus;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.controller.status.ProcessGroupStatus;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-import org.apache.nifi.util.Tuple;
-
-import java.util.Deque;
-import java.util.Iterator;
-import java.util.LinkedList;
-
-public class ProcessGroupStatusEnumerator implements Enumerator<Object> {
- private final ReportingContext context;
- private final ComponentLog logger;
- private final int[] fields;
-
- private Deque<Tuple<Iterator<ProcessGroupStatus>, String>> iteratorBreadcrumb;
- private Object currentRow;
- private int recordsRead = 0;
-
- public ProcessGroupStatusEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.context = context;
- this.logger = logger;
- this.fields = fields;
- reset();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- currentRow = null;
- final Tuple<ProcessGroupStatus, String> processGroupStatus;
- if (iteratorBreadcrumb.isEmpty()) {
- // Start the breadcrumb trail to follow recursively into process groups looking for connections
- ProcessGroupStatus rootStatus = context.getEventAccess().getControllerStatus();
- iteratorBreadcrumb.push(new Tuple<>(rootStatus.getProcessGroupStatus().iterator(), rootStatus.getId()));
- processGroupStatus = new Tuple<>(rootStatus, null);
- }else {
- processGroupStatus = getNextProcessGroupStatus();
- }
- if (processGroupStatus == null) {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- return false;
- }
-
- currentRow = filterColumns(processGroupStatus.getKey(), processGroupStatus.getValue());
-
- recordsRead++;
- return true;
- }
-
- protected int getRecordsRead() {
- return recordsRead;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final ProcessGroupStatus status, final String parentId) {
- if (status == null) {
- return null;
- }
-
- final Object[] row = new Object[]{
- status.getId(),
- parentId,
- status.getName(),
- status.getBytesRead(),
- status.getBytesWritten(),
- status.getBytesReceived(),
- status.getBytesSent(),
- status.getBytesTransferred(),
- status.getFlowFilesReceived(),
- status.getFlowFilesSent(),
- status.getFlowFilesTransferred(),
- status.getInputContentSize(),
- status.getInputCount(),
- status.getOutputContentSize(),
- status.getOutputCount(),
- status.getQueuedContentSize(),
- status.getActiveThreadCount(),
- status.getTerminatedThreadCount(),
- status.getQueuedCount(),
- status.getVersionedFlowState() == null ? null : status.getVersionedFlowState().name(),
- status.getProcessingNanos()
- };
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- // Clear the root PG status object so it is fetched fresh on the first record
- iteratorBreadcrumb = new LinkedList<>();
- }
-
- @Override
- public void close() {
- }
-
- private Tuple<ProcessGroupStatus, String> getNextProcessGroupStatus() {
- Tuple<Iterator<ProcessGroupStatus>, String> top = iteratorBreadcrumb.peek();
- if (top == null) {
- return null;
- }
- Iterator<ProcessGroupStatus> i = top.getKey();
- String parentId = top.getValue();
-
- if (i.hasNext()) {
- ProcessGroupStatus nextPG = i.next();
- iteratorBreadcrumb.push(new Tuple<>(nextPG.getProcessGroupStatus().iterator(), nextPG.getId()));
- return new Tuple<>(nextPG, parentId);
- } else {
- // No more child PGs, remove it from the breadcrumb trail and try again
- Tuple<Iterator<ProcessGroupStatus>, String> pop = iteratorBreadcrumb.pop();
- return getNextProcessGroupStatus();
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusProjectTableScanRule.java
deleted file mode 100644
index f7e684cecd..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.processgroupstatus;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link ProcessGroupStatusTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class ProcessGroupStatusProjectTableScanRule extends RelOptRule {
- public static final ProcessGroupStatusProjectTableScanRule INSTANCE = new ProcessGroupStatusProjectTableScanRule();
-
- private ProcessGroupStatusProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(ProcessGroupStatusTableScan.class, none())),
- "ProcessGroupStatusProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final ProcessGroupStatusTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new ProcessGroupStatusTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.processGroupStatusTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTable.java
deleted file mode 100644
index 0790eba069..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTable.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.processgroupstatus;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class ProcessGroupStatusTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private volatile ReportingContext context;
- private volatile int maxRecordsRead;
-
- private final Set<ProcessGroupStatusEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a ProcessGroup Status table.
- */
- public ProcessGroupStatusTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- }
-
- @Override
- public String toString() {
- return "ProcessGroupStatusTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final ProcessGroupStatusEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- public Enumerator<Object> enumerator() {
- final ProcessGroupStatusEnumerator processGroupStatusEnumerator = new ProcessGroupStatusEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(processGroupStatusEnumerator);
- }
-
- return processGroupStatusEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new ProcessGroupStatusTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = Arrays.asList(
- "id",
- "groupId",
- "name",
- "bytesRead",
- "bytesWritten",
- "bytesReceived",
- "bytesSent",
- "bytesTransferred",
- "flowFilesReceived",
- "flowFilesSent",
- "flowFilesTransferred",
- "inputContentSize",
- "inputCount",
- "outputContentSize",
- "outputCount",
- "queuedContentSize",
- "activeThreadCount",
- "terminatedThreadCount",
- "queuedCount",
- "versionedFlowState",
- "processingNanos"
- );
- final List<RelDataType> types = Arrays.asList(
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(long.class)
- );
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTableScan.java
deleted file mode 100644
index c4d6aed725..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.processgroupstatus;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a snapshot (at scan time) of status for all process groups (PGs) in the flow.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class ProcessGroupStatusTableScan extends TableScan implements EnumerableRel {
- final ProcessGroupStatusTable processGroupStatusTable;
- final int[] fields;
-
- protected ProcessGroupStatusTableScan(final RelOptCluster cluster, final RelOptTable table, final ProcessGroupStatusTable processGroupStatusTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.processGroupStatusTable = processGroupStatusTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new ProcessGroupStatusTableScan(getCluster(), table, processGroupStatusTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(ProcessGroupStatusProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(ProcessGroupStatusTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusEnumerator.java
deleted file mode 100644
index 4e550cef38..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusEnumerator.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.reporting.sql.processorstatus;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.controller.status.ProcessorStatus;
-import org.apache.nifi.controller.status.ProcessGroupStatus;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.util.Deque;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.concurrent.TimeUnit;
-
-public class ProcessorStatusEnumerator implements Enumerator<Object> {
- private final ReportingContext context;
- private final ComponentLog logger;
- private final int[] fields;
-
- private Deque<Iterator<ProcessGroupStatus>> iteratorBreadcrumb;
- private Iterator<ProcessorStatus> processorStatusIterator;
- private Object currentRow;
- private int recordsRead = 0;
-
- public ProcessorStatusEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.context = context;
- this.logger = logger;
- this.fields = fields;
- reset();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- currentRow = null;
- if (iteratorBreadcrumb.isEmpty()) {
- // Start the breadcrumb trail to follow recursively into process groups looking for connections
- ProcessGroupStatus rootStatus = context.getEventAccess().getControllerStatus();
- iteratorBreadcrumb.push(rootStatus.getProcessGroupStatus().iterator());
- processorStatusIterator = rootStatus.getProcessorStatus().iterator();
- }
-
- final ProcessorStatus connectionStatus = getNextProcessorStatus();
- if (connectionStatus == null) {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- return false;
- }
-
- currentRow = filterColumns(connectionStatus);
-
- recordsRead++;
- return true;
- }
-
- protected int getRecordsRead() {
- return recordsRead;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final ProcessorStatus status) {
- if (status == null) {
- return null;
- }
-
- final Object[] row = new Object[]{
- status.getId(),
- status.getGroupId(),
- status.getName(),
- status.getType(),
- status.getAverageLineageDuration(TimeUnit.MILLISECONDS),
- status.getBytesRead(),
- status.getBytesWritten(),
- status.getBytesReceived(),
- status.getBytesSent(),
- status.getFlowFilesRemoved(),
- status.getFlowFilesReceived(),
- status.getFlowFilesSent(),
- status.getInputCount(),
- status.getInputBytes(),
- status.getOutputCount(),
- status.getOutputBytes(),
- status.getActiveThreadCount(),
- status.getTerminatedThreadCount(),
- status.getInvocations(),
- status.getProcessingNanos(),
- status.getRunStatus().name(),
- status.getExecutionNode() == null ? null : status.getExecutionNode().name()
- };
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- // Clear the root PG status object so it is fetched fresh on the first record
- processorStatusIterator = null;
- iteratorBreadcrumb = new LinkedList<>();
- }
-
- @Override
- public void close() {
- }
-
- private ProcessorStatus getNextProcessorStatus() {
- if (processorStatusIterator != null && processorStatusIterator.hasNext()) {
- return processorStatusIterator.next();
- }
- // No more connections in this PG, so move to the next
- processorStatusIterator = null;
- Iterator<ProcessGroupStatus> i = iteratorBreadcrumb.peek();
- if (i == null) {
- return null;
- }
-
- if (i.hasNext()) {
- ProcessGroupStatus nextPG = i.next();
- iteratorBreadcrumb.push(nextPG.getProcessGroupStatus().iterator());
- processorStatusIterator = nextPG.getProcessorStatus().iterator();
- return getNextProcessorStatus();
- } else {
- // No more child PGs, remove it from the breadcrumb trail and try again
- iteratorBreadcrumb.pop();
- return getNextProcessorStatus();
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusProjectTableScanRule.java
deleted file mode 100644
index 7c6c375352..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.processorstatus;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link ProcessorStatusTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class ProcessorStatusProjectTableScanRule extends RelOptRule {
- public static final ProcessorStatusProjectTableScanRule INSTANCE = new ProcessorStatusProjectTableScanRule();
-
- private ProcessorStatusProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(ProcessorStatusTableScan.class, none())),
- "ProcessorStatusProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final ProcessorStatusTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new ProcessorStatusTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.processorStatusTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTable.java
deleted file mode 100644
index 13b2207c61..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTable.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.processorstatus;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class ProcessorStatusTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private volatile ReportingContext context;
- private volatile int maxRecordsRead;
-
- private final Set<ProcessorStatusEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a Processor Status table.
- */
- public ProcessorStatusTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- }
-
- @Override
- public String toString() {
- return "ProcessorStatusTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final ProcessorStatusEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"rawtypes"})
- public Enumerator<Object> enumerator() {
- final ProcessorStatusEnumerator processorStatusEnumerator = new ProcessorStatusEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(processorStatusEnumerator);
- }
-
- return processorStatusEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new ProcessorStatusTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = Arrays.asList(
- "id",
- "groupId",
- "name",
- "processorType",
- "averageLineageDuration",
- "bytesRead",
- "bytesWritten",
- "bytesReceived",
- "bytesSent",
- "flowFilesRemoved",
- "flowFilesReceived",
- "flowFilesSent",
- "inputCount",
- "inputBytes",
- "outputCount",
- "outputBytes",
- "activeThreadCount",
- "terminatedThreadCount",
- "invocations",
- "processingNanos",
- "runStatus",
- "executionNode"
- );
- final List<RelDataType> types = Arrays.asList(
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(int.class),
- typeFactory.createJavaType(long.class),
- typeFactory.createJavaType(String.class),
- typeFactory.createJavaType(String.class)
- );
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTableScan.java
deleted file mode 100644
index 0b726e00ad..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.processorstatus;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a snapshot (at scan time) of status for all processors in the flow.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class ProcessorStatusTableScan extends TableScan implements EnumerableRel {
- final ProcessorStatusTable processorStatusTable;
- final int[] fields;
-
- protected ProcessorStatusTableScan(final RelOptCluster cluster, final RelOptTable table, final ProcessorStatusTable processorStatusTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.processorStatusTable = processorStatusTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new ProcessorStatusTableScan(getCluster(), table, processorStatusTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(ProcessorStatusProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(ProcessorStatusTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceEnumerator.java
deleted file mode 100644
index 6bc2e58801..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceEnumerator.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.reporting.sql.provenance;
-
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.nifi.controller.status.ProcessGroupStatus;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.provenance.ProvenanceEventRecord;
-import org.apache.nifi.provenance.ProvenanceEventRepository;
-import org.apache.nifi.reporting.EventAccess;
-import org.apache.nifi.reporting.ReportingContext;
-import org.apache.nifi.reporting.util.provenance.ComponentMapHolder;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-
-public class ProvenanceEnumerator implements Enumerator<Object> {
- private static final int FETCH_SIZE = 1000;
- private final ComponentLog logger;
- private final int[] fields;
- private final ProvenanceEventRepository provenanceEventRepository;
- private List<ProvenanceEventRecord> provenanceEvents;
- private final ComponentMapHolder componentMapHolder;
- private final String nodeIdentifier;
-
- private Object currentRow;
- private long currentId = 0;
- private int currentIndex = 0; // Index into the current fetch
-
- public ProvenanceEnumerator(final ReportingContext context, final ComponentLog logger, final int[] fields) {
- this.logger = logger;
- this.fields = fields;
- final EventAccess eventAccess = context.getEventAccess();
- this.provenanceEventRepository = eventAccess.getProvenanceRepository();
- final ProcessGroupStatus procGroupStatus = eventAccess.getControllerStatus();
- this.componentMapHolder = ComponentMapHolder.createComponentMap(procGroupStatus);
-
- final boolean isClustered = context.isClustered();
- nodeIdentifier = context.getClusterNodeIdentifier();
- if (nodeIdentifier == null && isClustered) {
- logger.warn("This instance of NiFi is configured for clustering, but the Cluster Node Identifier is not yet available. "
- + "The contentPath and previousContentPath fields will be null for all rows in this query");
- }
-
- try {
- this.provenanceEvents = provenanceEventRepository.getEvents(0, FETCH_SIZE);
- } catch (IOException ioe) {
- logger.error("Error retrieving provenance events, queries will return no rows");
- }
- reset();
- }
-
- @Override
- public Object current() {
- return currentRow;
- }
-
- @Override
- public boolean moveNext() {
- if (provenanceEvents == null) {
- return false;
- }
- currentRow = null;
- if (currentIndex == provenanceEvents.size()) {
- // Need to fetch a new set of rows, starting with the last ID + 1
- try {
- provenanceEvents = provenanceEventRepository.getEvents(currentId + 1, FETCH_SIZE);
- currentIndex = 0;
- } catch (IOException ioe) {
- logger.error("Error retrieving provenance events, queries will return no further rows");
- return false;
- }
- }
-
- if (provenanceEvents.isEmpty()) {
- // If we are out of data, close the InputStream. We do this because
- // Calcite does not necessarily call our close() method.
- close();
- try {
- onFinish();
- } catch (final Exception e) {
- logger.error("Failed to perform tasks when enumerator was finished", e);
- }
-
- return false;
- }
-
- ProvenanceEventRecord provenanceEvent = provenanceEvents.get(currentIndex);
- currentRow = filterColumns(provenanceEvent);
- currentId = provenanceEvent.getEventId();
- currentIndex++;
- return true;
- }
-
- protected int getRecordsRead() {
- return 1;
- }
-
- protected void onFinish() {
- }
-
- private Object filterColumns(final ProvenanceEventRecord provenanceEvent) {
- if (provenanceEvent == null) {
- return null;
- }
-
- final ArrayList<Object> rowList = new ArrayList<>();
- rowList.add(provenanceEvent.getEventId());
- rowList.add(provenanceEvent.getEventType().name());
- rowList.add(provenanceEvent.getEventTime());
- rowList.add(provenanceEvent.getEventDuration());
- rowList.add(provenanceEvent.getLineageStartDate());
- rowList.add(provenanceEvent.getDetails());
- rowList.add(provenanceEvent.getComponentId());
- rowList.add(componentMapHolder.getComponentName(provenanceEvent.getComponentId()));
- rowList.add(provenanceEvent.getComponentType());
- final String processGroupId = componentMapHolder.getProcessGroupId(provenanceEvent.getComponentId(), provenanceEvent.getComponentType());
- rowList.add(processGroupId);
- rowList.add(componentMapHolder.getComponentName(processGroupId));
- rowList.add(provenanceEvent.getFlowFileUuid());
- rowList.add("org.apache.nifi.flowfile.FlowFile"); // entityType
- rowList.add(provenanceEvent.getFileSize());
- rowList.add(provenanceEvent.getPreviousFileSize());
- rowList.add(provenanceEvent.getUpdatedAttributes());
- rowList.add(provenanceEvent.getPreviousAttributes());
- if (nodeIdentifier != null) {
- final String contentPathBase = "/nifi-api/provenance-events/" + provenanceEvent.getEventId() + "/content/";
- final String nodeIdSuffix = "?clusterNodeId=" + nodeIdentifier;
- rowList.add(contentPathBase + "output" + nodeIdSuffix);
- rowList.add(contentPathBase + "input" + nodeIdSuffix);
- } else {
- rowList.add(null);
- rowList.add(null);
- }
- rowList.add(provenanceEvent.getParentUuids());
- rowList.add(provenanceEvent.getChildUuids());
- rowList.add(provenanceEvent.getTransitUri());
- rowList.add(provenanceEvent.getSourceSystemFlowFileIdentifier());
- rowList.add(provenanceEvent.getAlternateIdentifierUri());
-
- final Object[] row = rowList.toArray();
-
- // If we want no fields just return null
- if (fields == null) {
- return row;
- }
-
- // If we want only a single field, then Calcite is going to expect us to return
- // the actual value, NOT a 1-element array of values.
- if (fields.length == 1) {
- final int desiredCellIndex = fields[0];
- return row[desiredCellIndex];
- }
-
- // Create a new Object array that contains only the desired fields.
- final Object[] filtered = new Object[fields.length];
- for (int i = 0; i < fields.length; i++) {
- final int indexToKeep = fields[i];
- filtered[i] = row[indexToKeep];
- }
-
- return filtered;
- }
-
- @Override
- public void reset() {
- currentId = 0;
- currentIndex = 0;
- }
-
- @Override
- public void close() {
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceProjectTableScanRule.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceProjectTableScanRule.java
deleted file mode 100644
index 8062448273..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.provenance;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Planner rule that projects from a {@link ProvenanceTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class ProvenanceProjectTableScanRule extends RelOptRule {
- public static final ProvenanceProjectTableScanRule INSTANCE = new ProvenanceProjectTableScanRule();
-
- private ProvenanceProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(ProvenanceTableScan.class, none())),
- "ProvenanceProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final ProvenanceTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new ProvenanceTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.provenanceTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceTable.java
deleted file mode 100644
index 1c8263a66d..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceTable.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.provenance;
-
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-
-public class ProvenanceTable extends AbstractTable implements QueryableTable, TranslatableTable {
-
- private final ComponentLog logger;
-
- private RelDataType relDataType = null;
-
- private volatile ReportingContext context;
- private volatile int maxRecordsRead;
-
- private final Set<ProvenanceEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a Provenance events table.
- */
- public ProvenanceTable(final ReportingContext context, final ComponentLog logger) {
- this.context = context;
- this.logger = logger;
- }
-
- @Override
- public String toString() {
- return "ProvenanceTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final ProvenanceEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- public Enumerator<Object> enumerator() {
- final ProvenanceEnumerator provenanceEnumerator = new ProvenanceEnumerator(context, logger, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(provenanceEnumerator);
- }
-
- return provenanceEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new ProvenanceTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = Arrays.asList(
- "eventId",
- "eventType",
- "timestampMillis",
- "durationMillis",
- "lineageStart",
- "details",
- "componentId",
- "componentName",
- "componentType",
- "processGroupId",
- "processGroupName",
- "entityId",
- "entityType",
- "entitySize",
- "previousEntitySize",
- "updatedAttributes",
- "previousAttributes",
- "contentPath",
- "previousContentPath",
- "parentIds",
- "childIds",
- "transitUri",
- "remoteIdentifier",
- "alternateIdentifier"
- );
-
- final List<RelDataType> types = new ArrayList<>();
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createJavaType(long.class));
- types.add(typeFactory.createMapType(typeFactory.createJavaType(String.class), typeFactory.createJavaType(String.class)));
- types.add(typeFactory.createMapType(typeFactory.createJavaType(String.class), typeFactory.createJavaType(String.class)));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createArrayType(typeFactory.createJavaType(String.class), -1));
- types.add(typeFactory.createArrayType(typeFactory.createJavaType(String.class), -1));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
- types.add(typeFactory.createJavaType(String.class));
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceTableScan.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceTableScan.java
deleted file mode 100644
index f8420f0709..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/provenance/ProvenanceTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.provenance;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a query over the provenance repository.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class ProvenanceTableScan extends TableScan implements EnumerableRel {
- final ProvenanceTable provenanceTable;
- final int[] fields;
-
- protected ProvenanceTableScan(final RelOptCluster cluster, final RelOptTable table, final ProvenanceTable provenanceTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.provenanceTable = provenanceTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new ProvenanceTableScan(getCluster(), table, provenanceTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(ProvenanceProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(ProvenanceTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/ConnectionStatusRecursiveIterator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/ConnectionStatusRecursiveIterator.java
deleted file mode 100644
index 4701cbe823..0000000000
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/ConnectionStatusRecursiveIterator.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.reporting.sql.util;
-
-import org.apache.nifi.controller.status.ConnectionStatus;
-import org.apache.nifi.controller.status.ProcessGroupStatus;
-import org.apache.nifi.reporting.ReportingContext;
-
-import java.util.Deque;
-import java.util.Iterator;
-import java.util.LinkedList;
-
-public class ConnectionStatusRecursiveIterator implements Iterator<ConnectionStatus> {
- private final ReportingContext context;
-
- private Deque<Iterator<ProcessGroupStatus>> iteratorBreadcrumb;
- private Iterator<ConnectionStatus> connectionStatusIterator;
- private ConnectionStatus currentRow;
-
- public ConnectionStatusRecursiveIterator(final ReportingContext context) {
- this.context = context;
- iteratorBreadcrumb = new LinkedList<>();
- }
-
- @Override
- public boolean hasNext() {
- if (iteratorBreadcrumb.isEmpty()) {
- // Start the breadcrumb trail to follow recursively into process groups looking for connections
- ProcessGroupStatus rootStatus = context.getEventAccess().getControllerStatus();
- iteratorBreadcrumb.push(rootStatus.getProcessGroupStatus().iterator());
- connectionStatusIterator = rootStatus.getConnectionStatus().iterator();
- }
-
- currentRow = getNextConnectionStatus();
- return (currentRow != null);
- }
-
- @Override
- public ConnectionStatus next() {
- if (currentRow != null) {
- final ConnectionStatus result = currentRow;
- currentRow = null;
- return result;
- }
-
- if (hasNext()) {
- ConnectionStatus result = currentRow;
- currentRow = null;
- return result;
- } else {
- return null;
- }
- }
-
- private ConnectionStatus getNextConnectionStatus() {
- if (connectionStatusIterator != null && connectionStatusIterator.hasNext()) {
- return connectionStatusIterator.next();
- }
- // No more connections in this PG, so move to the next
- connectionStatusIterator = null;
- Iterator<ProcessGroupStatus> i = iteratorBreadcrumb.peek();
- if (i == null) {
- return null;
- }
-
- if (i.hasNext()) {
- ProcessGroupStatus nextPG = i.next();
- iteratorBreadcrumb.push(nextPG.getProcessGroupStatus().iterator());
- connectionStatusIterator = nextPG.getConnectionStatus().iterator();
- return getNextConnectionStatus();
- } else {
- // No more child PGs, remove it from the breadcrumb trail and try again
- iteratorBreadcrumb.pop();
- return getNextConnectionStatus();
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java
index 90eeff86d5..3b41617171 100644
--- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java
+++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java
@@ -59,6 +59,7 @@ import org.apache.nifi.util.db.JdbcProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.io.IOException;
@@ -81,6 +82,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
class TestQueryNiFiReportingTask {
@@ -528,6 +530,17 @@ class TestQueryNiFiReportingTask {
final List<Map<String, Object>> rows = mockRecordSinkService.getRows();
final String flowFileUuid = "testFlowFileUuid";
assertEquals(3, rows.size());
+
+ // Ensure that bulletins are properly ordered.
+ for (int i = 1; i < 3; i++) {
+ final Map<String, Object> values = rows.get(i);
+ final long timestamp = (long) values.get("bulletinTimestamp");
+
+ final Map<String, Object> previousValues = rows.get(i - 1);
+ final long previousTimestamp = (long) previousValues.get("bulletinTimestamp");
+ assertTrue(timestamp >= previousTimestamp);
+ }
+
// Validate the first row
Map<String, Object> row = rows.get(0);
assertEquals(14, row.size());
@@ -566,7 +579,7 @@ class TestQueryNiFiReportingTask {
assertEquals("Configure", row.get("actionOperation"));
}
- private MockQueryNiFiReportingTask initTask(Map<PropertyDescriptor, String> customProperties) throws InitializationException {
+ private MockQueryNiFiReportingTask initTask(final Map<PropertyDescriptor, String> customProperties) throws InitializationException {
final ComponentLog logger = mock(ComponentLog.class);
reportingTask = new MockQueryNiFiReportingTask();
@@ -575,7 +588,7 @@ class TestQueryNiFiReportingTask {
Mockito.when(initContext.getLogger()).thenReturn(logger);
reportingTask.initialize(initContext);
- Map<PropertyDescriptor, String> properties = new HashMap<>();
+ final Map<PropertyDescriptor, String> properties = new HashMap<>();
for (final PropertyDescriptor descriptor : reportingTask.getSupportedPropertyDescriptors()) {
properties.put(descriptor, descriptor.getDefaultValue());
}
@@ -653,10 +666,25 @@ class TestQueryNiFiReportingTask {
}
Mockito.when(eventAccess.getProvenanceRepository()).thenReturn(mockProvenanceRepository);
+ try {
+ Mockito.when(eventAccess.getProvenanceEvents(anyLong(), anyInt())).thenAnswer(new Answer<List<ProvenanceEventRecord>>() {
+ @Override
+ public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) throws Throwable {
+ final long startEventId = invocation.getArgument(0);
+ final int max = invocation.getArgument(1);
+ return mockProvenanceRepository.getEvents(startEventId, max);
+ }
+ });
+ } catch (final IOException e) {
+ // Won't happen
+ throw new RuntimeException(e);
+ }
mockBulletinRepository = new MockQueryBulletinRepository();
mockBulletinRepository.addBulletin(BulletinFactory.createBulletin("controller", "WARN", "test bulletin 2", "testFlowFileUuid"));
+ sleep(); // Sleep 2 milliseconds so that bulletins won't have the same timestamp
mockBulletinRepository.addBulletin(BulletinFactory.createBulletin(ComponentType.PROCESSOR.name().toLowerCase(), "INFO", "test bulletin 1", "testFlowFileUuid"));
+ sleep(); // Sleep 2 milliseconds so that bulletins won't have the same timestamp
mockBulletinRepository.addBulletin(BulletinFactory.createBulletin(ComponentType.CONTROLLER_SERVICE.name().toLowerCase(), "ERROR", "test bulletin 2", "testFlowFileUuid"));
Mockito.when(context.getBulletinRepository()).thenReturn(mockBulletinRepository);
@@ -664,6 +692,14 @@ class TestQueryNiFiReportingTask {
return reportingTask;
}
+ private void sleep() {
+ try {
+ Thread.sleep(2L);
+ } catch (final InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
private final class MockQueryNiFiReportingTask extends QueryNiFiReportingTask {
@Override
public long getCurrentTime() {
@@ -682,6 +718,15 @@ class TestQueryNiFiReportingTask {
@Override
public List<Bulletin> findBulletins(BulletinQuery bulletinQuery) {
+ if (bulletinQuery.getSourceType() == null) {
+ final List<Bulletin> allBulletins = new ArrayList<>();
+ for (final List<Bulletin> bulletins : bulletins.values()) {
+ allBulletins.addAll(bulletins);
+ }
+
+ return allBulletins;
+ }
+
return new ArrayList<>(
Optional.ofNullable(bulletins.get(bulletinQuery.getSourceType().name().toLowerCase()))
.orElse(Collections.emptyList()));
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 6e9bba75c7..41f809d0d4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -441,6 +441,11 @@
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-calcite-utils</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-schema-registry-service-api</artifactId>
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoinEnrichment.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoinEnrichment.java
index 34c04f9635..df44149c21 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoinEnrichment.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoinEnrichment.java
@@ -27,7 +27,6 @@ import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
@@ -71,8 +70,6 @@ import org.apache.nifi.util.db.JdbcProperties;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.sql.DriverManager;
-import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -244,16 +241,6 @@ public class JoinEnrichment extends BinFiles {
return relationships;
}
- @OnScheduled
- public void registerCalciteDriver() {
- // This is necessary in order to create a Calcite Connection for the SQL based Join strategy. We could put this in the SqlJoinStrategy but do it here to avoid doing this multiple times.
- try {
- DriverManager.registerDriver(new org.apache.calcite.jdbc.Driver());
- } catch (final SQLException e) {
- throw new ProcessException("Failed to load Calcite JDBC Driver", e);
- }
- }
-
@OnStopped
public synchronized void cleanup() throws Exception {
sqlJoinCache.close();
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
index 7fef2a5b30..21bf3372eb 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
@@ -19,30 +19,7 @@ package org.apache.nifi.processors.standard;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-import org.apache.calcite.config.CalciteConnectionProperty;
import org.apache.calcite.config.Lex;
-import org.apache.calcite.jdbc.CalciteConnection;
-import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParser.Config;
import org.apache.nifi.annotation.behavior.DynamicProperty;
@@ -64,6 +41,7 @@ import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
@@ -72,16 +50,35 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processors.standard.calcite.RecordPathFunctions;
import org.apache.nifi.processors.standard.calcite.RecordResultSetOutputStreamCallback;
-import org.apache.nifi.queryrecord.FlowFileTable;
+import org.apache.nifi.queryrecord.RecordDataSource;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.RecordSetWriterFactory;
import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.sql.CalciteDatabase;
+import org.apache.nifi.sql.NiFiTable;
+import org.apache.nifi.sql.NiFiTableSchema;
import org.apache.nifi.util.StopWatch;
import org.apache.nifi.util.StringUtils;
import org.apache.nifi.util.Tuple;
+import java.io.Closeable;
+import java.io.InputStream;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
import static org.apache.nifi.util.db.JdbcProperties.DEFAULT_PRECISION;
import static org.apache.nifi.util.db.JdbcProperties.DEFAULT_SCALE;
@@ -233,20 +230,13 @@ public class QueryRecord extends AbstractProcessor {
@Override
protected void init(final ProcessorInitializationContext context) {
- try {
- DriverManager.registerDriver(new org.apache.calcite.jdbc.Driver());
- } catch (final SQLException e) {
- throw new ProcessException("Failed to load Calcite JDBC Driver", e);
- }
-
- final List<PropertyDescriptor> properties = new ArrayList<>();
- properties.add(RECORD_READER_FACTORY);
- properties.add(RECORD_WRITER_FACTORY);
- properties.add(INCLUDE_ZERO_RECORD_FLOWFILES);
- properties.add(CACHE_SCHEMA);
- properties.add(DEFAULT_PRECISION);
- properties.add(DEFAULT_SCALE);
- this.properties = Collections.unmodifiableList(properties);
+ this.properties = List.of(
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY,
+ INCLUDE_ZERO_RECORD_FLOWFILES,
+ CACHE_SCHEMA,
+ DEFAULT_PRECISION,
+ DEFAULT_SCALE);
relationships.add(REL_FAILURE);
relationships.add(REL_ORIGINAL);
@@ -309,7 +299,7 @@ public class QueryRecord extends AbstractProcessor {
private void clearQueue(final BlockingQueue<CachedStatement> statementQueue) {
CachedStatement stmt;
while ((stmt = statementQueue.poll()) != null) {
- closeQuietly(stmt.getStatement(), stmt.getConnection());
+ stmt.close();
}
}
@@ -340,7 +330,7 @@ public class QueryRecord extends AbstractProcessor {
writerSchema = recordSetWriterFactory.getSchema(originalAttributes, readerSchema);
} catch (final Exception e) {
- getLogger().error("Failed to determine Record Schema from {}; routing to failure", new Object[] {original, e});
+ getLogger().error("Failed to determine Record Schema from {}; routing to failure", original, e);
original = session.putAttribute(original, ROUTE_ATTRIBUTE_KEY, REL_FAILURE.getName());
session.transfer(original, REL_FAILURE);
return;
@@ -374,7 +364,7 @@ public class QueryRecord extends AbstractProcessor {
try {
transformed = session.write(transformed, writer);
} finally {
- closeQuietly(rs, queryResult);
+ closeQuietly(getLogger(), rs, queryResult);
}
recordsRead = Math.max(recordsRead, queryResult.getRecordsRead());
@@ -384,7 +374,7 @@ public class QueryRecord extends AbstractProcessor {
session.remove(transformed);
flowFileRemoved = true;
transformedFlowFiles.remove(transformed);
- getLogger().info("Transformed {} but the result contained no data so will not pass on a FlowFile", new Object[] {original});
+ getLogger().info("Transformed {} but the result contained no data so will not pass on a FlowFile", original);
} else {
final Map<String, String> attributesToAdd = new HashMap<>();
if (result.getAttributes() != null) {
@@ -421,16 +411,16 @@ public class QueryRecord extends AbstractProcessor {
}
}
- getLogger().info("Successfully queried {} in {} millis", new Object[] {original, elapsedMillis});
+ getLogger().info("Successfully queried {} in {} millis", original, elapsedMillis);
original = session.putAttribute(original, ROUTE_ATTRIBUTE_KEY, REL_ORIGINAL.getName());
session.transfer(original, REL_ORIGINAL);
} catch (final SQLException e) {
- getLogger().error("Unable to query {} due to {}", new Object[] {original, e.getCause() == null ? e : e.getCause()});
+ getLogger().error("Unable to query {} due to {}", original, e.getCause() == null ? e : e.getCause());
original = session.putAttribute(original, ROUTE_ATTRIBUTE_KEY, REL_FAILURE.getName());
session.remove(createdFlowFiles);
session.transfer(original, REL_FAILURE);
} catch (final Exception e) {
- getLogger().error("Unable to query {} due to {}", new Object[] {original, e});
+ getLogger().error("Unable to query {} due to {}", original, e);
original = session.putAttribute(original, ROUTE_ATTRIBUTE_KEY, REL_FAILURE.getName());
session.remove(createdFlowFiles);
session.transfer(original, REL_FAILURE);
@@ -452,69 +442,48 @@ public class QueryRecord extends AbstractProcessor {
return statementBuilder.get();
}
- private CachedStatement buildCachedStatement(final String sql, final ProcessSession session, final FlowFile flowFile, final RecordSchema schema,
- final RecordReaderFactory recordReaderFactory) {
-
- final CalciteConnection connection = createConnection();
- final SchemaPlus rootSchema = RecordPathFunctions.createRootSchema(connection);
-
- final FlowFileTable flowFileTable = new FlowFileTable(session, flowFile, schema, recordReaderFactory, getLogger());
- rootSchema.add("FLOWFILE", flowFileTable);
- rootSchema.setCacheEnabled(false);
-
+ private CachedStatement buildCachedStatement(final String sql, final RecordSchema recordSchema) {
try {
- final PreparedStatement stmt = connection.prepareStatement(sql);
- return new CachedStatement(stmt, flowFileTable, connection);
+ final CalciteDatabase database = new CalciteDatabase();
+ final NiFiTableSchema tableSchema = RecordDataSource.createTableSchema(recordSchema);
+ final NiFiTable flowFileTable = new NiFiTable("FLOWFILE", tableSchema, getLogger());
+ database.addTable(flowFileTable);
+ RecordPathFunctions.addToDatabase(database);
+
+ final PreparedStatement stmt = database.getConnection().prepareStatement(sql);
+ return new CachedStatement(stmt, flowFileTable, database);
} catch (final SQLException e) {
throw new ProcessException(e);
}
}
- private CalciteConnection createConnection() {
- final Properties properties = new Properties();
- properties.put(CalciteConnectionProperty.LEX.camelName(), Lex.MYSQL_ANSI.name());
-
- try {
- final Connection connection = DriverManager.getConnection("jdbc:calcite:", properties);
- final CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
- return calciteConnection;
- } catch (final Exception e) {
- throw new ProcessException(e);
- }
- }
-
-
- protected QueryResult query(final ProcessSession session, final FlowFile flowFile, final RecordSchema schema, final String sql, final RecordReaderFactory recordReaderFactory)
+ private QueryResult query(final ProcessSession session, final FlowFile flowFile, final RecordSchema schema, final String sql, final RecordReaderFactory recordReaderFactory)
throws SQLException {
- final Supplier<CachedStatement> statementBuilder = () -> buildCachedStatement(sql, session, flowFile, schema, recordReaderFactory);
+ final Supplier<CachedStatement> statementBuilder = () -> buildCachedStatement(sql, schema);
final CachedStatement cachedStatement = getStatement(sql, schema, statementBuilder);
- final PreparedStatement stmt = cachedStatement.getStatement();
- final FlowFileTable table = cachedStatement.getTable();
- table.setFlowFile(session, flowFile);
+ final PreparedStatement stmt = cachedStatement.statement();
+ final NiFiTable table = cachedStatement.table();
+ table.setDataSource(new RecordDataSource(schema, session, flowFile, recordReaderFactory, getLogger()));
final ResultSet rs;
try {
rs = stmt.executeQuery();
} catch (final Throwable t) {
- table.close();
+ cachedStatement.close();
throw t;
}
return new QueryResult() {
@Override
- public void close() throws IOException {
+ public void close() {
table.close();
final BlockingQueue<CachedStatement> statementQueue = statementQueues.getIfPresent(new Tuple<>(sql, schema));
if (statementQueue == null || !statementQueue.offer(cachedStatement)) {
- try {
- cachedStatement.getConnection().close();
- } catch (SQLException e) {
- throw new IOException("Failed to close statement", e);
- }
+ cachedStatement.close();
}
}
@@ -531,9 +500,11 @@ public class QueryRecord extends AbstractProcessor {
};
}
+ private static void closeQuietly(final AutoCloseable... closeables) {
+ closeQuietly(null, closeables);
+ }
-
- private void closeQuietly(final AutoCloseable... closeables) {
+ private static void closeQuietly(final ComponentLog logger, final AutoCloseable... closeables) {
if (closeables == null) {
return;
}
@@ -546,7 +517,9 @@ public class QueryRecord extends AbstractProcessor {
try {
closeable.close();
} catch (final Exception e) {
- getLogger().warn("Failed to close SQL resource", e);
+ if (logger != null) {
+ logger.warn("Failed to close {}", closeable, e);
+ }
}
}
}
@@ -592,27 +565,10 @@ public class QueryRecord extends AbstractProcessor {
int getRecordsRead();
}
- private static class CachedStatement {
- private final FlowFileTable table;
- private final PreparedStatement statement;
- private final Connection connection;
-
- public CachedStatement(final PreparedStatement statement, final FlowFileTable table, final Connection connection) {
- this.statement = statement;
- this.table = table;
- this.connection = connection;
- }
-
- public FlowFileTable getTable() {
- return table;
- }
-
- public PreparedStatement getStatement() {
- return statement;
- }
-
- public Connection getConnection() {
- return connection;
+ private record CachedStatement(PreparedStatement statement, NiFiTable table, CalciteDatabase database) implements Closeable {
+ @Override
+ public void close() {
+ closeQuietly(statement, table, database);
}
}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunctions.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunctions.java
index 30ad08849c..b21c951329 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunctions.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunctions.java
@@ -17,9 +17,6 @@
package org.apache.nifi.processors.standard.calcite;
-import org.apache.calcite.jdbc.CalciteConnection;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.impl.ScalarFunctionImpl;
import org.apache.nifi.record.path.FieldValue;
import org.apache.nifi.record.path.RecordPath;
import org.apache.nifi.record.path.RecordPathResult;
@@ -30,6 +27,7 @@ import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordField;
import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.sql.CalciteDatabase;
import java.util.ArrayList;
import java.util.Collections;
@@ -39,17 +37,15 @@ import java.util.Map;
import java.util.stream.Collectors;
public class RecordPathFunctions {
- public static SchemaPlus createRootSchema(final CalciteConnection calciteConnection) {
- final SchemaPlus rootSchema = calciteConnection.getRootSchema();
- rootSchema.add("RPATH", ScalarFunctionImpl.create(ObjectRecordPath.class, "eval"));
- rootSchema.add("RPATH_STRING", ScalarFunctionImpl.create(StringRecordPath.class, "eval"));
- rootSchema.add("RPATH_INT", ScalarFunctionImpl.create(IntegerRecordPath.class, "eval"));
- rootSchema.add("RPATH_LONG", ScalarFunctionImpl.create(LongRecordPath.class, "eval"));
- rootSchema.add("RPATH_DATE", ScalarFunctionImpl.create(DateRecordPath.class, "eval"));
- rootSchema.add("RPATH_DOUBLE", ScalarFunctionImpl.create(DoubleRecordPath.class, "eval"));
- rootSchema.add("RPATH_FLOAT", ScalarFunctionImpl.create(FloatRecordPath.class, "eval"));
-
- return rootSchema;
+
+ public static void addToDatabase(final CalciteDatabase database) {
+ database.addUserDefinedFunction("RPATH", ObjectRecordPath.class, "eval");
+ database.addUserDefinedFunction("RPATH_STRING", StringRecordPath.class, "eval");
+ database.addUserDefinedFunction("RPATH_INT", IntegerRecordPath.class, "eval");
+ database.addUserDefinedFunction("RPATH_LONG", LongRecordPath.class, "eval");
+ database.addUserDefinedFunction("RPATH_DATE", DateRecordPath.class, "eval");
+ database.addUserDefinedFunction("RPATH_DOUBLE", DoubleRecordPath.class, "eval");
+ database.addUserDefinedFunction("RPATH_FLOAT", FloatRecordPath.class, "eval");
}
public static class ObjectRecordPath extends RecordPathFunction {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCache.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCache.java
index a53f21a7e2..70b79f0eec 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCache.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCache.java
@@ -20,23 +20,16 @@ package org.apache.nifi.processors.standard.enrichment;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
-import org.apache.calcite.config.CalciteConnectionProperty;
-import org.apache.calcite.config.Lex;
-import org.apache.calcite.jdbc.CalciteConnection;
-import org.apache.calcite.schema.SchemaPlus;
import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processors.standard.calcite.RecordPathFunctions;
-import org.apache.nifi.queryrecord.FlowFileTable;
+import org.apache.nifi.queryrecord.RecordDataSource;
import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.sql.CalciteDatabase;
+import org.apache.nifi.sql.NiFiTable;
import org.apache.nifi.util.Tuple;
-import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
-import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -52,8 +45,9 @@ public class SqlJoinCache implements AutoCloseable {
this.logger = logger;
}
- public SqlJoinCalciteParameters getCalciteParameters(final String sql, final ProcessSession session, final RecordSchema schema, final RecordJoinInput originalInput,
- final RecordJoinInput enrichmentInput) throws SQLException {
+ public SqlJoinCalciteParameters getCalciteParameters(final String sql, final RecordSchema schema, final RecordJoinInput originalInput, final RecordJoinInput enrichmentInput)
+ throws SQLException {
+
final Tuple<String, RecordSchema> tuple = new Tuple<>(sql, schema);
final BlockingQueue<SqlJoinCalciteParameters> queue = calciteParameterQueues.get(tuple, key -> new LinkedBlockingQueue<>());
@@ -62,7 +56,7 @@ public class SqlJoinCache implements AutoCloseable {
return cachedStmt;
}
- return createCalciteParameters(sql, session, originalInput, enrichmentInput, queue);
+ return createCalciteParameters(sql, originalInput, enrichmentInput);
}
public void returnCalciteParameters(final String sql, final RecordSchema schema, final SqlJoinCalciteParameters parameters) {
@@ -74,35 +68,18 @@ public class SqlJoinCache implements AutoCloseable {
}
}
- private SqlJoinCalciteParameters createCalciteParameters(final String sql, final ProcessSession session, final RecordJoinInput originalInput, final RecordJoinInput enrichmentInput,
- final BlockingQueue<SqlJoinCalciteParameters> parameterQueue) throws SQLException {
- final CalciteConnection connection = createCalciteConnection();
-
- final SchemaPlus rootSchema = RecordPathFunctions.createRootSchema(connection);
+ private SqlJoinCalciteParameters createCalciteParameters(final String sql, final RecordJoinInput originalInput, final RecordJoinInput enrichmentInput) throws SQLException {
+ final CalciteDatabase database = new CalciteDatabase();
+ RecordPathFunctions.addToDatabase(database);
- final FlowFileTable originalTable = new FlowFileTable(session, originalInput.getFlowFile(), originalInput.getRecordSchema(), originalInput.getRecordReaderFactory(), logger);
- rootSchema.add("ORIGINAL", originalTable);
+ final NiFiTable originalTable = new NiFiTable(SqlJoinStrategy.ORIGINAL_TABLE_NAME, RecordDataSource.createTableSchema(originalInput.getRecordSchema()), logger);
+ database.addTable(originalTable);
- final FlowFileTable enrichmentTable = new FlowFileTable(session, enrichmentInput.getFlowFile(), enrichmentInput.getRecordSchema(), enrichmentInput.getRecordReaderFactory(), logger);
- rootSchema.add("ENRICHMENT", enrichmentTable);
+ final NiFiTable enrichmentTable = new NiFiTable(SqlJoinStrategy.ENRICHMENT_TABLE_NAME, RecordDataSource.createTableSchema(enrichmentInput.getRecordSchema()), logger);
+ database.addTable(enrichmentTable);
- rootSchema.setCacheEnabled(false);
-
- final PreparedStatement preparedStatement = connection.prepareStatement(sql);
- return new SqlJoinCalciteParameters(sql, connection, preparedStatement, originalTable, enrichmentTable);
- }
-
- private CalciteConnection createCalciteConnection() {
- final Properties properties = new Properties();
- properties.put(CalciteConnectionProperty.LEX.camelName(), Lex.MYSQL_ANSI.name());
-
- try {
- final Connection connection = DriverManager.getConnection("jdbc:calcite:", properties);
- final CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
- return calciteConnection;
- } catch (final Exception e) {
- throw new ProcessException(e);
- }
+ final PreparedStatement preparedStatement = database.getConnection().prepareStatement(sql);
+ return new SqlJoinCalciteParameters(sql, database, preparedStatement);
}
private void onCacheEviction(final Tuple<String, RecordSchema> key, final BlockingQueue<SqlJoinCalciteParameters> queue, final RemovalCause cause) {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCalciteParameters.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCalciteParameters.java
index b845c3d84a..b90f3ceb82 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCalciteParameters.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCalciteParameters.java
@@ -17,8 +17,7 @@
package org.apache.nifi.processors.standard.enrichment;
-import org.apache.calcite.jdbc.CalciteConnection;
-import org.apache.nifi.queryrecord.FlowFileTable;
+import org.apache.nifi.sql.CalciteDatabase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,46 +27,32 @@ public class SqlJoinCalciteParameters implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(SqlJoinCalciteParameters.class);
private final String sql;
- private final CalciteConnection connection;
+ private final CalciteDatabase database;
private final PreparedStatement preparedStatement;
- private final FlowFileTable originalTable;
- private final FlowFileTable enrichmentTable;
- public SqlJoinCalciteParameters(final String sql, final CalciteConnection connection, final PreparedStatement preparedStatement,
- final FlowFileTable originalTable, final FlowFileTable enrichmentTable) {
+ public SqlJoinCalciteParameters(final String sql, final CalciteDatabase database, final PreparedStatement preparedStatement) {
this.sql = sql;
- this.connection = connection;
+ this.database = database;
this.preparedStatement = preparedStatement;
- this.originalTable = originalTable;
- this.enrichmentTable = enrichmentTable;
}
public String getSql() {
return sql;
}
- public CalciteConnection getConnection() {
- return connection;
+ public CalciteDatabase getDatabase() {
+ return database;
}
public PreparedStatement getPreparedStatement() {
return preparedStatement;
}
- public FlowFileTable getOriginalTable() {
- return originalTable;
- }
-
- public FlowFileTable getEnrichmentTable() {
- return enrichmentTable;
- }
@Override
public void close() {
closeQuietly(preparedStatement, "Calcite Prepared Statement");
- closeQuietly(connection, "Calcite Connection");
- closeQuietly(originalTable, "Calcite 'Original' Table");
- closeQuietly(enrichmentTable, "Calcite 'Enrichment' Table");
+ closeQuietly(database, "Calcite Database");
}
private void closeQuietly(final AutoCloseable closeable, final String description) {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinStrategy.java
index 1db67f7dfe..00dbe1f2d1 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinStrategy.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinStrategy.java
@@ -19,16 +19,19 @@ package org.apache.nifi.processors.standard.enrichment;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.queryrecord.FlowFileTable;
+import org.apache.nifi.queryrecord.RecordDataSource;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
import org.apache.nifi.serialization.record.ResultSetRecordSet;
+import org.apache.nifi.sql.NiFiTable;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class SqlJoinStrategy implements RecordJoinStrategy {
+ public static final String ENRICHMENT_TABLE_NAME = "ENRICHMENT";
+ public static final String ORIGINAL_TABLE_NAME = "ORIGINAL";
private final SqlJoinCache cache;
private final ComponentLog logger;
@@ -46,13 +49,13 @@ public class SqlJoinStrategy implements RecordJoinStrategy {
@Override
public RecordJoinResult join(final RecordJoinInput originalInput, final RecordJoinInput enrichmentInput, final ProcessSession session, final RecordSchema outputSchema) throws SQLException {
- final SqlJoinCalciteParameters calciteParameters = cache.getCalciteParameters(sql, session, outputSchema, originalInput, enrichmentInput);
+ final SqlJoinCalciteParameters calciteParameters = cache.getCalciteParameters(sql, outputSchema, originalInput, enrichmentInput);
- final FlowFileTable originalTable = calciteParameters.getOriginalTable();
- final FlowFileTable enrichmentTable = calciteParameters.getEnrichmentTable();
+ final NiFiTable originalTable = calciteParameters.getDatabase().getTable(ORIGINAL_TABLE_NAME);
+ originalTable.setDataSource(new RecordDataSource(originalInput.getRecordSchema(), session, originalInput.getFlowFile(), originalInput.getRecordReaderFactory(), logger));
- originalTable.setFlowFile(session, originalInput.getFlowFile());
- enrichmentTable.setFlowFile(session, enrichmentInput.getFlowFile());
+ final NiFiTable enrichmentTable = calciteParameters.getDatabase().getTable(ENRICHMENT_TABLE_NAME);
+ enrichmentTable.setDataSource(new RecordDataSource(enrichmentInput.getRecordSchema(), session, enrichmentInput.getFlowFile(), enrichmentInput.getRecordReaderFactory(), logger));
final PreparedStatement stmt = calciteParameters.getPreparedStatement();
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileProjectTableScanRule.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileProjectTableScanRule.java
deleted file mode 100644
index a44a5d8cf8..0000000000
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileProjectTableScanRule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.queryrecord;
-
-import java.util.List;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-
-/**
- * Planner rule that projects from a {@link FlowFileTableScan} scan just the columns
- * needed to satisfy a projection. If the projection's expressions are trivial,
- * the projection is removed.
- */
-public class FlowFileProjectTableScanRule extends RelOptRule {
- public static final FlowFileProjectTableScanRule INSTANCE = new FlowFileProjectTableScanRule();
-
- private FlowFileProjectTableScanRule() {
- super(
- operand(LogicalProject.class,
- operand(FlowFileTableScan.class, none())),
- "FlowFileProjectTableScanRule");
- }
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final LogicalProject project = call.rel(0);
- final FlowFileTableScan scan = call.rel(1);
- final int[] fields = getProjectFields(project.getProjects());
-
- if (fields == null) {
- // Project contains expressions more complex than just field references.
- return;
- }
-
- call.transformTo(
- new FlowFileTableScan(
- scan.getCluster(),
- scan.getTable(),
- scan.flowFileTable,
- fields));
- }
-
- private int[] getProjectFields(List<RexNode> exps) {
- final int[] fields = new int[exps.size()];
-
- for (int i = 0; i < exps.size(); i++) {
- final RexNode exp = exps.get(i);
-
- if (exp instanceof RexInputRef) {
- fields[i] = ((RexInputRef) exp).getIndex();
- } else {
- return null; // not a simple projection
- }
- }
-
- return fields;
- }
-}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTable.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTable.java
deleted file mode 100644
index e462972387..0000000000
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTable.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.queryrecord;
-
-import org.apache.calcite.adapter.java.JavaTypeFactory;
-import org.apache.calcite.linq4j.AbstractEnumerable;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.linq4j.QueryProvider;
-import org.apache.calcite.linq4j.Queryable;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.QueryableTable;
-import org.apache.calcite.schema.Schema.TableType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.Schemas;
-import org.apache.calcite.schema.TranslatableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-import org.apache.calcite.util.Pair;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.serialization.RecordReaderFactory;
-import org.apache.nifi.serialization.record.DataType;
-import org.apache.nifi.serialization.record.Record;
-import org.apache.nifi.serialization.record.RecordField;
-import org.apache.nifi.serialization.record.RecordSchema;
-import org.apache.nifi.serialization.record.type.ArrayDataType;
-import org.apache.nifi.serialization.record.type.ChoiceDataType;
-
-import java.io.Closeable;
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public class FlowFileTable extends AbstractTable implements QueryableTable, TranslatableTable, Closeable {
-
- private final RecordReaderFactory recordReaderFactory;
- private final ComponentLog logger;
-
- private RecordSchema recordSchema;
- private RelDataType relDataType = null;
-
- private volatile ProcessSession session;
- private volatile FlowFile flowFile;
- private volatile int maxRecordsRead;
-
- private final Set<FlowFileEnumerator> enumerators = new HashSet<>();
-
- /**
- * Creates a FlowFile table.
- */
- public FlowFileTable(final ProcessSession session, final FlowFile flowFile, final RecordSchema schema, final RecordReaderFactory recordReaderFactory, final ComponentLog logger) {
- this.session = session;
- this.flowFile = flowFile;
- this.recordSchema = schema;
- this.recordReaderFactory = recordReaderFactory;
- this.logger = logger;
- }
-
- public void setFlowFile(final ProcessSession session, final FlowFile flowFile) {
- this.session = session;
- this.flowFile = flowFile;
- this.maxRecordsRead = 0;
- }
-
-
- @Override
- public String toString() {
- return "FlowFileTable";
- }
-
- public void close() {
- synchronized (enumerators) {
- for (final FlowFileEnumerator enumerator : enumerators) {
- enumerator.close();
- }
- }
- }
-
- /**
- * Returns an enumerable over a given projection of the fields.
- *
- * <p>
- * Called from generated code.
- */
- public Enumerable<Object> project(final int[] fields) {
- return new AbstractEnumerable<Object>() {
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- public Enumerator<Object> enumerator() {
- final FlowFileEnumerator flowFileEnumerator = new FlowFileEnumerator(session, flowFile, logger, recordReaderFactory, fields) {
- @Override
- protected void onFinish() {
- final int recordCount = getRecordsRead();
- if (recordCount > maxRecordsRead) {
- maxRecordsRead = recordCount;
- }
- }
-
- @Override
- public void close() {
- synchronized (enumerators) {
- enumerators.remove(this);
- }
- super.close();
- }
- };
-
- synchronized (enumerators) {
- enumerators.add(flowFileEnumerator);
- }
-
- return flowFileEnumerator;
- }
- };
- }
-
- public int getRecordsRead() {
- return maxRecordsRead;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public Expression getExpression(final SchemaPlus schema, final String tableName, final Class clazz) {
- return Schemas.tableExpression(schema, getElementType(), tableName, clazz);
- }
-
- @Override
- public Type getElementType() {
- return Object[].class;
- }
-
- @Override
- public <T> Queryable<T> asQueryable(final QueryProvider queryProvider, final SchemaPlus schema, final String tableName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RelNode toRel(final RelOptTable.ToRelContext context, final RelOptTable relOptTable) {
- // Request all fields.
- final int fieldCount = relOptTable.getRowType().getFieldCount();
- final int[] fields = new int[fieldCount];
- for (int i = 0; i < fieldCount; i++) {
- fields[i] = i;
- }
-
- return new FlowFileTableScan(context.getCluster(), relOptTable, this, fields);
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (relDataType != null) {
- return relDataType;
- }
-
- final List<String> names = new ArrayList<>();
- final List<RelDataType> types = new ArrayList<>();
-
- final JavaTypeFactory javaTypeFactory = (JavaTypeFactory) typeFactory;
- for (final RecordField field : recordSchema.getFields()) {
- names.add(field.getFieldName());
- final RelDataType relDataType = getRelDataType(field.getDataType(), javaTypeFactory);
- types.add(javaTypeFactory.createTypeWithNullability(relDataType, field.isNullable()));
- }
-
- relDataType = typeFactory.createStructType(Pair.zip(names, types));
- return relDataType;
- }
-
- private RelDataType getRelDataType(final DataType fieldType, final JavaTypeFactory typeFactory) {
- switch (fieldType.getFieldType()) {
- case BOOLEAN:
- return typeFactory.createJavaType(boolean.class);
- case BYTE:
- return typeFactory.createJavaType(byte.class);
- case CHAR:
- return typeFactory.createJavaType(char.class);
- case DATE:
- return typeFactory.createJavaType(java.sql.Date.class);
- case DOUBLE:
- return typeFactory.createJavaType(double.class);
- case FLOAT:
- return typeFactory.createJavaType(float.class);
- case INT:
- return typeFactory.createJavaType(int.class);
- case SHORT:
- return typeFactory.createJavaType(short.class);
- case TIME:
- return typeFactory.createJavaType(java.sql.Time.class);
- case TIMESTAMP:
- return typeFactory.createJavaType(java.sql.Timestamp.class);
- case LONG:
- return typeFactory.createJavaType(long.class);
- case STRING:
- return typeFactory.createJavaType(String.class);
- case ARRAY:
- ArrayDataType array = (ArrayDataType) fieldType;
- return typeFactory.createArrayType(getRelDataType(array.getElementType(), typeFactory), -1);
- case RECORD:
- return typeFactory.createJavaType(Record.class);
- case MAP:
- return typeFactory.createJavaType(HashMap.class);
- case BIGINT:
- return typeFactory.createJavaType(BigInteger.class);
- case DECIMAL:
- return typeFactory.createJavaType(BigDecimal.class);
- case ENUM:
- return typeFactory.createJavaType(Enum.class);
- case CHOICE:
- final ChoiceDataType choiceDataType = (ChoiceDataType) fieldType;
- DataType widestDataType = choiceDataType.getPossibleSubTypes().get(0);
- for (final DataType possibleType : choiceDataType.getPossibleSubTypes()) {
- if (possibleType == widestDataType) {
- continue;
- }
- if (possibleType.getFieldType().isWiderThan(widestDataType.getFieldType())) {
- widestDataType = possibleType;
- continue;
- }
- if (widestDataType.getFieldType().isWiderThan(possibleType.getFieldType())) {
- continue;
- }
-
- // Neither is wider than the other.
- widestDataType = null;
- break;
- }
-
- // If one of the CHOICE data types is the widest, use it.
- if (widestDataType != null) {
- return getRelDataType(widestDataType, typeFactory);
- }
-
- // None of the data types is strictly the widest. Check if all data types are numeric.
- // This would happen, for instance, if the data type is a choice between float and integer.
- // If that is the case, we can use a String type for the table schema because all values will fit
- // into a String. This will still allow for casting, etc. if the query requires it.
- boolean allNumeric = true;
- for (final DataType possibleType : choiceDataType.getPossibleSubTypes()) {
- if (!isNumeric(possibleType)) {
- allNumeric = false;
- break;
- }
- }
-
- if (allNumeric) {
- return typeFactory.createJavaType(String.class);
- }
-
- // There is no specific type that we can use for the schema. This would happen, for instance, if our
- // CHOICE is between an integer and a Record.
- return typeFactory.createJavaType(Object.class);
- }
-
- throw new IllegalArgumentException("Unknown Record Field Type: " + fieldType);
- }
-
- private boolean isNumeric(final DataType dataType) {
- switch (dataType.getFieldType()) {
- case BIGINT:
- case BYTE:
- case DECIMAL:
- case DOUBLE:
- case FLOAT:
- case INT:
- case LONG:
- case SHORT:
- return true;
- default:
- return false;
- }
- }
-
- @Override
- public TableType getJdbcTableType() {
- return TableType.TEMPORARY_TABLE;
- }
-}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTableScan.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTableScan.java
deleted file mode 100644
index a9281ada7a..0000000000
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTableScan.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.queryrecord;
-
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.adapter.enumerable.PhysType;
-import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
-import org.apache.calcite.linq4j.tree.Blocks;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-
-import java.util.List;
-
-/**
- * Relational expression representing a scan of a FlowFile.
- *
- * <p>
- * Like any table scan, it serves as a leaf node of a query tree.
- * </p>
- */
-public class FlowFileTableScan extends TableScan implements EnumerableRel {
- final FlowFileTable flowFileTable;
- final int[] fields;
-
- protected FlowFileTableScan(final RelOptCluster cluster, final RelOptTable table, final FlowFileTable flowFileTable, final int[] fields) {
- super(cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), table);
-
- this.flowFileTable = flowFileTable;
- this.fields = fields;
- }
-
- @Override
- public RelNode copy(final RelTraitSet traitSet, final List<RelNode> inputs) {
- return new FlowFileTableScan(getCluster(), table, flowFileTable, fields);
- }
-
- @Override
- public RelWriter explainTerms(final RelWriter pw) {
- return super.explainTerms(pw).item("fields", Primitive.asList(fields));
- }
-
- @Override
- public RelDataType deriveRowType() {
- final List<RelDataTypeField> fieldList = table.getRowType().getFieldList();
- final RelDataTypeFactory.Builder builder = getCluster().getTypeFactory().builder();
- for (int field : fields) {
- builder.add(fieldList.get(field));
- }
- return builder.build();
- }
-
- @Override
- public void register(RelOptPlanner planner) {
- planner.addRule(FlowFileProjectTableScanRule.INSTANCE);
- }
-
- @Override
- public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
- PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
-
- return implementor.result(physType, Blocks.toBlock(
- Expressions.call(table.getExpression(FlowFileTable.class), "project", Expressions.constant(fields))));
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/RecordDataSource.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/RecordDataSource.java
new file mode 100644
index 0000000000..f82ce414ef
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/RecordDataSource.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.queryrecord;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.RecordSet;
+import org.apache.nifi.serialization.record.type.ArrayDataType;
+import org.apache.nifi.serialization.record.type.ChoiceDataType;
+import org.apache.nifi.serialization.record.type.MapDataType;
+import org.apache.nifi.sql.ArrayType;
+import org.apache.nifi.sql.ColumnSchema;
+import org.apache.nifi.sql.ColumnType;
+import org.apache.nifi.sql.MapType;
+import org.apache.nifi.sql.NiFiTableSchema;
+import org.apache.nifi.sql.ResettableDataSource;
+import org.apache.nifi.sql.RowStream;
+import org.apache.nifi.sql.ScalarType;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+public class RecordDataSource implements ResettableDataSource {
+ private final NiFiTableSchema tableSchema;
+ private final ProcessSession session;
+ private final FlowFile flowFile;
+ private final RecordReaderFactory readerFactory;
+ private final ComponentLog logger;
+
+
+ public RecordDataSource(final RecordSchema recordSchema, final ProcessSession session, final FlowFile flowFile, final RecordReaderFactory recordReaderFactory, final ComponentLog logger) {
+ this.tableSchema = createTableSchema(recordSchema);
+ this.session = session;
+ this.flowFile = flowFile;
+ this.readerFactory = recordReaderFactory;
+ this.logger = logger;
+ }
+
+ @Override
+ public NiFiTableSchema getSchema() {
+ return tableSchema;
+ }
+
+ @Override
+ public RowStream reset() throws IOException {
+ final InputStream in = session.read(flowFile);
+ final RecordReader reader;
+ try {
+ reader = readerFactory.createRecordReader(flowFile, in, logger);
+ } catch (final Exception e) {
+ in.close();
+ throw new IOException(e);
+ }
+
+ final RecordSet recordSet = reader.createRecordSet();
+
+ return new RowStream() {
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+
+ @Override
+ public Object[] nextRow() throws IOException {
+ final Record record = recordSet.next();
+ return record == null ? null : record.getValues();
+ }
+ };
+ }
+
+ public static NiFiTableSchema createTableSchema(final RecordSchema recordSchema) {
+ final List<ColumnSchema> columns = new ArrayList<>();
+
+ for (final RecordField field : recordSchema.getFields()) {
+ final String columnName = field.getFieldName();
+ final ColumnType columnType = getColumnType(field.getDataType());
+ final boolean nullable = field.isNullable();
+ columns.add(new ColumnSchema(columnName, columnType, nullable));
+ }
+
+ return new NiFiTableSchema(columns);
+ }
+
+ public static ColumnType getColumnType(final DataType fieldType) {
+ return switch (fieldType.getFieldType()) {
+ case BOOLEAN -> ScalarType.BOOLEAN;
+ case BYTE -> ScalarType.BYTE;
+ case UUID -> ScalarType.UUID;
+ case CHAR -> ScalarType.CHARACTER;
+ case DATE -> ScalarType.DATE;
+ case DOUBLE -> ScalarType.DOUBLE;
+ case FLOAT -> ScalarType.FLOAT;
+ case INT -> ScalarType.INTEGER;
+ case SHORT -> ScalarType.SHORT;
+ case TIME -> ScalarType.TIME;
+ case TIMESTAMP -> ScalarType.TIMESTAMP;
+ case LONG -> ScalarType.LONG;
+ case STRING, ENUM -> ScalarType.STRING;
+ case ARRAY -> new ArrayType(getColumnType(((ArrayDataType) fieldType).getElementType()));
+ case RECORD -> new ScalarType(Record.class);
+ case MAP -> {
+ final MapDataType mapDataType = (MapDataType) fieldType;
+ yield new MapType(ScalarType.STRING, getColumnType(mapDataType.getValueType()));
+ }
+ case BIGINT -> ScalarType.BIGINT;
+ case DECIMAL -> ScalarType.DECIMAL;
+ case CHOICE -> getChoiceColumnType(fieldType);
+ };
+ }
+
+ private static ColumnType getChoiceColumnType(final DataType fieldType) {
+ final ChoiceDataType choiceDataType = (ChoiceDataType) fieldType;
+ DataType widestDataType = choiceDataType.getPossibleSubTypes().get(0);
+ for (final DataType possibleType : choiceDataType.getPossibleSubTypes()) {
+ if (possibleType == widestDataType) {
+ continue;
+ }
+ if (possibleType.getFieldType().isWiderThan(widestDataType.getFieldType())) {
+ widestDataType = possibleType;
+ continue;
+ }
+ if (widestDataType.getFieldType().isWiderThan(possibleType.getFieldType())) {
+ continue;
+ }
+
+ // Neither is wider than the other.
+ widestDataType = null;
+ break;
+ }
+
+ // If one of the CHOICE data types is the widest, use it.
+ if (widestDataType != null) {
+ return getColumnType(widestDataType);
+ }
+
+ // None of the data types is strictly the widest. Check if all data types are numeric.
+ // This would happen, for instance, if the data type is a choice between float and integer.
+ // If that is the case, we can use a String type for the table schema because all values will fit
+ // into a String. This will still allow for casting, etc. if the query requires it.
+ boolean allNumeric = true;
+ for (final DataType possibleType : choiceDataType.getPossibleSubTypes()) {
+ if (!isNumeric(possibleType)) {
+ allNumeric = false;
+ break;
+ }
+ }
+
+ if (allNumeric) {
+ return ScalarType.STRING;
+ }
+
+ // There is no specific type that we can use for the schema. This would happen, for instance, if our
+ // CHOICE is between an integer and a Record.
+ return ScalarType.OBJECT;
+ }
+
+
+ private static boolean isNumeric(final DataType dataType) {
+ return switch (dataType.getFieldType()) {
+ case BIGINT, BYTE, DECIMAL, DOUBLE, FLOAT, INT, LONG, SHORT -> true;
+ default -> false;
+ };
+ }
+}