You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2018/12/18 17:55:37 UTC
[GitHub] bowenli86 closed pull request #6979: [FLINK-10168] [Core &
DataStream] Add FileFilter to FileInputFormat and FileModTimeFilter which
sets a read start position for files by modification time
bowenli86 closed pull request #6979: [FLINK-10168] [Core & DataStream] Add FileFilter to FileInputFormat and FileModTimeFilter which sets a read start position for files by modification time
URL: https://github.com/apache/flink/pull/6979
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/docs/_includes/generated/checkpointing_configuration.html b/docs/_includes/generated/checkpointing_configuration.html
index 8f5ce7b341d..6ad5eeeb091 100644
--- a/docs/_includes/generated/checkpointing_configuration.html
+++ b/docs/_includes/generated/checkpointing_configuration.html
@@ -30,7 +30,7 @@
<tr>
<td><h5>state.backend.local-recovery</h5></td>
<td style="word-wrap: break-word;">false</td>
- <td></td>
+ <td>This option configures local recovery for this state backend. By default, local recovery is deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option.</td>
</tr>
<tr>
<td><h5>state.checkpoints.dir</h5></td>
@@ -50,7 +50,7 @@
<tr>
<td><h5>taskmanager.state.local.root-dirs</h5></td>
<td style="word-wrap: break-word;">(none)</td>
- <td></td>
+ <td>The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option</td>
</tr>
</tbody>
</table>
diff --git a/docs/dev/batch/dataset_transformations.md b/docs/dev/batch/dataset_transformations.md
index 96c04c9d548..cf2da3fa91a 100644
--- a/docs/dev/batch/dataset_transformations.md
+++ b/docs/dev/batch/dataset_transformations.md
@@ -712,7 +712,7 @@ class MyCombinableGroupReducer
out: Collector[String]): Unit =
{
val r: (String, Int) =
- in.asScala.reduce( (a,b) => (a._1, a._2 + b._2) )
+ in.iterator.asScala.reduce( (a,b) => (a._1, a._2 + b._2) )
// concat key and sum and emit
out.collect (r._1 + "-" + r._2)
}
@@ -722,7 +722,7 @@ class MyCombinableGroupReducer
out: Collector[(String, Int)]): Unit =
{
val r: (String, Int) =
- in.asScala.reduce( (a,b) => (a._1, a._2 + b._2) )
+ in.iterator.asScala.reduce( (a,b) => (a._1, a._2 + b._2) )
// emit tuple with key and sum
out.collect(r)
}
diff --git a/docs/dev/batch/index.md b/docs/dev/batch/index.md
index d0043647227..0a498dfe2a1 100644
--- a/docs/dev/batch/index.md
+++ b/docs/dev/batch/index.md
@@ -401,12 +401,14 @@ DataSet<Integer> result = in.partitionByRange(0)
<tr>
<td><strong>Custom Partitioning</strong></td>
<td>
- <p>Manually specify a partitioning over the data.
+ <p>Assigns records based on a key to a specific partition using a custom Partitioner function.
+ The key can be specified as position key, expression key, and key selector function.
<br/>
- <i>Note</i>: This method works only on single field keys.</p>
+ <i>Note</i>: This method only works with a single field key.</p>
{% highlight java %}
DataSet<Tuple2<String,Integer>> in = // [...]
-DataSet<Integer> result = in.partitionCustom(Partitioner<K> partitioner, key)
+DataSet<Integer> result = in.partitionCustom(partitioner, key)
+ .mapPartition(new PartitionMapper());
{% endhighlight %}
</td>
</tr>
@@ -703,13 +705,14 @@ val result = in.partitionByRange(0).mapPartition { ... }
<tr>
<td><strong>Custom Partitioning</strong></td>
<td>
- <p>Manually specify a partitioning over the data.
+ <p>Assigns records based on a key to a specific partition using a custom Partitioner function.
+ The key can be specified as position key, expression key, and key selector function.
<br/>
- <i>Note</i>: This method works only on single field keys.</p>
+ <i>Note</i>: This method only works with a single field key.</p>
{% highlight scala %}
val in: DataSet[(Int, String)] = // [...]
val result = in
- .partitionCustom(partitioner: Partitioner[K], key)
+ .partitionCustom(partitioner, key).mapPartition { ... }
{% endhighlight %}
</td>
</tr>
diff --git a/docs/dev/connectors/streamfile_sink.md b/docs/dev/connectors/streamfile_sink.md
index 8f50675ccbc..82ab5620571 100644
--- a/docs/dev/connectors/streamfile_sink.md
+++ b/docs/dev/connectors/streamfile_sink.md
@@ -60,17 +60,14 @@ Basic usage thus looks like this:
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
{% highlight java %}
-import org.apache.flink.api.common.serialization.Encoder;
+import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
DataStream<String> input = ...;
final StreamingFileSink<String> sink = StreamingFileSink
- .forRowFormat(new Path(outputPath), (Encoder<String>) (element, stream) -> {
- PrintStream out = new PrintStream(stream);
- out.println(element.f1);
- })
+ .forRowFormat(new Path(outputPath), new SimpleStringEncoder<>("UTF-8"))
.build();
input.addSink(sink);
@@ -79,19 +76,16 @@ input.addSink(sink);
</div>
<div data-lang="scala" markdown="1">
{% highlight scala %}
-import org.apache.flink.api.common.serialization.Encoder
+import org.apache.flink.api.common.serialization.SimpleStringEncoder
import org.apache.flink.core.fs.Path
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink
val input: DataStream[String] = ...
-final StreamingFileSink[String] sink = StreamingFileSink
- .forRowFormat(new Path(outputPath), (element, stream) => {
- val out = new PrintStream(stream)
- out.println(element.f1)
- })
- .build()
-
+val sink: StreamingFileSink[String] = StreamingFileSink
+ .forRowFormat(new Path(outputPath), new SimpleStringEncoder[String]("UTF-8"))
+ .build()
+
input.addSink(sink)
{% endhighlight %}
diff --git a/docs/dev/libs/ml/quickstart.md b/docs/dev/libs/ml/quickstart.md
index e056b28b505..2e9a7b9505c 100644
--- a/docs/dev/libs/ml/quickstart.md
+++ b/docs/dev/libs/ml/quickstart.md
@@ -153,6 +153,8 @@ A conversion can be done using a simple normalizer mapping function:
{% highlight scala %}
+import org.apache.flink.ml.math.Vector
+
def normalizer : LabeledVector => LabeledVector = {
lv => LabeledVector(if (lv.label > 0.0) 1.0 else -1.0, lv.vector)
}
diff --git a/docs/dev/stream/operators/index.md b/docs/dev/stream/operators/index.md
index 422dbbf9136..b89fa898fe7 100644
--- a/docs/dev/stream/operators/index.md
+++ b/docs/dev/stream/operators/index.md
@@ -434,14 +434,14 @@ IterativeStream<Long> iteration = initialStream.iterate();
DataStream<Long> iterationBody = iteration.map (/*do something*/);
DataStream<Long> feedback = iterationBody.filter(new FilterFunction<Long>(){
@Override
- public boolean filter(Integer value) throws Exception {
+ public boolean filter(Long value) throws Exception {
return value > 0;
}
});
iteration.closeWith(feedback);
DataStream<Long> output = iterationBody.filter(new FilterFunction<Long>(){
@Override
- public boolean filter(Integer value) throws Exception {
+ public boolean filter(Long value) throws Exception {
return value <= 0;
}
});
diff --git a/docs/dev/stream/operators/joining.md b/docs/dev/stream/operators/joining.md
index b95aaddbbaa..7e9915e5e83 100644
--- a/docs/dev/stream/operators/joining.md
+++ b/docs/dev/stream/operators/joining.md
@@ -101,7 +101,7 @@ orangeStream.join(greenStream)
</div>
## Sliding Window Join
-When performing a sliding window join, all elements with a common key and common sliding window are joined are pairwise combinations and passed on to the `JoinFunction` or `FlatJoinFunction`. Elements of one stream that do not have elements from the other stream in the current sliding window are not emitted! Note that some elements might be joined in one sliding window but not in another!
+When performing a sliding window join, all elements with a common key and common sliding window are joined as pairwise combinations and passed on to the `JoinFunction` or `FlatJoinFunction`. Elements of one stream that do not have elements from the other stream in the current sliding window are not emitted! Note that some elements might be joined in one sliding window but not in another!
<img src="{{ site.baseurl }}/fig/sliding-window-join.svg" class="center" style="width: 80%;" />
diff --git a/docs/dev/stream/operators/windows.md b/docs/dev/stream/operators/windows.md
index cd47ed567fc..5f6c8ac45b3 100644
--- a/docs/dev/stream/operators/windows.md
+++ b/docs/dev/stream/operators/windows.md
@@ -783,7 +783,7 @@ When the window is closed, the `ProcessWindowFunction` will be provided with the
This allows it to incrementally compute windows while having access to the
additional window meta information of the `ProcessWindowFunction`.
-<span class="label label-info">Note</span> You can also the legacy `WindowFunction` instead of
+<span class="label label-info">Note</span> You can also use the legacy `WindowFunction` instead of
`ProcessWindowFunction` for incremental window aggregation.
#### Incremental Window Aggregation with ReduceFunction
@@ -1034,7 +1034,7 @@ different keys and events for all of them currently fall into the *[12:00, 13:00
then there will be 1000 window instances that each have their own keyed per-window state.
There are two methods on the `Context` object that a `process()` invocation receives that allow
-access two the two types of state:
+access to the two types of state:
- `globalState()`, which allows access to keyed state that is not scoped to a window
- `windowState()`, which allows access to keyed state that is also scoped to the window
diff --git a/docs/dev/stream/state/broadcast_state.md b/docs/dev/stream/state/broadcast_state.md
index f336a855a3a..628a6308b57 100644
--- a/docs/dev/stream/state/broadcast_state.md
+++ b/docs/dev/stream/state/broadcast_state.md
@@ -94,7 +94,7 @@ The exact type of the function depends on the type of the non-broadcasted stream
</div>
{% highlight java %}
-DataStream<Match> output = colorPartitionedStream
+DataStream<String> output = colorPartitionedStream
.connect(ruleBroadcastStream)
.process(
@@ -107,7 +107,7 @@ DataStream<Match> output = colorPartitionedStream
new KeyedBroadcastProcessFunction<Color, Item, Rule, String>() {
// my matching logic
}
- )
+ );
{% endhighlight %}
### BroadcastProcessFunction and KeyedBroadcastProcessFunction
diff --git a/docs/dev/stream/state/schema_evolution.md b/docs/dev/stream/state/schema_evolution.md
index 2fb10a74ff1..e4c2d4aa441 100644
--- a/docs/dev/stream/state/schema_evolution.md
+++ b/docs/dev/stream/state/schema_evolution.md
@@ -74,7 +74,7 @@ serialization schema than the previous serializer; if so, the previous serialize
and written back to bytes again with the new serializer.
Further details about the migration process is out of the scope of this documentation; please refer to
-[here]({{ site.baseurl }}/dev/stream/state/custom_serialization).
+[here]({{ site.baseurl }}/dev/stream/state/custom_serialization.html).
## Supported data types for schema evolution
diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md
index decf1dbe9de..a3109f2c2fa 100644
--- a/docs/dev/stream/state/state.md
+++ b/docs/dev/stream/state/state.md
@@ -142,7 +142,7 @@ is available in a `RichFunction` has these methods for accessing state:
* `ValueState<T> getState(ValueStateDescriptor<T>)`
* `ReducingState<T> getReducingState(ReducingStateDescriptor<T>)`
* `ListState<T> getListState(ListStateDescriptor<T>)`
-* `AggregatingState<IN, OUT> getAggregatingState(AggregatingState<IN, OUT>)`
+* `AggregatingState<IN, OUT> getAggregatingState(AggregatingStateDescriptor<IN, ACC, OUT>)`
* `FoldingState<T, ACC> getFoldingState(FoldingStateDescriptor<T, ACC>)`
* `MapState<UK, UV> getMapState(MapStateDescriptor<UK, UV>)`
diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md
index d8677714fa8..c05102cf7a9 100644
--- a/docs/dev/table/connect.md
+++ b/docs/dev/table/connect.md
@@ -34,7 +34,7 @@ This page describes how to declare built-in table sources and/or table sinks and
Dependencies
------------
-The following table list all available connectors and formats. Their mutual compatibility is tagged in the corresponding sections for [table connectors](connect.html#table-connectors) and [table formats](connect.html#table-formats). The following table provides dependency information for both projects using a build automation tool (such as Maven or SBT) and SQL Client with SQL JAR bundles.
+The following tables list all available connectors and formats. Their mutual compatibility is tagged in the corresponding sections for [table connectors](connect.html#table-connectors) and [table formats](connect.html#table-formats). The following tables provide dependency information for both projects using a build automation tool (such as Maven or SBT) and SQL Client with SQL JAR bundles.
{% if site.is_stable %}
@@ -43,7 +43,7 @@ The following table list all available connectors and formats. Their mutual comp
| Name | Version | Maven dependency | SQL Client JAR |
| :---------------- | :------------------ | :--------------------------- | :----------------------|
| Filesystem | | Built-in | Built-in |
-| Elasticsearch | 6 | `flink-connector-elasticsearch6` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-elasticsearch6{{site.scala_version_suffix}}/{{site.version}}/flink-connector-elasticsearch6{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) |
+| Elasticsearch | 6 | `flink-connector-elasticsearch6` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch6{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-elasticsearch6{{site.scala_version_suffix}}-{{site.version}}.jar) |
| Apache Kafka | 0.8 | `flink-connector-kafka-0.8` | Not available |
| Apache Kafka | 0.9 | `flink-connector-kafka-0.9` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.9{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.9{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) |
| Apache Kafka | 0.10 | `flink-connector-kafka-0.10` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.10{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.10{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) |
@@ -60,7 +60,7 @@ The following table list all available connectors and formats. Their mutual comp
{% else %}
-This table is only available for stable releases.
+These tables are only available for stable releases.
{% endif %}
@@ -145,7 +145,7 @@ tableEnvironment
" {\"name\": \"user\", \"type\": \"long\"}," +
" {\"name\": \"message\", \"type\": [\"string\", \"null\"]}" +
" ]" +
- "}" +
+ "}"
)
)
@@ -154,7 +154,7 @@ tableEnvironment
new Schema()
.field("rowtime", Types.SQL_TIMESTAMP)
.rowtime(new Rowtime()
- .timestampsFromField("ts")
+ .timestampsFromField("timestamp")
.watermarksPeriodicBounded(60000)
)
.field("user", Types.LONG)
@@ -1166,7 +1166,7 @@ ClusterBuilder builder = ... // configure Cassandra cluster connection
CassandraAppendTableSink sink = new CassandraAppendTableSink(
builder,
// the query must match the schema of the table
- INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?));
+ "INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)");
tableEnv.registerTableSink(
"cassandraOutputTable",
@@ -1187,7 +1187,7 @@ val builder: ClusterBuilder = ... // configure Cassandra cluster connection
val sink: CassandraAppendTableSink = new CassandraAppendTableSink(
builder,
// the query must match the schema of the table
- INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?))
+ "INSERT INTO flink.myTable (id, name, value) VALUES (?, ?, ?)")
tableEnv.registerTableSink(
"cassandraOutputTable",
diff --git a/docs/dev/table/functions.md b/docs/dev/table/functions.md
index e652c304e93..682f519eb3c 100644
--- a/docs/dev/table/functions.md
+++ b/docs/dev/table/functions.md
@@ -3487,7 +3487,7 @@ DATE_FORMAT(timestamp, string)
{% endhighlight %}
</td>
<td>
- <p>Returns a string that formats <i>timestamp</i> with a specified format <i>string</i>. The format specification is given in the <a href="#date-format-specifiers">Date Format Specifier table</a>.</p>
+ <p><span class="label label-danger">Attention</span> This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use EXTRACT as a workaround.</p>
</td>
</tr>
@@ -3756,8 +3756,7 @@ dateFormat(TIMESTAMP, STRING)
{% endhighlight %}
</td>
<td>
- <p>Returns a string that formats <i>TIMESTAMP</i> with a specified format <i>STRING</i>. The format specification is given in the <a href="#date-format-specifiers">Date Format Specifier table</a>.</p>
- <p>E.g., <code>dateFormat(ts, '%Y, %d %M')</code> results in strings formatted as "2017, 05 May".</p>
+ <p><span class="label label-danger">Attention</span> This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use extract() as a workaround.</p>
</td>
</tr>
@@ -4014,8 +4013,7 @@ dateFormat(TIMESTAMP, STRING)
{% endhighlight %}
</td>
<td>
- <p>Returns a string that formats <i>TIMESTAMP</i> with a specified format <i>STRING</i>. The format specification is given in the <a href="#date-format-specifiers">Date Format Specifier table</a>.</p>
- <p>E.g., <code>dateFormat('ts, "%Y, %d %M")</code> results in strings formatted as "2017, 05 May".</p>
+ <p><span class="label label-danger">Attention</span> This function has serious bugs and should not be used for now. Please implement a custom UDF instead or use extract() as a workaround.</p>
</td>
</tr>
diff --git a/docs/dev/table/sql.md b/docs/dev/table/sql.md
index 55c144a8ed2..1e7ef83fcb7 100644
--- a/docs/dev/table/sql.md
+++ b/docs/dev/table/sql.md
@@ -22,7 +22,7 @@ specific language governing permissions and limitations
under the License.
-->
-SQL queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can seamlessly mixed and are holistically optimized and translated into a single program.
+SQL queries are specified with the `sqlQuery()` method of the `TableEnvironment`. The method returns the result of the SQL query as a `Table`. A `Table` can be used in [subsequent SQL and Table API queries](common.html#mixing-table-api-and-sql), be [converted into a DataSet or DataStream](common.html#integration-with-datastream-and-dataset-api), or [written to a TableSink](common.html#emit-a-table)). SQL and Table API queries can be seamlessly mixed and are holistically optimized and translated into a single program.
In order to access a table in a SQL query, it must be [registered in the TableEnvironment](common.html#register-tables-in-the-catalog). A table can be registered from a [TableSource](common.html#register-a-tablesource), [Table](common.html#register-a-table), [DataStream, or DataSet](common.html#register-a-datastream-or-dataset-as-table). Alternatively, users can also [register external catalogs in a TableEnvironment](common.html#register-an-external-catalog) to specify the location of the data sources.
diff --git a/docs/dev/table/streaming/dynamic_tables.md b/docs/dev/table/streaming/dynamic_tables.md
index 698cd673185..f8bcb9405c6 100644
--- a/docs/dev/table/streaming/dynamic_tables.md
+++ b/docs/dev/table/streaming/dynamic_tables.md
@@ -53,12 +53,12 @@ The following table compares traditional relational algebra and stream processin
</tr>
</table>
-Despite these differences, processing streams with relational queries and SQL is not impossible. Advanced relational database systems offer a feature called *Materialized Views*. A materialized view is defined as a SQL query, just like a regular virtual view. In contrast to a virtual view, a materialized view caches the result of the query such that the query does not need to be evaluated when the view is accessed. A common challenge for caching is to prevent a cache from serving outdated results. A materialized view becomes outdated when the base tables of its definition query are modified. *Eager View Maintenance* is a technique to update materialized views and updates a materialized view as soon as its base tables are updated.
+Despite these differences, processing streams with relational queries and SQL is not impossible. Advanced relational database systems offer a feature called *Materialized Views*. A materialized view is defined as a SQL query, just like a regular virtual view. In contrast to a virtual view, a materialized view caches the result of the query such that the query does not need to be evaluated when the view is accessed. A common challenge for caching is to prevent a cache from serving outdated results. A materialized view becomes outdated when the base tables of its definition query are modified. *Eager View Maintenance* is a technique to update a materialized view as soon as its base tables are updated.
The connection between eager view maintenance and SQL queries on streams becomes obvious if we consider the following:
- A database table is the result of a *stream* of `INSERT`, `UPDATE`, and `DELETE` DML statements, often called *changelog stream*.
-- A materialized view is defined as a SQL query. In order to update the view, the query is continuously processes the changelog streams of the view's base relations.
+- A materialized view is defined as a SQL query. In order to update the view, the query continuously processes the changelog streams of the view's base relations.
- The materialized view is the result of the streaming SQL query.
With these points in mind, we introduce following concept of *Dynamic tables* in the next section.
@@ -66,7 +66,7 @@ With these points in mind, we introduce following concept of *Dynamic tables* in
Dynamic Tables & Continuous Queries
---------------------------------------
-*Dynamic tables* are the core concept of Flink's Table API and SQL support for streaming data. In contrast to the static tables that represent batch data, dynamic table are changing over time. They can be queried like static batch tables. Querying a dynamic table yields a *Continuous Query*. A continuous query never terminates and produces a dynamic table as result. The query continuously updates its (dynamic) result table to reflect the changes on its input (dynamic) table. Essentially, a continuous query on a dynamic table is very similar to the definition query of a materialized view.
+*Dynamic tables* are the core concept of Flink's Table API and SQL support for streaming data. In contrast to the static tables that represent batch data, dynamic tables are changing over time. They can be queried like static batch tables. Querying dynamic tables yields a *Continuous Query*. A continuous query never terminates and produces a dynamic table as result. The query continuously updates its (dynamic) result table to reflect the changes on its (dynamic) input tables. Essentially, a continuous query on a dynamic table is very similar to a query that defines a materialized view.
It is important to note that the result of a continuous query is always semantically equivalent to the result of the same query being executed in batch mode on a snapshot of the input tables.
@@ -177,7 +177,7 @@ When converting a dynamic table into a stream or writing it to an external syste
</center>
<br><br>
-* **Upsert stream:** An upsert stream is a stream with two types of messages, *upsert messages* and *delete message*. A dynamic table that is converted into an upsert stream requires a (possibly composite) unique key. A dynamic table with unique key is converted into a dynamic table by encoding `INSERT` and `UPDATE` changes as upsert message and `DELETE` changes as delete message. The stream consuming operator needs to be aware of the unique key attribute in order to apply messages correctly. The main difference to a retract stream is that `UPDATE` changes are encoded with a single message and hence more efficient. The following figure visualizes the conversion of a dynamic table into an upsert stream.
+* **Upsert stream:** An upsert stream is a stream with two types of messages, *upsert messages* and *delete messages*. A dynamic table that is converted into an upsert stream requires a (possibly composite) unique key. A dynamic table with unique key is converted into a stream by encoding `INSERT` and `UPDATE` changes as upsert messages and `DELETE` changes as delete messages. The stream consuming operator needs to be aware of the unique key attribute in order to apply messages correctly. The main difference to a retract stream is that `UPDATE` changes are encoded with a single message and hence more efficient. The following figure visualizes the conversion of a dynamic table into an upsert stream.
<center>
<img alt="Dynamic tables" src="{{ site.baseurl }}/fig/table-streaming/redo-mode.png" width="85%">
diff --git a/docs/dev/table/streaming/joins.md b/docs/dev/table/streaming/joins.md
index f2934066c07..508e8c70221 100644
--- a/docs/dev/table/streaming/joins.md
+++ b/docs/dev/table/streaming/joins.md
@@ -143,7 +143,7 @@ WHERE r.currency = o.currency
Each record from the probe side will be joined with the version of the build side table at the time of the correlated time attribute of the probe side record.
In order to support updates (overwrites) of previous values on the build side table, the table must define a primary key.
-In our example, each record from `Orders` will be joined with the version of `Rates` at time `o.rowtime`. The `currency` field has been defined as the primary key of `Rates` before and is used to connect both tables in our example. If the query were using a processing-time notion, a newly appended order would always be joined with the most recent version of `Rates` when executing the operation.
+In our example, each record from `Orders` will be joined with the version of `Rates` at time `o.rowtime`. The `currency` field has been defined as the primary key of `Rates` before and is used to connect both tables in our example. If the query were using a processing-time notion, a newly appended order would always be joined with the most recent version of `Rates` when executing the operation.
In contrast to [regular joins](#regular-joins), this means that if there is a new record on the build side, it will not affect the previous results of the join.
This again allows Flink to limit the number of elements that must be kept in the state.
@@ -199,7 +199,7 @@ By definition, it is always the current timestamp. Thus, invocations of a proces
and any updates in the underlying history table will also immediately overwrite the current values.
Only the latest versions (with respect to the defined primary key) of the build side records are kept in the state.
-New updates will have no effect on the previously results emitted/processed records from the probe side.
+Updates of the build side will have no effect on previously emitted join results.
One can think about a processing-time temporal join as a simple `HashMap<K, V>` that stores all of the records from the build side.
When a new record from the build side has the same key as some previous record, the old value is just simply overwritten.
diff --git a/docs/dev/table/streaming/match_recognize.md b/docs/dev/table/streaming/match_recognize.md
index 01799cc47b4..33afc27e75a 100644
--- a/docs/dev/table/streaming/match_recognize.md
+++ b/docs/dev/table/streaming/match_recognize.md
@@ -41,19 +41,19 @@ The following example illustrates the syntax for basic pattern recognition:
{% highlight sql %}
SELECT T.aid, T.bid, T.cid
FROM MyTable
-MATCH_RECOGNIZE (
- PARTITION BY userid
- ORDER BY proctime
- MEASURES
- A.id AS aid,
- B.id AS bid,
- C.id AS cid
- PATTERN (A B C)
- DEFINE
- A AS name = 'a',
- B AS name = 'b',
- C AS name = 'c'
-) AS T
+ MATCH_RECOGNIZE (
+ PARTITION BY userid
+ ORDER BY proctime
+ MEASURES
+ A.id AS aid,
+ B.id AS bid,
+ C.id AS cid
+ PATTERN (A B C)
+ DEFINE
+ A AS name = 'a',
+ B AS name = 'b',
+ C AS name = 'c'
+ ) AS T
{% endhighlight %}
This page will explain each keyword in more detail and will illustrate more complex examples.
@@ -107,7 +107,7 @@ The table has a following schema:
{% highlight text %}
Ticker
- |-- symbol: Long # symbol of the stock
+ |-- symbol: String # symbol of the stock
|-- price: Long # price of the stock
|-- tax: Long # tax liability of the stock
|-- rowtime: TimeIndicatorTypeInfo(rowtime) # point in time when the change to those values happened
@@ -136,22 +136,22 @@ The task is now to find periods of a constantly decreasing price of a single tic
{% highlight sql %}
SELECT *
FROM Ticker
-MATCH_RECOGNIZE (
- PARTITION BY symbol
- ORDER BY rowtime
- MEASURES
- START_ROW.rowtime AS start_tstamp,
- LAST(PRICE_DOWN.rowtime) AS bottom_tstamp,
- LAST(PRICE_UP.rowtime) AS end_tstamp
- ONE ROW PER MATCH
- AFTER MATCH SKIP TO LAST PRICE_UP
- PATTERN (START_ROW PRICE_DOWN+ PRICE_UP)
- DEFINE
- PRICE_DOWN AS
- (LAST(PRICE_DOWN.price, 1) IS NULL AND PRICE_DOWN.price < START_ROW.price) OR
- PRICE_DOWN.price < LAST(PRICE_DOWN.price, 1),
- PRICE_UP AS
- PRICE_UP.price > LAST(PRICE_DOWN.price, 1)
+ MATCH_RECOGNIZE (
+ PARTITION BY symbol
+ ORDER BY rowtime
+ MEASURES
+ START_ROW.rowtime AS start_tstamp,
+ LAST(PRICE_DOWN.rowtime) AS bottom_tstamp,
+ LAST(PRICE_UP.rowtime) AS end_tstamp
+ ONE ROW PER MATCH
+ AFTER MATCH SKIP TO LAST PRICE_UP
+ PATTERN (START_ROW PRICE_DOWN+ PRICE_UP)
+ DEFINE
+ PRICE_DOWN AS
+ (LAST(PRICE_DOWN.price, 1) IS NULL AND PRICE_DOWN.price < START_ROW.price) OR
+ PRICE_DOWN.price < LAST(PRICE_DOWN.price, 1),
+ PRICE_UP AS
+ PRICE_UP.price > LAST(PRICE_DOWN.price, 1)
) MR;
{% endhighlight %}
@@ -211,6 +211,65 @@ If a condition is not defined for a pattern variable, a default condition will b
For a more detailed explanation about expressions that can be used in those clauses, please have a look at the [event stream navigation](#pattern-navigation) section.
+### Aggregations
+
+Aggregations can be used in `DEFINE` and `MEASURES` clauses. Both [built-in]({{ site.baseurl }}/dev/table/sql.html#built-in-functions) and custom [user defined]({{ site.baseurl }}/dev/table/udfs.html) functions are supported.
+
+Aggregate functions are applied to each subset of rows mapped to a match. In order to understand how those subsets are evaluated have a look at the [event stream navigation](#pattern-navigation) section.
+
+The task of the following example is to find the longest period of time for which the average price of a ticker did not go below certain threshold. It shows how expressible `MATCH_RECOGNIZE` can become with aggregations.
+This task can be performed with the following query:
+
+{% highlight sql %}
+SELECT *
+FROM Ticker
+ MATCH_RECOGNIZE (
+ PARTITION BY symbol
+ ORDER BY rowtime
+ MEASURES
+ FIRST(A.rowtime) AS start_tstamp,
+ LAST(A.rowtime) AS end_tstamp,
+ AVG(A.price) AS avgPrice
+ ONE ROW PER MATCH
+ AFTER MATCH SKIP TO FIRST B
+ PATTERN (A+ B)
+ DEFINE
+ A AS AVG(A.price) < 15
+ ) MR;
+{% endhighlight %}
+
+Given this query and following input values:
+
+{% highlight text %}
+symbol rowtime price tax
+====== ==================== ======= =======
+'ACME' '01-Apr-11 10:00:00' 12 1
+'ACME' '01-Apr-11 10:00:01' 17 2
+'ACME' '01-Apr-11 10:00:02' 13 1
+'ACME' '01-Apr-11 10:00:03' 16 3
+'ACME' '01-Apr-11 10:00:04' 25 2
+'ACME' '01-Apr-11 10:00:05' 2 1
+'ACME' '01-Apr-11 10:00:06' 4 1
+'ACME' '01-Apr-11 10:00:07' 10 2
+'ACME' '01-Apr-11 10:00:08' 15 2
+'ACME' '01-Apr-11 10:00:09' 25 2
+'ACME' '01-Apr-11 10:00:10' 30 1
+{% endhighlight %}
+
+The query will accumulate events as part of the pattern variable `A` as long as the average price of them does not exceed `15`. For example, such a limit exceeding happens at `01-Apr-11 10:00:04`.
+The following period exceeds the average price of `15` again at `01-Apr-11 10:00:10`. Thus the results for said query will be:
+
+{% highlight text %}
+ symbol start_tstamp end_tstamp avgPrice
+========= ================== ================== ============
+ACME 01-APR-11 10:00:00 01-APR-11 10:00:03 14.5
+ACME 01-APR-11 10:00:04 01-APR-11 10:00:09 13.5
+{% endhighlight %}
+
+<span class="label label-info">Note</span> Aggregations can be applied to expressions, but only if they reference a single pattern variable. Thus `SUM(A.price * A.tax)` is a valid one, but `AVG(A.price * B.tax)` is not.
+
+<span class="label label-danger">Attention</span> `DISTINCT` aggregations are not supported.
+
Defining a Pattern
------------------
@@ -256,13 +315,13 @@ FROM Ticker
ORDER BY rowtime
MEASURES
C.price AS lastPrice
- PATTERN (A B* C)
ONE ROW PER MATCH
AFTER MATCH SKIP PAST LAST ROW
+ PATTERN (A B* C)
DEFINE
A AS A.price > 10,
B AS B.price < 15,
- C AS B.price > 12
+ C AS C.price > 12
)
{% endhighlight %}
@@ -293,6 +352,7 @@ The same query where `B*` is modified to `B*?`, which means that `B*` should be
symbol lastPrice
======== ===========
XYZ 13
+ XYZ 16
{% endhighlight %}
The pattern variable `B` matches only to the row with price `12` instead of swallowing the rows with prices `12`, `13`, and `14`.
@@ -335,9 +395,9 @@ FROM Ticker
MEASURES
C.rowtime AS dropTime,
A.price - C.price AS dropDiff
- PATTERN (A B* C) WITHIN INTERVAL '1' HOUR
ONE ROW PER MATCH
AFTER MATCH SKIP PAST LAST ROW
+ PATTERN (A B* C) WITHIN INTERVAL '1' HOUR
DEFINE
B AS B.price > A.price - 10
C AS C.price < A.price - 10
@@ -399,8 +459,8 @@ FROM Ticker
FIRST(A.price) AS startPrice,
LAST(A.price) AS topPrice,
B.price AS lastPrice
- PATTERN (A+ B)
ONE ROW PER MATCH
+ PATTERN (A+ B)
DEFINE
A AS LAST(A.price, 1) IS NULL OR A.price > LAST(A.price, 1),
B AS B.price < LAST(A.price)
@@ -547,8 +607,6 @@ The table consists of the following columns:
As can be seen in the table, the first row is mapped to pattern variable `A` and subsequent rows are mapped to pattern variable `B`. However, the last row does not fulfill the `B` condition because the sum over all mapped rows `SUM(price)` and the sum over all rows in `B` exceed the specified thresholds.
-<span class="label label-danger">Attention</span> Please note that aggregations such as `SUM` are not supported yet. They are only used for explanation here.
-
### Logical Offsets
_Logical offsets_ enable navigation within the events that were mapped to a particular pattern variable. This can be expressed
@@ -769,9 +827,9 @@ FROM Ticker
SUM(A.price) AS sumPrice,
FIRST(rowtime) AS startTime,
LAST(rowtime) AS endTime
- PATTERN (A+ C)
ONE ROW PER MATCH
[AFTER MATCH STRATEGY]
+ PATTERN (A+ C)
DEFINE
A AS SUM(A.price) < 30
)
@@ -779,8 +837,6 @@ FROM Ticker
The query returns the sum of the prices of all rows mapped to `A` and the first and last timestamp of the overall match.
-<span class="label label-danger">Attention</span> Please note that aggregations such as `SUM` are not supported yet. They are only used for explanation here.
-
The query will produce different results based on which `AFTER MATCH` strategy was used:
##### `AFTER MATCH SKIP PAST LAST ROW`
@@ -842,7 +898,7 @@ The last result matched against the rows #5, #6.
This combination will produce a runtime exception because one would always try to start a new match where the
last one started. This would produce an infinite loop and, thus, is prohibited.
-One has to keep in mind that in case of the `SKIP TO FIRST/LAST variable`strategy it might be possible that there are no rows mapped to that
+One has to keep in mind that in case of the `SKIP TO FIRST/LAST variable` strategy it might be possible that there are no rows mapped to that
variable (e.g. for pattern `A*`). In such cases, a runtime exception will be thrown as the standard requires a valid row to continue the
matching.
@@ -901,5 +957,6 @@ Unsupported features include:
* `SUBSET` - which allows creating logical groups of pattern variables and using those groups in the `DEFINE` and `MEASURES` clauses.
* Physical offsets - `PREV/NEXT`, which indexes all events seen rather than only those that were mapped to a pattern variable(as in [logical offsets](#logical-offsets) case).
* Extracting time attributes - there is currently no possibility to get a time attribute for subsequent time-based operations.
-* Aggregates - one cannot use aggregates in `MEASURES` nor `DEFINE` clauses.
* `MATCH_RECOGNIZE` is supported only for SQL. There is no equivalent in the Table API.
+* Aggregations:
+ * distinct aggregations are not supported.
diff --git a/docs/dev/table/streaming/temporal_tables.md b/docs/dev/table/streaming/temporal_tables.md
index b45052790b5..4ebb4a6e8a7 100644
--- a/docs/dev/table/streaming/temporal_tables.md
+++ b/docs/dev/table/streaming/temporal_tables.md
@@ -114,7 +114,7 @@ Each query to `Rates(timeAttribute)` would return the state of the `Rates` for t
**Note**: Currently, Flink doesn't support directly querying the temporal table functions with a constant time attribute parameter. At the moment, temporal table functions can only be used in joins.
The example above was used to provide an intuition about what the function `Rates(timeAttribute)` returns.
-See also the [joining page for continuous queries](joins.html) for more information about how to join with a temporal table.
+See also the page about [joins for continuous queries](joins.html) for more information about how to join with a temporal table.
### Defining Temporal Table Function
@@ -171,7 +171,6 @@ val ratesHistory = env
.fromCollection(ratesHistoryData)
.toTable(tEnv, 'r_currency, 'r_rate, 'r_proctime.proctime)
-tEnv.registerTable("Orders", orders)
tEnv.registerTable("RatesHistory", ratesHistory)
// Create and register TemporalTableFunction.
diff --git a/docs/dev/table/streaming/time_attributes.md b/docs/dev/table/streaming/time_attributes.md
index 01658132fb6..101bad68b80 100644
--- a/docs/dev/table/streaming/time_attributes.md
+++ b/docs/dev/table/streaming/time_attributes.md
@@ -30,7 +30,7 @@ Flink is able to process streaming data based on different notions of *time*.
For more information about time handling in Flink, see the introduction about [Event Time and Watermarks]({{ site.baseurl }}/dev/event_time.html).
-This pages explains how time attributes can be defined for time-based operations in Flink's Table API & SQL.
+This page explains how time attributes can be defined for time-based operations in Flink's Table API & SQL.
* This will be replaced by the TOC
{:toc}
@@ -40,7 +40,7 @@ Introduction to Time Attributes
Time-based operations such as windows in both the [Table API]({{ site.baseurl }}/dev/table/tableApi.html#group-windows) and [SQL]({{ site.baseurl }}/dev/table/sql.html#group-windows) require information about the notion of time and its origin. Therefore, tables can offer *logical time attributes* for indicating time and accessing corresponding timestamps in table programs.
-Time attributes can be part of every table schema. They are defined when creating a table from a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can used in time-based operations.
+Time attributes can be part of every table schema. They are defined when creating a table from a `DataStream` or are pre-defined when using a `TableSource`. Once a time attribute has been defined at the beginning, it can be referenced as a field and can be used in time-based operations.
As long as a time attribute is not modified and is simply forwarded from one part of the query to another, it remains a valid time attribute. Time attributes behave like regular timestamps and can be accessed for calculations. If a time attribute is used in a calculation, it will be materialized and becomes a regular timestamp. Regular timestamps do not cooperate with Flink's time and watermarking system and thus can not be used for time-based operations anymore.
diff --git a/docs/dev/table/tableApi.md b/docs/dev/table/tableApi.md
index 80930ca86ad..e44df24ad5e 100644
--- a/docs/dev/table/tableApi.md
+++ b/docs/dev/table/tableApi.md
@@ -59,7 +59,7 @@ Table counts = orders
.select("a, b.count as cnt");
// conversion to DataSet
-DataSet<Row> result = tableEnv.toDataSet(counts, Row.class);
+DataSet<Row> result = tEnv.toDataSet(counts, Row.class);
result.print();
{% endhighlight %}
@@ -1728,7 +1728,7 @@ This is the EBNF grammar for expressions:
expressionList = expression , { "," , expression } ;
-expression = timeIndicator | overConstant | alias ;
+expression = overConstant | alias ;
alias = logic | ( logic , "as" , fieldReference ) | ( logic , "as" , "(" , fieldReference , { "," , fieldReference } , ")" ) ;
@@ -1744,7 +1744,7 @@ unary = [ "!" | "-" | "+" ] , composite ;
composite = over | suffixed | nullLiteral | prefixed | atom ;
-suffixed = interval | suffixAs | suffixCast | suffixIf | suffixDistinct | suffixFunctionCall ;
+suffixed = interval | suffixAs | suffixCast | suffixIf | suffixDistinct | suffixFunctionCall | timeIndicator ;
prefixed = prefixAs | prefixCast | prefixIf | prefixDistinct | prefixFunctionCall ;
diff --git a/docs/dev/table/udfs.md b/docs/dev/table/udfs.md
index 20bf49d5999..f2f6b7bcedd 100644
--- a/docs/dev/table/udfs.md
+++ b/docs/dev/table/udfs.md
@@ -72,7 +72,7 @@ tableEnv.registerFunction("hashCode", new HashCode(10));
myTable.select("string, string.hashCode(), hashCode(string)");
// use the function in SQL API
-tableEnv.sqlQuery("SELECT string, HASHCODE(string) FROM MyTable");
+tableEnv.sqlQuery("SELECT string, hashCode(string) FROM MyTable");
{% endhighlight %}
</div>
@@ -93,7 +93,7 @@ myTable.select('string, hashCode('string))
// register and use the function in SQL
tableEnv.registerFunction("hashCode", new HashCode(10))
-tableEnv.sqlQuery("SELECT string, HASHCODE(string) FROM MyTable")
+tableEnv.sqlQuery("SELECT string, hashCode(string) FROM MyTable")
{% endhighlight %}
</div>
</div>
@@ -110,8 +110,8 @@ public static class TimestampModifier extends ScalarFunction {
return t % 1000;
}
- public TypeInformation<?> getResultType(signature: Class<?>[]) {
- return Types.TIMESTAMP;
+ public TypeInformation<?> getResultType(Class<?>[] signature) {
+ return Types.SQL_TIMESTAMP;
}
}
{% endhighlight %}
@@ -230,7 +230,7 @@ public class CustomTypeSplit extends TableFunction<Row> {
for (String s : str.split(" ")) {
Row row = new Row(2);
row.setField(0, s);
- row.setField(1, s.length);
+ row.setField(1, s.length());
collect(row);
}
}
diff --git a/docs/ops/deployment/oss.md b/docs/ops/deployment/oss.md
new file mode 100644
index 00000000000..543d5a2f176
--- /dev/null
+++ b/docs/ops/deployment/oss.md
@@ -0,0 +1,233 @@
+---
+title: "Aliyun Open Storage Service (OSS)"
+nav-title: Aliyun OSS
+nav-parent_id: deployment
+nav-pos: 9
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+* ToC
+{:toc}
+
+
+## OSS: Open Storage Service
+
+[Aliyun Open Storage Service](https://www.aliyun.com/product/oss) (Aliyun OSS) is widely used especially among China’s cloud users, and it provides cloud object storage for a variety of use cases.
+
+[Hadoop file system](http://hadoop.apache.org/docs/current/hadoop-aliyun/tools/hadoop-aliyun/index.html) supports OSS since version 2.9.1. Now, you can also use OSS with Fink for **reading** and **writing data**.
+
+You can access OSS objects like this:
+
+{% highlight plain %}
+oss://<your-bucket>/<object-name>
+{% endhighlight %}
+
+Below shows how to use OSS with Flink:
+
+{% highlight java %}
+// Read from OSS bucket
+env.readTextFile("oss://<your-bucket>/<object-name>");
+
+// Write to OSS bucket
+dataSet.writeAsText("oss://<your-bucket>/<object-name>")
+
+{% endhighlight %}
+
+There are two ways to use OSS with Flink, our shaded `flink-oss-fs-hadoop` will cover most scenarios. However, you may need to set up a specific Hadoop OSS FileSystem implementation if you want use OSS as YARN's resource storage dir ([This patch](https://issues.apache.org/jira/browse/HADOOP-15919) enables YARN to use OSS). Both ways are described below.
+
+### Shaded Hadoop OSS file system (recommended)
+
+In order to use `flink-oss-fs-hadoop`, copy the respective JAR file from the opt directory to the lib directory of your Flink distribution before starting Flink, e.g.
+
+{% highlight bash %}
+cp ./opt/flink-oss-fs-hadoop-{{ site.version }}.jar ./lib/
+{% endhighlight %}
+
+`flink-oss-fs-hadoop` registers default FileSystem wrappers for URIs with the oss:// scheme.
+
+#### Configurations setup
+After setting up the OSS FileSystem wrapper, you need to add some configurations to make sure that Flink is allowed to access your OSS buckets.
+
+In order to use OSS with Flink more easily, you can use the same configuration keys in `flink-conf.yaml` as in Hadoop's `core-site.xml`
+
+You can see the configuration keys in the [Hadoop OSS documentation](http://hadoop.apache.org/docs/current/hadoop-aliyun/tools/hadoop-aliyun/index.html).
+
+There are some required configurations that must be added to `flink-conf.yaml` (**Other configurations defined in Hadoop OSS documentation are advanced configurations which used by performance tuning**):
+
+{% highlight yaml %}
+fs.oss.endpoint: Aliyun OSS endpoint to connect to
+fs.oss.accessKeyId: Aliyun access key ID
+fs.oss.accessKeySecret: Aliyun access key secret
+{% endhighlight %}
+
+### Hadoop-provided OSS file system - manual setup
+This setup is a bit more complex and we recommend using our shaded Hadoop file systems instead (see above) unless required otherwise, e.g. for using OSS as YARN’s resource storage dir via the fs.defaultFS configuration property in Hadoop’s core-site.xml.
+
+#### Set OSS FileSystem
+You need to point Flink to a valid Hadoop configuration, which contains the following properties in core-site.xml:
+
+{% highlight xml %}
+<configuration>
+
+<property>
+ <name>fs.oss.impl</name>
+ <value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
+ </property>
+
+ <property>
+ <name>fs.oss.endpoint</name>
+ <value>Aliyun OSS endpoint to connect to</value>
+ <description>Aliyun OSS endpoint to connect to. An up-to-date list is provided in the Aliyun OSS Documentation.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.accessKeyId</name>
+ <description>Aliyun access key ID</description>
+ </property>
+
+ <property>
+ <name>fs.oss.accessKeySecret</name>
+ <description>Aliyun access key secret</description>
+ </property>
+
+ <property>
+ <name>fs.oss.buffer.dir</name>
+ <value>/tmp/oss</value>
+ </property>
+
+</property>
+
+</configuration>
+{% endhighlight %}
+
+#### Hadoop Configuration
+
+You can specify the [Hadoop configuration](../config.html#hdfs) in various ways pointing Flink to
+the path of the Hadoop configuration directory, for example
+- by setting the environment variable `HADOOP_CONF_DIR`, or
+- by setting the `fs.hdfs.hadoopconf` configuration option in `flink-conf.yaml`:
+{% highlight yaml %}
+fs.hdfs.hadoopconf: /path/to/etc/hadoop
+{% endhighlight %}
+
+This registers `/path/to/etc/hadoop` as Hadoop's configuration directory with Flink. Flink will look for the `core-site.xml` and `hdfs-site.xml` files in the specified directory.
+
+#### Provide OSS FileSystem Dependency
+
+You can find Hadoop OSS FileSystem are packaged in the hadoop-aliyun artifact. This JAR and all its dependencies need to be added to Flink’s classpath, i.e. the class path of both Job and TaskManagers.
+
+There are multiple ways of adding JARs to Flink’s class path, the easiest being simply to drop the JARs in Flink’s lib folder. You need to copy the hadoop-aliyun JAR with all its dependencies (You can find these as part of the Hadoop binaries in hadoop-3/share/hadoop/tools/lib). You can also export the directory containing these JARs as part of the HADOOP_CLASSPATH environment variable on all machines.
+
+## An Example
+Below is an example shows the result of our setup (data is generated by TPC-DS tool)
+
+{% highlight java %}
+// Read from OSS bucket
+scala> val dataSet = benv.readTextFile("oss://<your-bucket>/50/call_center/data-m-00049")
+dataSet: org.apache.flink.api.scala.DataSet[String] = org.apache.flink.api.scala.DataSet@31940704
+
+scala> dataSet.print()
+1|AAAAAAAABAAAAAAA|1998-01-01|||2450952|NY Metro|large|2935|1670015|8AM-4PM|Bob Belcher|6|More than other authori|Shared others could not count fully dollars. New members ca|Julius Tran|3|pri|6|cally|730|Ash Hill|Boulevard|Suite 0|Oak Grove|Williamson County|TN|38370|United States|-5|0.11|
+2|AAAAAAAACAAAAAAA|1998-01-01|2000-12-31||2450806|Mid Atlantic|medium|1574|594972|8AM-8AM|Felipe Perkins|2|A bit narrow forms matter animals. Consist|Largely blank years put substantially deaf, new others. Question|Julius Durham|5|anti|1|ought|984|Center Hill|Way|Suite 70|Midway|Williamson County|TN|31904|United States|-5|0.12|
+3|AAAAAAAACAAAAAAA|2001-01-01|||2450806|Mid Atlantic|medium|1574|1084486|8AM-4PM|Mark Hightower|2|Wrong troops shall work sometimes in a opti|Largely blank years put substantially deaf, new others. Question|Julius Durham|1|ought|2|able|984|Center Hill|Way|Suite 70|Midway|Williamson County|TN|31904|United States|-5|0.01|
+4|AAAAAAAAEAAAAAAA|1998-01-01|2000-01-01||2451063|North Midwest|medium|10137|6578913|8AM-4PM|Larry Mccray|2|Dealers make most historical, direct students|Rich groups catch longer other fears; future,|Matthew Clifton|4|ese|3|pri|463|Pine Ridge|RD|Suite U|Five Points|Ziebach County|SD|56098|United States|-6|0.05|
+5|AAAAAAAAEAAAAAAA|2000-01-02|2001-12-31||2451063|North Midwest|small|17398|4610470|8AM-8AM|Larry Mccray|2|Dealers make most historical, direct students|Blue, due beds come. Politicians would not make far thoughts. Specifically new horses partic|Gary Colburn|4|ese|3|pri|463|Pine Ridge|RD|Suite U|Five Points|Ziebach County|SD|56098|United States|-6|0.12|
+6|AAAAAAAAEAAAAAAA|2002-01-01|||2451063|North Midwest|medium|13118|6585236|8AM-4PM|Larry Mccray|5|Silly particles could pro|Blue, due beds come. Politicians would not make far thoughts. Specifically new horses partic|Gary Colburn|5|anti|3|pri|463|Pine Ridge|RD|Suite U|Five Points|Ziebach County|SD|56098|United States|-6|0.11|
+7|AAAAAAAAHAAAAAAA|1998-01-01|||2451024|Pacific Northwest|small|6280|1739560|8AM-4PM|Alden Snyder|6|Major, formal states can suppor|Reduced, subsequent bases could not lik|Frederick Weaver|5|anti|4|ese|415|Jefferson Tenth|Court|Suite 180|Riverside|Walker County|AL|39231|United States|-6|0.00|
+8|AAAAAAAAIAAAAAAA|1998-01-01|2000-12-31||2450808|California|small|4766|2459256|8AM-12AM|Wayne Ray|6|Here possible notions arrive only. Ar|Common, free creditors should exper|Daniel Weller|5|anti|2|able|550|Cedar Elm|Ct.|Suite I|Fairview|Williamson County|TN|35709|United States|-5|0.06|
+
+scala> dataSet.count()
+res0: Long = 8
+
+// Write to OSS bucket
+scala> dataSet.writeAsText("oss://<your-bucket>/50/call_center/data-m-00049.1")
+
+scala> benv.execute("My batch program")
+res1: org.apache.flink.api.common.JobExecutionResult = org.apache.flink.api.common.JobExecutionResult@77476fcf
+
+scala> val newDataSet = benv.readTextFile("oss://<your-bucket>/50/call_center/data-m-00049.1")
+newDataSet: org.apache.flink.api.scala.DataSet[String] = org.apache.flink.api.scala.DataSet@40b70f31
+
+scala> newDataSet.count()
+res2: Long = 8
+
+{% endhighlight %}
+
+## Common Issues
+### Could not find OSS file system
+If your job submission fails with an Exception message like below, please check if our shaded jar (flink-oss-fs-hadoop-{{ site.version }}.jar) is in the lib dir.
+
+{% highlight plain %}
+Caused by: org.apache.flink.runtime.client.JobExecutionException: Could not set up JobManager
+ at org.apache.flink.runtime.jobmaster.JobManagerRunner.<init>(JobManagerRunner.java:176)
+ at org.apache.flink.runtime.dispatcher.Dispatcher$DefaultJobManagerRunnerFactory.createJobManagerRunner(Dispatcher.java:1058)
+ at org.apache.flink.runtime.dispatcher.Dispatcher.lambda$createJobManagerRunner$5(Dispatcher.java:308)
+ at org.apache.flink.util.function.CheckedSupplier.lambda$unchecked$0(CheckedSupplier.java:34)
+ ... 7 more
+Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: Could not find a file system implementation for scheme 'oss'. The scheme is not directly supported by Flink and no Hadoop file system to support this scheme could be loaded.
+ at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:273)
+ at org.apache.flink.runtime.executiongraph.ExecutionGraph.attachJobGraph(ExecutionGraph.java:827)
+ at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:232)
+ at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:100)
+ at org.apache.flink.runtime.jobmaster.JobMaster.createExecutionGraph(JobMaster.java:1151)
+ at org.apache.flink.runtime.jobmaster.JobMaster.createAndRestoreExecutionGraph(JobMaster.java:1131)
+ at org.apache.flink.runtime.jobmaster.JobMaster.<init>(JobMaster.java:294)
+ at org.apache.flink.runtime.jobmaster.JobManagerRunner.<init>(JobManagerRunner.java:157)
+ ... 10 more
+Caused by: org.apache.flink.core.fs.UnsupportedFileSystemSchemeException: Could not find a file system implementation for scheme 'oss'. The scheme is not directly supported by Flink and no Hadoop file system to support this scheme could be loaded.
+ at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:403)
+ at org.apache.flink.core.fs.FileSystem.get(FileSystem.java:318)
+ at org.apache.flink.core.fs.Path.getFileSystem(Path.java:298)
+ at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:587)
+ at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:62)
+ at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:259)
+ ... 17 more
+Caused by: org.apache.flink.core.fs.UnsupportedFileSystemSchemeException: Hadoop is not in the classpath/dependencies.
+ at org.apache.flink.core.fs.UnsupportedSchemeFactory.create(UnsupportedSchemeFactory.java:64)
+ at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:399)
+ ... 22 more
+{% endhighlight %}
+
+### Missing configuration(s)
+If your job submission fails with an Exception message like below, please check if the corresponding configurations exits in `flink-conf.yaml`
+
+{% highlight plain %}
+Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: Aliyun OSS endpoint should not be null or empty. Please set proper endpoint with 'fs.oss.endpoint'.
+ at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:273)
+ at org.apache.flink.runtime.executiongraph.ExecutionGraph.attachJobGraph(ExecutionGraph.java:827)
+ at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:232)
+ at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:100)
+ at org.apache.flink.runtime.jobmaster.JobMaster.createExecutionGraph(JobMaster.java:1151)
+ at org.apache.flink.runtime.jobmaster.JobMaster.createAndRestoreExecutionGraph(JobMaster.java:1131)
+ at org.apache.flink.runtime.jobmaster.JobMaster.<init>(JobMaster.java:294)
+ at org.apache.flink.runtime.jobmaster.JobManagerRunner.<init>(JobManagerRunner.java:157)
+ ... 10 more
+Caused by: java.lang.IllegalArgumentException: Aliyun OSS endpoint should not be null or empty. Please set proper endpoint with 'fs.oss.endpoint'.
+ at org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:145)
+ at org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323)
+ at org.apache.flink.fs.osshadoop.OSSFileSystemFactory.create(OSSFileSystemFactory.java:87)
+ at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:395)
+ at org.apache.flink.core.fs.FileSystem.get(FileSystem.java:318)
+ at org.apache.flink.core.fs.Path.getFileSystem(Path.java:298)
+ at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:587)
+ at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:62)
+ at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:259)
+ ... 17 more
+{% endhighlight %}
diff --git a/docs/ops/deployment/yarn_setup.md b/docs/ops/deployment/yarn_setup.md
index a3342d154db..3d13e2db9b3 100644
--- a/docs/ops/deployment/yarn_setup.md
+++ b/docs/ops/deployment/yarn_setup.md
@@ -324,9 +324,9 @@ This section briefly describes how Flink and YARN interact.
<img src="{{ site.baseurl }}/fig/FlinkOnYarn.svg" class="img-responsive">
-The YARN client needs to access the Hadoop configuration to connect to the YARN resource manager and to HDFS. It determines the Hadoop configuration using the following strategy:
+The YARN client needs to access the Hadoop configuration to connect to the YARN resource manager and HDFS. It determines the Hadoop configuration using the following strategy:
-* Test if `YARN_CONF_DIR`, `HADOOP_CONF_DIR` or `HADOOP_CONF_PATH` are set (in that order). If one of these variables are set, they are used to read the configuration.
+* Test if `YARN_CONF_DIR`, `HADOOP_CONF_DIR` or `HADOOP_CONF_PATH` are set (in that order). If one of these variables is set, it is used to read the configuration.
* If the above strategy fails (this should not be the case in a correct YARN setup), the client is using the `HADOOP_HOME` environment variable. If it is set, the client tries to access `$HADOOP_HOME/etc/hadoop` (Hadoop 2) and `$HADOOP_HOME/conf` (Hadoop 1).
When starting a new Flink YARN session, the client first checks if the requested resources (containers and memory) are available. After that, it uploads a jar that contains Flink and the configuration to HDFS (step 1).
diff --git a/docs/ops/state/large_state_tuning.md b/docs/ops/state/large_state_tuning.md
index 62b3ee557f6..50c5a533cae 100644
--- a/docs/ops/state/large_state_tuning.md
+++ b/docs/ops/state/large_state_tuning.md
@@ -324,7 +324,9 @@ and occupy local disk space. In the future, we might also offer an implementatio
and occupy local disk space. For *incremental snapshots*, the local state is based on RocksDB's native checkpointing mechanism. This mechanism is also used as the first step
to create the primary copy, which means that in this case no additional cost is introduced for creating the secondary copy. We simply keep the native checkpoint directory around
instead of deleting it after uploading to the distributed store. This local copy can share active files with the working directory of RocksDB (via hard links), so for active
-files also no additional disk space is consumed for task-local recovery with incremental snapshots.
+files also no additional disk space is consumed for task-local recovery with incremental snapshots. Using hard links also means that the RocksDB directories must be on
+the same physical device as all the configure local recovery directories that can be used to store local state, or else establishing hard links can fail (see FLINK-10954).
+Currently, this also prevents using local recovery when RocksDB directories are configured to be located on more than one physical device.
### Allocation-preserving scheduling
diff --git a/docs/ops/upgrading.md b/docs/ops/upgrading.md
index 0451940723e..22cea4c53f0 100644
--- a/docs/ops/upgrading.md
+++ b/docs/ops/upgrading.md
@@ -274,7 +274,10 @@ Savepoints are compatible across Flink versions as indicated by the table below:
<td class="text-center">O</td>
<td class="text-center">O</td>
<td class="text-center">O</td>
- <td class="text-left"></td>
+ <td class="text-left">There is a known issue with resuming broadcast state created with 1.5.x in versions
+ 1.6.x up to 1.6.2, and 1.7.0: <a href="https://issues.apache.org/jira/browse/FLINK-11087">FLINK-11087</a>. Users
+ upgrading to 1.6.x or 1.7.x series need to directly migrate to minor versions higher than 1.6.2 and 1.7.0,
+ respectively.</td>
</tr>
<tr>
<td class="text-center"><strong>1.6.x</strong></td>
diff --git a/flink-connectors/flink-connector-elasticsearch6/pom.xml b/flink-connectors/flink-connector-elasticsearch6/pom.xml
index 40b9ab6efaf..4c79e898bc9 100644
--- a/flink-connectors/flink-connector-elasticsearch6/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch6/pom.xml
@@ -174,95 +174,6 @@ under the License.
</dependencies>
- <profiles>
- <!-- Create SQL Client uber jars by default -->
- <profile>
- <id>sql-jars</id>
- <activation>
- <property>
- <name>!skipSqlJars</name>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>sql-jar</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <!-- It is difficult to find out artifacts that are really required by ES. -->
- <!-- We use hard filters for now to clean up the SQL JAR. -->
- <excludes>
- <exclude>com/carrotsearch/**</exclude>
- <exclude>com/sun/**</exclude>
- <exclude>com/tdunning/**</exclude>
- <exclude>config/**</exclude>
- <exclude>forbidden/**</exclude>
- <exclude>joptsimple/**</exclude>
- <exclude>META-INF/services/com.fasterxml.**</exclude>
- <exclude>META-INF/services/org.apache.lucene.**</exclude>
- <exclude>META-INF/services/org.elasticsearch.**</exclude>
- <exclude>META-INF/versions/**</exclude>
- <exclude>modules.txt</exclude>
- <exclude>mozilla/**</exclude>
- <exclude>org/HdrHistogram/**</exclude>
- <exclude>org/joda/**</exclude>
- <exclude>org/tartarus/**</exclude>
- <exclude>org/yaml/**</exclude>
- <exclude>plugins.txt</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <!-- Force relocation of all Elasticsearch dependencies. -->
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.commons</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.http</pattern>
- <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.http</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.lucene</pattern>
- <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.lucene</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.elasticsearch</pattern>
- <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.elasticsearch</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.logging</pattern>
- <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.logging</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.flink.elasticsearch6.shaded.com.fasterxml.jackson</shadedPattern>
- </relocation>
- </relocations>
- <!-- Relocate the table format factory service file. -->
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
<build>
<plugins>
<!--
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
index 74770d04549..decb5cfebe3 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
@@ -98,7 +98,8 @@
MigrationVersion.v1_3,
MigrationVersion.v1_4,
MigrationVersion.v1_5,
- MigrationVersion.v1_6);
+ MigrationVersion.v1_6,
+ MigrationVersion.v1_7);
}
public FlinkKafkaConsumerBaseMigrationTest(MigrationVersion testMigrateVersion) {
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.7-empty-state-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.7-empty-state-snapshot
new file mode 100644
index 00000000000..3b3985224a0
Binary files /dev/null and b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.7-empty-state-snapshot differ
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.7-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.7-snapshot
new file mode 100644
index 00000000000..da23cec6306
Binary files /dev/null and b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.7-snapshot differ
diff --git a/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml b/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml
new file mode 100644
index 00000000000..85b73a9c837
--- /dev/null
+++ b/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml
@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connectors</artifactId>
+ <version>1.8-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-sql-connector-elasticsearch6_${scala.binary.version}</artifactId>
+ <name>flink-sql-connector-elasticsearch6</name>
+
+ <packaging>jar</packaging>
+
+ <!-- Allow users to pass custom connector versions -->
+ <properties>
+ <elasticsearch.version>6.3.1</elasticsearch.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-elasticsearch6_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ <excludes>
+ <!-- These dependencies are not required. -->
+ <exclude>com.carrotsearch:hppc</exclude>
+ <exclude>com.tdunning:t-digest</exclude>
+ <exclude>joda-time:joda-time</exclude>
+ <exclude>net.sf.jopt-simple:jopt-simple</exclude>
+ <exclude>org.elasticsearch:jna</exclude>
+ <exclude>org.hdrhistogram:HdrHistogram</exclude>
+ <exclude>org.yaml:snakeyaml</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <!-- Unless otherwise noticed these filters only serve to reduce the size of the resulting
+ jar by removing unnecessary files -->
+ <filter>
+ <artifact>org.elasticsearch:elasticsearch</artifact>
+ <excludes>
+ <exclude>config/**</exclude>
+ <exclude>modules.txt</exclude>
+ <exclude>plugins.txt</exclude>
+ <exclude>org/joda/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.elasticsearch.client:elasticsearch-rest-high-level-client</artifact>
+ <excludes>
+ <exclude>forbidden/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.apache.httpcomponents:httpclient</artifact>
+ <excludes>
+ <exclude>mozilla/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.apache.lucene:lucene-analyzers-common</artifact>
+ <excludes>
+ <exclude>org/tartarus/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <!-- exclude Java 9 specific classes as otherwise the shade-plugin crashes -->
+ <exclude>META-INF/versions/**</exclude>
+ <exclude>META-INF/services/com.fasterxml.**</exclude>
+ <exclude>META-INF/services/org.apache.lucene.**</exclude>
+ <exclude>META-INF/services/org.elasticsearch.**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <!-- Force relocation of all Elasticsearch dependencies. -->
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.commons</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.http</pattern>
+ <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.http</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.lucene</pattern>
+ <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.lucene</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.elasticsearch</pattern>
+ <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.elasticsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.logging</pattern>
+ <shadedPattern>org.apache.flink.elasticsearch6.shaded.org.apache.logging</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.flink.elasticsearch6.shaded.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml
index d42247f30e7..807f03d5343 100644
--- a/flink-connectors/pom.xml
+++ b/flink-connectors/pom.xml
@@ -82,6 +82,20 @@ under the License.
<!-- See main pom.xml for explanation of profiles -->
<profiles>
+
+ <!-- Create SQL Client uber jars by default -->
+ <profile>
+ <id>sql-jars</id>
+ <activation>
+ <property>
+ <name>!skipSqlJars</name>
+ </property>
+ </activation>
+ <modules>
+ <module>flink-sql-connector-elasticsearch6</module>
+ </modules>
+ </profile>
+
<!-- there's no Kafka 0.8 dependency for Scala 2.12, we only include when building
for Scala 2.11 -->
<profile>
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java
index efbc4fac039..32eed69e2ce 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java
@@ -23,6 +23,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
@@ -361,10 +362,14 @@ public int getMaximumParallelism() {
private static final class MaxDopVisitor implements Visitor<Operator<?>> {
+ private final Set<Operator> visitedOperators = new HashSet<>();
private int maxDop = -1;
-
+
@Override
public boolean preVisit(Operator<?> visitable) {
+ if (!visitedOperators.add(visitable)) {
+ return false;
+ }
this.maxDop = Math.max(this.maxDop, visitable.getParallelism());
return true;
}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java b/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
index 14cf647cd24..a7b6b39d5db 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
@@ -24,6 +24,7 @@
import org.apache.flink.api.common.io.compression.GzipInflaterInputStreamFactory;
import org.apache.flink.api.common.io.compression.InflaterInputStreamFactory;
import org.apache.flink.api.common.io.compression.XZInputStreamFactory;
+import org.apache.flink.api.common.io.filters.FileFilter;
import org.apache.flink.api.common.io.statistics.BaseStatistics;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
@@ -236,7 +237,7 @@ protected static String extractFileExtension(String fileName) {
/**
* Files filter for determining what files/directories should be included.
*/
- private FilePathFilter filesFilter = new GlobFilePathFilter();
+ private FileFilter filesFilter = new GlobFilePathFilter();
// --------------------------------------------------------------------------------------------
// Constructors
@@ -434,10 +435,24 @@ public long getSplitLength() {
return splitLength;
}
+ /**
+ * @deprecated Use {@link #setFilesFilter(FileFilter)}.
+ */
+ @Deprecated
public void setFilesFilter(FilePathFilter filesFilter) {
this.filesFilter = Preconditions.checkNotNull(filesFilter, "Files filter should not be null");
}
+ /**
+ * Set file filter to keep only accepted files.
+ *
+ * @param filesFilter The file filter.
+ */
+ public void setFilesFilter(FileFilter filesFilter) {
+ this.filesFilter = Preconditions.checkNotNull(filesFilter, "File filter should not be " +
+ "null");
+ }
+
// --------------------------------------------------------------------------------------------
// Pre-flight: Configuration, Splits, Sampling
// --------------------------------------------------------------------------------------------
@@ -530,9 +545,11 @@ protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path f
if (file.isDir()) {
totalLength += addFilesInDir(file.getPath(), files, false);
} else {
- files.add(file);
- testForUnsplittable(file);
- totalLength += file.getLen();
+ if (acceptFile(file)) {
+ files.add(file);
+ testForUnsplittable(file);
+ totalLength += file.getLen();
+ }
}
// check the modification time stamp
@@ -547,7 +564,7 @@ protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path f
}
// sanity check
- if (totalLength <= 0) {
+ if (totalLength < 0) {
totalLength = BaseStatistics.SIZE_UNKNOWN;
}
return new FileBaseStatistics(latestModTime, totalLength, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN);
@@ -757,9 +774,10 @@ protected boolean testForUnsplittable(FileStatus pathFile) {
*/
public boolean acceptFile(FileStatus fileStatus) {
final String name = fileStatus.getPath().getName();
+
return !name.startsWith("_")
&& !name.startsWith(".")
- && !filesFilter.filterPath(fileStatus.getPath());
+ && filesFilter.accept(fileStatus);
}
/**
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/FilePathFilter.java b/flink-core/src/main/java/org/apache/flink/api/common/io/FilePathFilter.java
index 097909636dd..afcf1eea574 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/FilePathFilter.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/FilePathFilter.java
@@ -17,17 +17,17 @@
package org.apache.flink.api.common.io;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.io.filters.FileFilter;
+import org.apache.flink.core.fs.FileStatus;
import org.apache.flink.core.fs.Path;
-import java.io.Serializable;
-
/**
- * The {@link #filterPath(Path)} method is responsible for deciding if a path is eligible for further
+ * A filter that is responsible for deciding if a path is eligible for further
* processing or not. This can serve to exclude temporary or partial files that
* are still being written.
*/
@PublicEvolving
-public abstract class FilePathFilter implements Serializable {
+public abstract class FilePathFilter implements FileFilter {
private static final long serialVersionUID = 1L;
@@ -44,9 +44,15 @@
* return filePath.getName().startsWith(".") || filePath.getName().contains("_COPYING_");
* }
* }</pre>
+ *
*/
public abstract boolean filterPath(Path filePath);
+ @Override
+ public boolean accept(FileStatus fileStatus) {
+ return !filterPath(fileStatus.getPath());
+ }
+
/**
* Returns the default filter, which excludes the following files:
*
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/GlobFilePathFilter.java b/flink-core/src/main/java/org/apache/flink/api/common/io/GlobFilePathFilter.java
index 748ed284248..993ce00c191 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/GlobFilePathFilter.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/GlobFilePathFilter.java
@@ -19,6 +19,7 @@
package org.apache.flink.api.common.io;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.fs.FileStatus;
import org.apache.flink.core.fs.Path;
import org.apache.flink.util.Preconditions;
@@ -135,4 +136,8 @@ private boolean shouldExclude(java.nio.file.Path nioPath) {
return false;
}
+ @Override
+ public boolean accept(FileStatus fileStatus) {
+ return !filterPath(fileStatus.getPath());
+ }
}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/filters/FileFilter.java b/flink-core/src/main/java/org/apache/flink/api/common/io/filters/FileFilter.java
new file mode 100644
index 00000000000..2507208370c
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/filters/FileFilter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.io.filters;
+
+import org.apache.flink.core.fs.FileStatus;
+
+import java.io.Serializable;
+
+/**
+ * A filter that decides if a file should be processed or not.
+ */
+public interface FileFilter extends Serializable {
+ /**
+ * Decides if a file should be accepted for processing or not.
+ *
+ * @param fileStatus The file
+ * @return True if the file should be accepted for processing; False, otherwise
+ */
+ boolean accept(FileStatus fileStatus);
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/filters/FileModTimeFilter.java b/flink-core/src/main/java/org/apache/flink/api/common/io/filters/FileModTimeFilter.java
new file mode 100644
index 00000000000..949ac13f41d
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/filters/FileModTimeFilter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.io.filters;
+
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * A filter that only keeps files whose their modification time is no less than a given timestamp
+ * as consuming time position. It basically sets a read start position in file system for Flink.
+ */
+public class FileModTimeFilter implements FileFilter {
+ /**
+ * The consuming time position to process files whose their modification time is no less than this timestamp.
+ */
+ private long consumingTimePosition;
+
+ public FileModTimeFilter(long consumingTimePosition) {
+ Preconditions.checkArgument(consumingTimePosition >= 0);
+
+ this.consumingTimePosition = consumingTimePosition;
+ }
+
+ @Override
+ public boolean accept(FileStatus fileStatus) {
+ // Always accept dir, because it may contain files newer than itself
+ return fileStatus.isDir() || fileStatus.getModificationTime() >= consumingTimePosition;
+ }
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
index 55f3df7d31c..3a0b84c8449 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
@@ -83,7 +83,7 @@
private final Map<String, Aggregator<?>> aggregators;
- private final ClassLoader classLoader;
+ private final ClassLoader userCodeClassLoader;
private final ExecutionConfig executionConfig;
@@ -99,7 +99,7 @@ public CollectionExecutor(ExecutionConfig executionConfig) {
this.previousAggregates = new HashMap<String, Value>();
this.aggregators = new HashMap<String, Aggregator<?>>();
this.cachedFiles = new HashMap<String, Future<Path>>();
- this.classLoader = getClass().getClassLoader();
+ this.userCodeClassLoader = Thread.currentThread().getContextClassLoader();
}
// --------------------------------------------------------------------------------------------
@@ -191,8 +191,8 @@ else if (operator instanceof GenericDataSinkBase) {
MetricGroup metrics = new UnregisteredMetricsGroup();
if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
} else {
ctx = null;
}
@@ -211,8 +211,8 @@ else if (operator instanceof GenericDataSinkBase) {
MetricGroup metrics = new UnregisteredMetricsGroup();
if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
} else {
ctx = null;
}
@@ -237,8 +237,8 @@ else if (operator instanceof GenericDataSinkBase) {
MetricGroup metrics = new UnregisteredMetricsGroup();
if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
List<?> bcData = execute(bcInputs.getValue());
@@ -278,8 +278,8 @@ else if (operator instanceof GenericDataSinkBase) {
MetricGroup metrics = new UnregisteredMetricsGroup();
if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
List<?> bcData = execute(bcInputs.getValue());
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
index 6557a9f7dd4..2b4bba66c5e 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
@@ -71,17 +71,31 @@
/**
* This option configures local recovery for this state backend. By default, local recovery is deactivated.
+ *
+ * <p>Local recovery currently only covers keyed state backends.
+ * Currently, MemoryStateBackend does not support local recovery and ignore
+ * this option.
*/
public static final ConfigOption<Boolean> LOCAL_RECOVERY = ConfigOptions
- .key("state.backend.local-recovery")
- .defaultValue(false);
+ .key("state.backend.local-recovery")
+ .defaultValue(false)
+ .withDescription("This option configures local recovery for this state backend. By default, local recovery is " +
+ "deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does " +
+ "not support local recovery and ignore this option.");
/**
* The config parameter defining the root directories for storing file-based state for local recovery.
+ *
+ * <p>Local recovery currently only covers keyed state backends.
+ * Currently, MemoryStateBackend does not support local recovery and ignore
+ * this option.
*/
public static final ConfigOption<String> LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS = ConfigOptions
- .key("taskmanager.state.local.root-dirs")
- .noDefaultValue();
+ .key("taskmanager.state.local.root-dirs")
+ .noDefaultValue()
+ .withDescription("The config parameter defining the root directories for storing file-based state for local " +
+ "recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does " +
+ "not support local recovery and ignore this option");
// ------------------------------------------------------------------------
// Options specific to the file-system-based state backends
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/SafetyNetCloseableRegistry.java b/flink-core/src/main/java/org/apache/flink/core/fs/SafetyNetCloseableRegistry.java
index ccf944eb2ac..870dcbff991 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/SafetyNetCloseableRegistry.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/SafetyNetCloseableRegistry.java
@@ -87,7 +87,7 @@ protected void doRegister(
assert Thread.holdsLock(getSynchronizationLock());
- Closeable innerCloseable = WrappingProxyUtil.stripProxy(wrappingProxyCloseable.getWrappedDelegate());
+ Closeable innerCloseable = WrappingProxyUtil.stripProxy(wrappingProxyCloseable);
if (null == innerCloseable) {
return;
@@ -108,7 +108,7 @@ protected boolean doUnRegister(
assert Thread.holdsLock(getSynchronizationLock());
- Closeable innerCloseable = WrappingProxyUtil.stripProxy(closeable.getWrappedDelegate());
+ Closeable innerCloseable = WrappingProxyUtil.stripProxy(closeable);
return null != innerCloseable && closeableMap.remove(innerCloseable) != null;
}
diff --git a/flink-core/src/main/java/org/apache/flink/util/WrappingProxyUtil.java b/flink-core/src/main/java/org/apache/flink/util/WrappingProxyUtil.java
index 3fbd6df56da..dee950803c1 100644
--- a/flink-core/src/main/java/org/apache/flink/util/WrappingProxyUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/util/WrappingProxyUtil.java
@@ -19,27 +19,53 @@
package org.apache.flink.util;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
+
+import javax.annotation.Nullable;
+
+import static java.lang.String.format;
/**
- * Utilits for working with {@link WrappingProxy}.
+ * Utilities for working with {@link WrappingProxy}.
*/
@Internal
public final class WrappingProxyUtil {
+ @VisibleForTesting
+ static final int SAFETY_NET_MAX_ITERATIONS = 128;
+
private WrappingProxyUtil() {
throw new AssertionError();
}
+ /**
+ * Expects a proxy, and returns the unproxied delegate.
+ *
+ * @param wrappingProxy The initial proxy.
+ * @param <T> The type of the delegate. Note that all proxies in the chain must be assignable to T.
+ * @return The unproxied delegate.
+ */
@SuppressWarnings("unchecked")
- public static <T> T stripProxy(T object) {
+ public static <T> T stripProxy(@Nullable final WrappingProxy<T> wrappingProxy) {
+ if (wrappingProxy == null) {
+ return null;
+ }
- T previous = null;
+ T delegate = wrappingProxy.getWrappedDelegate();
- while (object instanceof WrappingProxy && previous != object) {
- previous = object;
- object = ((WrappingProxy<T>) object).getWrappedDelegate();
+ int numProxiesStripped = 0;
+ while (delegate instanceof WrappingProxy) {
+ throwIfSafetyNetExceeded(++numProxiesStripped);
+ delegate = ((WrappingProxy<T>) delegate).getWrappedDelegate();
}
- return object;
+ return delegate;
+ }
+
+ private static void throwIfSafetyNetExceeded(final int numProxiesStripped) {
+ if (numProxiesStripped >= SAFETY_NET_MAX_ITERATIONS) {
+ throw new IllegalArgumentException(format("Already stripped %d proxies. " +
+ "Are there loops in the object graph?", SAFETY_NET_MAX_ITERATIONS));
+ }
}
}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/FileInputFormatTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/FileInputFormatTest.java
index 5f79875fa06..a5d92d402fa 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/FileInputFormatTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/FileInputFormatTest.java
@@ -19,6 +19,7 @@
package org.apache.flink.api.common.io;
import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
+import org.apache.flink.api.common.io.filters.FileModTimeFilter;
import org.apache.flink.api.common.io.statistics.BaseStatistics;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
@@ -324,11 +325,17 @@ public void testGetStatisticsOneFileWithCachedVersion() {
DummyFileInputFormat format = new DummyFileInputFormat();
format.setFilePath(tempFile);
format.configure(new Configuration());
-
-
+
FileBaseStatistics stats = format.getStatistics(null);
Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
-
+
+ // Test consuming time position
+ long position = System.currentTimeMillis();
+ format.setFilesFilter(new FileModTimeFilter(position));
+
+ FileBaseStatistics stats2 = format.getStatistics(null);
+ Assert.assertEquals("The file size from the statistics is wrong.", 0, stats2.getTotalInputSize());
+
format = new DummyFileInputFormat();
format.setFilePath(tempFile);
format.configure(new Configuration());
@@ -336,6 +343,11 @@ public void testGetStatisticsOneFileWithCachedVersion() {
FileBaseStatistics newStats = format.getStatistics(stats);
Assert.assertTrue("Statistics object was changed", newStats == stats);
+ // Test consuming time position
+ format.setFilesFilter(new FileModTimeFilter(position));
+ FileBaseStatistics newStats2 = format.getStatistics(null);
+ Assert.assertEquals("The file size from the statistics is wrong.", 0, newStats2.getTotalInputSize());
+
// insert fake stats with the correct modification time. the call should return the fake stats
format = new DummyFileInputFormat();
format.setFilePath(tempFile);
@@ -431,44 +443,79 @@ public void testGetStatisticsMultipleNonExistingFile() throws IOException {
}
@Test
- public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException {
+ public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException, InterruptedException {
final long size1 = 1024 * 500;
String tempFile = TestFileUtils.createTempFile(size1);
+ long position1 = System.currentTimeMillis();
+ // Must take a break here, otherwise all temp files will create all at once with the same modification time
+ Thread.sleep(1000);
+
final long size2 = 1024 * 505;
String tempFile2 = TestFileUtils.createTempFile(size2);
- final long totalSize = size1 + size2;
-
+ // Test multi files
final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
format.setFilePaths(tempFile, tempFile2);
format.configure(new Configuration());
BaseStatistics stats = format.getStatistics(null);
- Assert.assertEquals("The file size from the statistics is wrong.", totalSize, stats.getTotalInputSize());
+ Assert.assertEquals("The file size from the statistics is wrong.", size1 + size2, stats
+ .getTotalInputSize());
+
+ // Test multi files with consuming time position1
+ format.setFilesFilter(new FileModTimeFilter(position1));
+ stats = format.getStatistics(null);
+ Assert.assertEquals("The file size from the statistics is wrong.", size2, stats.getTotalInputSize());
+
+ // Test multi files with consuming time position2
+ long position2 = System.currentTimeMillis();
+ format.setFilesFilter(new FileModTimeFilter(position2));
+ stats = format.getStatistics(null);
+ Assert.assertEquals("The file size from the statistics is wrong.", 0, stats.getTotalInputSize());
}
@Test
- public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException {
+ public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException, InterruptedException {
final long size1 = 2077;
final long size2 = 31909;
final long size3 = 10;
final long totalSize123 = size1 + size2 + size3;
-
- String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size1, size2, size3);
-
+
+ File dir = temporaryFolder.newFolder();
+ String tempDir = TestFileUtils.createTempFileDir(dir, size1, size2, size3);
+
+ long position = System.currentTimeMillis();
+ // Must take a break here, otherwise all temp files will create all at once with the same modification time
+ Thread.sleep(1000);
+ // Create an extra 1-byte file in the same dir to test consuming time position
+ TestFileUtils.createTempFileDir(dir, 1);
+
final long size4 = 2051;
final long size5 = 31902;
final long size6 = 15;
final long totalSize456 = size4 + size5 + size6;
- String tempDir2 = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size4, size5, size6);
+ String tempDir2 = TestFileUtils.createTempFileDir(dir, size4, size5, size6);
+ // Test multi files multi paths
final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
format.setFilePaths(tempDir, tempDir2);
format.configure(new Configuration());
BaseStatistics stats = format.getStatistics(null);
Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456, stats.getTotalInputSize());
+
+ // Test multi files nested multi paths
+ format.setFilePaths(dir.toPath().toString());
+ format.setNestedFileEnumeration(true);
+ stats = format.getStatistics(null);
+ Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456 + 1, stats.getTotalInputSize());
+
+ // Test multi files multi paths with consuming time position
+ format.setFilesFilter(new FileModTimeFilter(position));
+ format.setNestedFileEnumeration(true);
+ stats = format.getStatistics(null);
+ Assert.assertEquals("The file size from the statistics is wrong.", totalSize456 + 1, stats.getTotalInputSize());
}
@Test
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/filters/FileModTimeFilterTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/filters/FileModTimeFilterTest.java
new file mode 100644
index 00000000000..6d362a88e8a
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/filters/FileModTimeFilterTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.io.filters;
+
+import org.apache.flink.core.fs.FileStatus;
+import org.apache.flink.core.fs.Path;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for the FileModTimeFilter.
+ */
+public class FileModTimeFilterTest {
+ private static final long TEST_TIMESTAMP = 10L;
+
+ private static final FileStatus MOCKED_DIR = new MockedDirFileStatus();
+ private static final FileStatus MOCKED_FILE = new MockedFileStatus();
+
+ @Test
+ public void testAlwaysAcceptDir() {
+ assertTrue(new FileModTimeFilter(TEST_TIMESTAMP - 1).accept(MOCKED_DIR));
+ assertTrue(new FileModTimeFilter(TEST_TIMESTAMP).accept(MOCKED_DIR));
+ assertTrue(new FileModTimeFilter(TEST_TIMESTAMP + 1).accept(MOCKED_DIR));
+ }
+
+ @Test
+ public void testFilterFile() {
+ assertTrue(new FileModTimeFilter(TEST_TIMESTAMP - 1).accept(MOCKED_FILE));
+ assertTrue(new FileModTimeFilter(TEST_TIMESTAMP).accept(MOCKED_FILE));
+ assertFalse(new FileModTimeFilter(TEST_TIMESTAMP + 1).accept(MOCKED_FILE));
+ }
+
+ private static class MockedFileStatus implements FileStatus {
+
+ @Override
+ public long getLen() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getBlockSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public short getReplication() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getModificationTime() {
+ return 10;
+ }
+
+ @Override
+ public long getAccessTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDir() {
+ return false;
+ }
+
+ @Override
+ public Path getPath() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static class MockedDirFileStatus implements FileStatus {
+
+ @Override
+ public long getLen() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getBlockSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public short getReplication() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getModificationTime() {
+ return 20;
+ }
+
+ @Override
+ public long getAccessTime() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDir() {
+ return true;
+ }
+
+ @Override
+ public Path getPath() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java
index 93ae552af2d..45dbae81737 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/MissingTypeInfoTest.java
@@ -20,10 +20,6 @@
import org.apache.flink.api.common.functions.InvalidTypesException;
import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
-import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
public class MissingTypeInfoTest extends TypeInformationTestBase<MissingTypeInfo> {
private static final String functionName = "foobar";
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java
index b67d7545b3c..1bd983f6f27 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/ValueTypeInfoTest.java
@@ -23,14 +23,11 @@
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.types.Record;
import org.apache.flink.types.Value;
-import org.apache.flink.util.TestLogger;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
-import static org.junit.Assert.*;
-
/**
* Test for {@link ListTypeInfo}.
*/
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/AbstractCloseableRegistryTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/AbstractCloseableRegistryTest.java
index eb07378493b..d8d639e5f11 100644
--- a/flink-core/src/test/java/org/apache/flink/core/fs/AbstractCloseableRegistryTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/AbstractCloseableRegistryTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.core.fs;
-import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.util.AbstractCloseableRegistry;
import org.junit.Assert;
@@ -26,22 +25,27 @@
import java.io.Closeable;
import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.spy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Tests for the {@link AbstractCloseableRegistry}.
*/
public abstract class AbstractCloseableRegistryTest<C extends Closeable, T> {
+ private static final int TEST_TIMEOUT_SECONDS = 10;
+
protected ProducerThread[] streamOpenThreads;
protected AbstractCloseableRegistry<C, T> closeableRegistry;
protected AtomicInteger unclosedCounter;
- protected abstract C createCloseable();
+ protected abstract void registerCloseable(Closeable closeable) throws IOException;
protected abstract AbstractCloseableRegistry<C, T> createRegistry();
@@ -74,7 +78,6 @@ protected void joinThreads() throws InterruptedException {
@Test
public void testClose() throws Exception {
-
setup(Integer.MAX_VALUE);
startThreads();
@@ -87,45 +90,29 @@ public void testClose() throws Exception {
joinThreads();
- Assert.assertEquals(0, unclosedCounter.get());
- Assert.assertEquals(0, closeableRegistry.getNumberOfRegisteredCloseables());
+ assertEquals(0, unclosedCounter.get());
+ assertEquals(0, closeableRegistry.getNumberOfRegisteredCloseables());
- final C testCloseable = spy(createCloseable());
+ final TestCloseable testCloseable = new TestCloseable();
try {
+ registerCloseable(testCloseable);
+ fail("Closed registry should not accept closeables!");
+ } catch (IOException expected) {}
- closeableRegistry.registerCloseable(testCloseable);
-
- Assert.fail("Closed registry should not accept closeables!");
-
- } catch (IOException expected) {
- //expected
- }
-
- Assert.assertEquals(0, unclosedCounter.get());
- Assert.assertEquals(0, closeableRegistry.getNumberOfRegisteredCloseables());
- verify(testCloseable).close();
+ assertTrue(testCloseable.isClosed());
+ assertEquals(0, unclosedCounter.get());
+ assertEquals(0, closeableRegistry.getNumberOfRegisteredCloseables());
}
@Test
public void testNonBlockingClose() throws Exception {
setup(Integer.MAX_VALUE);
- final OneShotLatch waitRegistryClosedLatch = new OneShotLatch();
- final OneShotLatch blockCloseLatch = new OneShotLatch();
-
- final C spyCloseable = spy(createCloseable());
-
- doAnswer(invocationOnMock -> {
- invocationOnMock.callRealMethod();
- waitRegistryClosedLatch.trigger();
- blockCloseLatch.await();
- return null;
- }).when(spyCloseable).close();
-
- closeableRegistry.registerCloseable(spyCloseable);
+ final BlockingTestCloseable blockingCloseable = new BlockingTestCloseable();
+ registerCloseable(blockingCloseable);
- Assert.assertEquals(1, closeableRegistry.getNumberOfRegisteredCloseables());
+ assertEquals(1, closeableRegistry.getNumberOfRegisteredCloseables());
Thread closer = new Thread(() -> {
try {
@@ -134,23 +121,20 @@ public void testNonBlockingClose() throws Exception {
}
});
-
closer.start();
- waitRegistryClosedLatch.await();
-
- final C testCloseable = spy(createCloseable());
+ blockingCloseable.awaitClose(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ final TestCloseable testCloseable = new TestCloseable();
try {
- closeableRegistry.registerCloseable(testCloseable);
- Assert.fail("Closed registry should not accept closeables!");
+ registerCloseable(testCloseable);
+ fail("Closed registry should not accept closeables!");
} catch (IOException ignored) {}
- blockCloseLatch.trigger();
+ blockingCloseable.unblockClose();
closer.join();
- verify(spyCloseable).close();
- verify(testCloseable).close();
- Assert.assertEquals(0, closeableRegistry.getNumberOfRegisteredCloseables());
+ assertTrue(testCloseable.isClosed());
+ assertEquals(0, closeableRegistry.getNumberOfRegisteredCloseables());
}
/**
@@ -225,4 +209,55 @@ public synchronized void close() throws IOException {
refCount.decrementAndGet();
}
}
+
+ /**
+ * A noop {@link Closeable} implementation that blocks inside {@link #close()}.
+ */
+ private static class BlockingTestCloseable implements Closeable {
+
+ private final CountDownLatch closeCalledLatch = new CountDownLatch(1);
+
+ private final CountDownLatch blockCloseLatch = new CountDownLatch(1);
+
+ @Override
+ public void close() throws IOException {
+ closeCalledLatch.countDown();
+ try {
+ blockCloseLatch.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Unblocks {@link #close()}.
+ */
+ public void unblockClose() {
+ blockCloseLatch.countDown();
+ }
+
+ /**
+ * Causes the current thread to wait until {@link #close()} is called.
+ */
+ public void awaitClose(final long timeout, final TimeUnit timeUnit) throws InterruptedException {
+ closeCalledLatch.await(timeout, timeUnit);
+ }
+ }
+
+ /**
+ * A noop {@link Closeable} implementation that tracks whether it was closed.
+ */
+ private static class TestCloseable implements Closeable {
+
+ private final AtomicBoolean closed = new AtomicBoolean();
+
+ @Override
+ public void close() throws IOException {
+ assertTrue("TestCloseable was already closed", closed.compareAndSet(false, true));
+ }
+
+ public boolean isClosed() {
+ return closed.get();
+ }
+ }
}
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/CloseableRegistryTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/CloseableRegistryTest.java
index 8a0fb9679b9..63d08d5a363 100644
--- a/flink-core/src/test/java/org/apache/flink/core/fs/CloseableRegistryTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/CloseableRegistryTest.java
@@ -30,13 +30,8 @@
public class CloseableRegistryTest extends AbstractCloseableRegistryTest<Closeable, Object> {
@Override
- protected Closeable createCloseable() {
- return new Closeable() {
- @Override
- public void close() throws IOException {
-
- }
- };
+ protected void registerCloseable(final Closeable closeable) throws IOException {
+ closeableRegistry.registerCloseable(closeable);
}
@Override
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java
index 598b1e11a03..f6d3f45a44b 100644
--- a/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java
@@ -19,6 +19,7 @@
package org.apache.flink.core.fs;
import org.apache.flink.core.fs.local.LocalFileSystem;
+import org.apache.flink.util.WrappingProxy;
import org.apache.flink.util.WrappingProxyUtil;
import org.junit.Test;
@@ -38,21 +39,32 @@
public void testGet() throws URISyntaxException, IOException {
String scheme = "file";
- assertTrue(WrappingProxyUtil.stripProxy(FileSystem.get(new URI(scheme + ":///test/test"))) instanceof LocalFileSystem);
+ assertTrue(getFileSystemWithoutSafetyNet(scheme + ":///test/test") instanceof LocalFileSystem);
try {
- FileSystem.get(new URI(scheme + "://test/test"));
+ getFileSystemWithoutSafetyNet(scheme + "://test/test");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().startsWith("Found local file path with authority '"));
}
- assertTrue(WrappingProxyUtil.stripProxy(FileSystem.get(new URI(scheme + ":/test/test"))) instanceof LocalFileSystem);
+ assertTrue(getFileSystemWithoutSafetyNet(scheme + ":/test/test") instanceof LocalFileSystem);
- assertTrue(WrappingProxyUtil.stripProxy(FileSystem.get(new URI(scheme + ":test/test"))) instanceof LocalFileSystem);
+ assertTrue(getFileSystemWithoutSafetyNet(scheme + ":test/test") instanceof LocalFileSystem);
- assertTrue(WrappingProxyUtil.stripProxy(FileSystem.get(new URI("/test/test"))) instanceof LocalFileSystem);
+ assertTrue(getFileSystemWithoutSafetyNet("/test/test") instanceof LocalFileSystem);
- assertTrue(WrappingProxyUtil.stripProxy(FileSystem.get(new URI("test/test"))) instanceof LocalFileSystem);
+ assertTrue(getFileSystemWithoutSafetyNet("test/test") instanceof LocalFileSystem);
+ }
+
+ private static FileSystem getFileSystemWithoutSafetyNet(final String uri) throws URISyntaxException, IOException {
+ final FileSystem fileSystem = FileSystem.get(new URI(uri));
+
+ if (fileSystem instanceof WrappingProxy) {
+ //noinspection unchecked
+ return WrappingProxyUtil.stripProxy((WrappingProxy<FileSystem>) fileSystem);
+ }
+
+ return fileSystem;
}
}
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/SafetyNetCloseableRegistryTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/SafetyNetCloseableRegistryTest.java
index 5474f9905ae..44461a3cb38 100644
--- a/flink-core/src/test/java/org/apache/flink/core/fs/SafetyNetCloseableRegistryTest.java
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/SafetyNetCloseableRegistryTest.java
@@ -43,17 +43,20 @@
public final TemporaryFolder tmpFolder = new TemporaryFolder();
@Override
- protected WrappingProxyCloseable<? extends Closeable> createCloseable() {
- return new WrappingProxyCloseable<Closeable>() {
+ protected void registerCloseable(final Closeable closeable) throws IOException {
+ final WrappingProxyCloseable<Closeable> wrappingProxyCloseable = new WrappingProxyCloseable<Closeable>() {
@Override
- public void close() throws IOException {}
+ public void close() throws IOException {
+ closeable.close();
+ }
@Override
public Closeable getWrappedDelegate() {
- return this;
+ return closeable;
}
};
+ closeableRegistry.registerCloseable(wrappingProxyCloseable);
}
@Override
diff --git a/flink-core/src/test/java/org/apache/flink/util/WrappingProxyUtilTest.java b/flink-core/src/test/java/org/apache/flink/util/WrappingProxyUtilTest.java
new file mode 100644
index 00000000000..b557b099ebf
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/util/WrappingProxyUtilTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flink.util;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests for {@link WrappingProxyUtil}.
+ */
+public class WrappingProxyUtilTest {
+
+ @Test
+ public void testThrowsExceptionIfTooManyProxies() {
+ try {
+ WrappingProxyUtil.stripProxy(new SelfWrappingProxy(WrappingProxyUtil.SAFETY_NET_MAX_ITERATIONS));
+ fail("Expected exception not thrown");
+ } catch (final IllegalArgumentException e) {
+ assertThat(e.getMessage(), containsString("Are there loops in the object graph?"));
+ }
+ }
+
+ @Test
+ public void testStripsAllProxies() {
+ final SelfWrappingProxy wrappingProxy = new SelfWrappingProxy(WrappingProxyUtil.SAFETY_NET_MAX_ITERATIONS - 1);
+ assertThat(WrappingProxyUtil.stripProxy(wrappingProxy), is(not(instanceOf(SelfWrappingProxy.class))));
+ }
+
+ private static class Wrapped {
+ }
+
+ /**
+ * Wraps around {@link Wrapped} a specified number of times.
+ */
+ private static class SelfWrappingProxy extends Wrapped implements WrappingProxy<Wrapped> {
+
+ private int levels;
+
+ private SelfWrappingProxy(final int levels) {
+ this.levels = levels;
+ }
+
+ @Override
+ public Wrapped getWrappedDelegate() {
+ if (levels-- == 0) {
+ return new Wrapped();
+ } else {
+ return this;
+ }
+ }
+ }
+
+}
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index bff3eb0bd96..e5991e12228 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -343,6 +343,13 @@ under the License.
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-oss-fs-hadoop</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-queryable-state-runtime_${scala.binary.version}</artifactId>
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 0d9acf3c3ae..aa18ef31278 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -140,6 +140,13 @@
<fileMode>0644</fileMode>
</file>
+ <file>
+ <source>../flink-filesystems/flink-oss-fs-hadoop/target/flink-oss-fs-hadoop-${project.version}.jar</source>
+ <outputDirectory>opt/</outputDirectory>
+ <destName>flink-oss-fs-hadoop-${project.version}.jar</destName>
+ <fileMode>0644</fileMode>
+ </file>
+
<!-- Queryable State -->
<file>
<source>../flink-queryable-state/flink-queryable-state-runtime/target/flink-queryable-state-runtime_${scala.binary.version}-${project.version}.jar</source>
diff --git a/flink-end-to-end-tests/flink-sql-client-test/pom.xml b/flink-end-to-end-tests/flink-sql-client-test/pom.xml
index d5646688d58..3b04f8f61c9 100644
--- a/flink-end-to-end-tests/flink-sql-client-test/pom.xml
+++ b/flink-end-to-end-tests/flink-sql-client-test/pom.xml
@@ -95,9 +95,8 @@ under the License.
<dependency>
<!-- Used by maven-dependency-plugin -->
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-elasticsearch6_${scala.binary.version}</artifactId>
+ <artifactId>flink-sql-connector-elasticsearch6_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <classifier>sql-jar</classifier>
<scope>provided</scope>
</dependency>
</dependencies>
@@ -195,9 +194,8 @@ under the License.
</artifactItem>
<artifactItem>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-elasticsearch6_${scala.binary.version}</artifactId>
+ <artifactId>flink-sql-connector-elasticsearch6_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <classifier>sql-jar</classifier>
<type>jar</type>
</artifactItem>
</artifactItems>
diff --git a/flink-end-to-end-tests/test-scripts/common_s3.sh b/flink-end-to-end-tests/test-scripts/common_s3.sh
index fbe8cadecd6..63fa941387a 100644
--- a/flink-end-to-end-tests/test-scripts/common_s3.sh
+++ b/flink-end-to-end-tests/test-scripts/common_s3.sh
@@ -58,7 +58,7 @@ s3util="java -jar ${END_TO_END_DIR}/flink-e2e-test-utils/target/S3UtilProgram.ja
# IT_CASE_S3_ACCESS_KEY
# IT_CASE_S3_SECRET_KEY
# Arguments:
-# None
+# $1 - s3 filesystem type (hadoop/presto)
# Returns:
# None
###################################
@@ -73,13 +73,11 @@ function s3_setup {
}
trap s3_cleanup EXIT
- cp $FLINK_DIR/opt/flink-s3-fs-hadoop-*.jar $FLINK_DIR/lib/
+ cp $FLINK_DIR/opt/flink-s3-fs-$1-*.jar $FLINK_DIR/lib/
echo "s3.access-key: $IT_CASE_S3_ACCESS_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
echo "s3.secret-key: $IT_CASE_S3_SECRET_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
}
-s3_setup
-
###################################
# List s3 objects by full path prefix.
#
diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
index 489d0df6ade..ddbb6868424 100755
--- a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
+++ b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
@@ -22,6 +22,7 @@
source "$(dirname "$0")"/common.sh
source "$(dirname "$0")"/common_s3.sh
+s3_setup hadoop
start_cluster
$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out
diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
index a963e096131..9ebbb0d8c08 100755
--- a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
+++ b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
@@ -22,6 +22,7 @@
source "$(dirname "$0")"/common.sh
source "$(dirname "$0")"/common_s3.sh
+s3_setup presto
start_cluster
$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh b/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh
index e810e68bde7..2756197fdf6 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh
@@ -22,6 +22,7 @@ OUT_TYPE="${1:-local}" # other type: s3
source "$(dirname "$0")"/common.sh
source "$(dirname "$0")"/common_s3.sh
+s3_setup hadoop
set_conf_ssl "mutual"
OUT=temp/test_streaming_file_sink-$(uuidgen)
diff --git a/flink-filesystems/flink-oss-fs-hadoop/pom.xml b/flink-filesystems/flink-oss-fs-hadoop/pom.xml
new file mode 100644
index 00000000000..b4f70a49e45
--- /dev/null
+++ b/flink-filesystems/flink-oss-fs-hadoop/pom.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>flink-filesystems</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <version>1.8-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>flink-oss-fs-hadoop</artifactId>
+
+ <properties>
+ <fs.oss.sdk.version>3.1.0</fs.oss.sdk.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-hadoop-fs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-fs-hadoop-shaded</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <version>${fs.hadoopshaded.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-oss-sdk</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-sdk-oss</artifactId>
+ <version>${fs.oss.sdk.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-fs-hadoop-shaded</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+
+ <!-- this is merely an intermediate build artifact and should not be -->
+ <!-- deployed to maven central -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Relocate all OSS related classes -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.hadoop</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop</shadedPattern>
+ </relocation>
+ <!-- relocate the OSS dependencies -->
+ <relocation>
+ <pattern>com.aliyun</pattern>
+ <shadedPattern>org.apache.flink.fs.osshadoop.shaded.com.aliyun</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.aliyuncs</pattern>
+ <shadedPattern>org.apache.flink.fs.osshadoop.shaded.com.aliyuncs</shadedPattern>
+ </relocation>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>.gitkeep</exclude>
+ <exclude>mime.types</exclude>
+ <exclude>mozilla/**</exclude>
+ <exclude>META-INF/maven/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink-filesystems/flink-oss-fs-hadoop/src/main/java/org/apache/flink/fs/osshadoop/OSSFileSystemFactory.java b/flink-filesystems/flink-oss-fs-hadoop/src/main/java/org/apache/flink/fs/osshadoop/OSSFileSystemFactory.java
new file mode 100644
index 00000000000..52335f5891f
--- /dev/null
+++ b/flink-filesystems/flink-oss-fs-hadoop/src/main/java/org/apache/flink/fs/osshadoop/OSSFileSystemFactory.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.fs.osshadoop;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.FileSystemFactory;
+import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
+
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * Simple factory for the OSS file system.
+ */
+public class OSSFileSystemFactory implements FileSystemFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(OSSFileSystemFactory.class);
+
+ private Configuration flinkConfig;
+
+ private org.apache.hadoop.conf.Configuration hadoopConfig;
+
+ private static final Set<String> CONFIG_KEYS_TO_SHADE = Collections.singleton("fs.oss.credentials.provider");
+
+ private static final String FLINK_SHADING_PREFIX = "org.apache.flink.fs.shaded.hadoop3.";
+
+ /**
+ * In order to simplify, we make flink oss configuration keys same with hadoop oss module.
+ * So, we add all configuration key with prefix `fs.oss` in flink conf to hadoop conf
+ */
+ private static final String[] FLINK_CONFIG_PREFIXES = { "fs.oss."};
+
+ @Override
+ public String getScheme() {
+ return "oss";
+ }
+
+ @Override
+ public void configure(Configuration config) {
+ flinkConfig = config;
+ hadoopConfig = null;
+ }
+
+ @Override
+ public FileSystem create(URI fsUri) throws IOException {
+ this.hadoopConfig = getHadoopConfiguration();
+
+ final String scheme = fsUri.getScheme();
+ final String authority = fsUri.getAuthority();
+
+ if (scheme == null && authority == null) {
+ fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
+ } else if (scheme != null && authority == null) {
+ URI defaultUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
+ if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
+ fsUri = defaultUri;
+ }
+ }
+
+ final AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+ fs.initialize(fsUri, hadoopConfig);
+ return new HadoopFileSystem(fs);
+ }
+
+ @VisibleForTesting
+ org.apache.hadoop.conf.Configuration getHadoopConfiguration() {
+ org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
+ if (flinkConfig == null) {
+ return conf;
+ }
+
+ // read all configuration with prefix 'FLINK_CONFIG_PREFIXES'
+ for (String key : flinkConfig.keySet()) {
+ for (String prefix : FLINK_CONFIG_PREFIXES) {
+ if (key.startsWith(prefix)) {
+ String value = flinkConfig.getString(key, null);
+ conf.set(key, value);
+ if (CONFIG_KEYS_TO_SHADE.contains(key)) {
+ conf.set(key, FLINK_SHADING_PREFIX + value);
+ }
+
+ LOG.debug("Adding Flink config entry for {} as {} to Hadoop config", key, conf.get(key));
+ }
+ }
+ }
+ return conf;
+ }
+}
diff --git a/flink-filesystems/flink-oss-fs-hadoop/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory b/flink-filesystems/flink-oss-fs-hadoop/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory
new file mode 100644
index 00000000000..4810f8a643b
--- /dev/null
+++ b/flink-filesystems/flink-oss-fs-hadoop/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.fs.osshadoop.OSSFileSystemFactory
diff --git a/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemITCase.java b/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemITCase.java
new file mode 100644
index 00000000000..33f96b85333
--- /dev/null
+++ b/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemITCase.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.fs.osshadoop;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FSDataOutputStream;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Unit tests for the OSS file system support via AliyunOSSFileSystem.
+ * These tests do actually read from or write to OSS.
+ */
+public class HadoopOSSFileSystemITCase extends TestLogger {
+
+ private static final String ENDPOINT = System.getenv("ARTIFACTS_OSS_ENDPOINT");
+ private static final String BUCKET = System.getenv("ARTIFACTS_OSS_BUCKET");
+ private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID();
+ private static final String ACCESS_KEY = System.getenv("ARTIFACTS_OSS_ACCESS_KEY");
+ private static final String SECRET_KEY = System.getenv("ARTIFACTS_OSS_SECRET_KEY");
+
+ @BeforeClass
+ public static void checkIfCredentialsArePresent() {
+ Assume.assumeTrue("Aliyun OSS endpoint not configured, skipping test...", ENDPOINT != null);
+ Assume.assumeTrue("Aliyun OSS bucket not configured, skipping test...", BUCKET != null);
+ Assume.assumeTrue("Aliyun OSS access key not configured, skipping test...", ACCESS_KEY != null);
+ Assume.assumeTrue("Aliyun OSS secret key not configured, skipping test...", SECRET_KEY != null);
+ }
+
+ @Test
+ public void testReadAndWrite() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setString("fs.oss.endpoint", ENDPOINT);
+ conf.setString("fs.oss.accessKeyId", ACCESS_KEY);
+ conf.setString("fs.oss.accessKeySecret", SECRET_KEY);
+ final String testLine = "Aliyun OSS";
+
+ FileSystem.initialize(conf);
+ final Path path = new Path("oss://" + BUCKET + '/' + TEST_DATA_DIR);
+ final FileSystem fs = path.getFileSystem();
+ try {
+ for (int i = 0; i < 10; ++i) {
+ final Path file = new Path(path.getPath() + "/test.data." + i);
+ try (FSDataOutputStream out = fs.create(file, FileSystem.WriteMode.OVERWRITE)) {
+ try (OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
+ writer.write(testLine);
+ }
+ }
+ try (FSDataInputStream in = fs.open(file);
+ InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
+ BufferedReader reader = new BufferedReader(ir)) {
+ String line = reader.readLine();
+ assertEquals(testLine, line);
+ }
+ }
+ assertTrue(fs.exists(path));
+ assertEquals(10, fs.listStatus(path).length);
+ } finally {
+ fs.delete(path, true);
+ }
+ }
+
+ @Test
+ public void testShadedConfigurations() {
+ final Configuration conf = new Configuration();
+ conf.setString("fs.oss.endpoint", ENDPOINT);
+ conf.setString("fs.oss.accessKeyId", ACCESS_KEY);
+ conf.setString("fs.oss.accessKeySecret", SECRET_KEY);
+ conf.setString("fs.oss.credentials.provider", "org.apache.hadoop.fs.aliyun.oss.AliyunCredentialsProvider");
+
+ OSSFileSystemFactory ossfsFactory = new OSSFileSystemFactory();
+ ossfsFactory.configure(conf);
+ org.apache.hadoop.conf.Configuration configuration = ossfsFactory.getHadoopConfiguration();
+ // shaded
+ assertEquals("org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.aliyun.oss.AliyunCredentialsProvider",
+ configuration.get("fs.oss.credentials.provider"));
+ // should not shaded
+ assertEquals(ENDPOINT, configuration.get("fs.oss.endpoint"));
+ assertEquals(ACCESS_KEY, configuration.get("fs.oss.accessKeyId"));
+ assertEquals(SECRET_KEY, configuration.get("fs.oss.accessKeySecret"));
+ }
+}
diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml
index 5baed430cd1..913f4ade62f 100644
--- a/flink-filesystems/flink-s3-fs-presto/pom.xml
+++ b/flink-filesystems/flink-s3-fs-presto/pom.xml
@@ -194,6 +194,29 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>ban-openjdk.jol</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <bannedDependencies>
+ <excludes>
+ <!-- Incompatible license -->
+ <exclude>org.openjdk.jol:*</exclude>
+ </excludes>
+ </bannedDependencies>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -210,9 +233,6 @@ under the License.
<includes>
<include>*:*</include>
</includes>
- <excludes>
- <exclude>org.openjdk.jol</exclude>
- </excludes>
</artifactSet>
<relocations>
<!-- relocate the references to Hadoop to match the pre-shaded hadoop artifact -->
diff --git a/flink-filesystems/pom.xml b/flink-filesystems/pom.xml
index b4742f53e96..cde662cd987 100644
--- a/flink-filesystems/pom.xml
+++ b/flink-filesystems/pom.xml
@@ -46,6 +46,7 @@ under the License.
<module>flink-s3-fs-hadoop</module>
<module>flink-s3-fs-presto</module>
<module>flink-swift-fs-hadoop</module>
+ <module>flink-oss-fs-hadoop</module>
</modules>
<!-- Common dependency setup for all filesystems -->
diff --git a/flink-formats/flink-avro-confluent-registry/pom.xml b/flink-formats/flink-avro-confluent-registry/pom.xml
index 72a74ac11ba..4c99f71e544 100644
--- a/flink-formats/flink-avro-confluent-registry/pom.xml
+++ b/flink-formats/flink-avro-confluent-registry/pom.xml
@@ -78,10 +78,31 @@ under the License.
<goal>shade</goal>
</goals>
<configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>io.confluent:*</include>
+ <include>com.fasterxml.jackson.core:*</include>
+ <include>org.apache.zookeeper:zookeeper</include>
+ <include>com.101tec:zkclient</include>
+ </includes>
+ </artifactSet>
<relocations combine.children="append">
<relocation>
- <pattern>com.fasterxml.jackson.core</pattern>
- <shadedPattern>org.apache.flink.formats.avro.registry.confluent.shaded.com.fasterxml.jackson.core</shadedPattern>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.flink.formats.avro.registry.confluent.shaded.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.zookeeper</pattern>
+ <shadedPattern>org.apache.flink.formats.avro.registry.confluent.shaded.org.apache.zookeeper</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.jute</pattern>
+ <shadedPattern>org.apache.flink.formats.avro.registry.confluent.shaded.org.apache.jute</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.I0Itec.zkclient</pattern>
+ <shadedPattern>org.apache.flink.formats.avro.registry.confluent.shaded.org.101tec</shadedPattern>
</relocation>
</relocations>
</configuration>
diff --git a/flink-formats/flink-avro-confluent-registry/src/main/resources/META-INF/NOTICE b/flink-formats/flink-avro-confluent-registry/src/main/resources/META-INF/NOTICE
new file mode 100644
index 00000000000..24f2ff84657
--- /dev/null
+++ b/flink-formats/flink-avro-confluent-registry/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,15 @@
+flink-avro-confluent-registry
+Copyright 2014-2018 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- com.101tec:zkclient:0.10
+- com.fasterxml.jackson.core:jackson-databind:2.8.4
+- com.fasterxml.jackson.core:jackson-annotations:2.8.0
+- com.fasterxml.jackson.core:jackson-core:2.8.4
+- io.confluent:common-utils:3.3.1
+- io.confluent:kafka-schema-registry-client:3.3.1
+- org.apache.zookeeper:zookeeper:3.4.10
diff --git a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java
index d1f0a5f5778..484ef860e2d 100644
--- a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java
+++ b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/avro/ParquetStreamingFileSinkITCase.java
@@ -19,13 +19,14 @@
package org.apache.flink.formats.parquet.avro;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.fs.Path;
import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo;
import org.apache.flink.formats.parquet.generated.Address;
-import org.apache.flink.formats.parquet.testutils.FiniteTestSource;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
+import org.apache.flink.streaming.util.FiniteTestSource;
import org.apache.flink.test.util.AbstractTestBase;
import org.apache.avro.Schema;
@@ -156,11 +157,14 @@ public void testWriteParquetAvroReflect() throws Exception {
File[] partFiles = buckets[0].listFiles();
assertNotNull(partFiles);
- assertEquals(1, partFiles.length);
- assertTrue(partFiles[0].length() > 0);
+ assertEquals(2, partFiles.length);
- List<Address> results = readParquetFile(partFiles[0], dataModel);
- assertEquals(expected, results);
+ for (File partFile : partFiles) {
+ assertTrue(partFile.length() > 0);
+
+ final List<Tuple2<Long, String>> fileContent = readParquetFile(partFile, dataModel);
+ assertEquals(expected, fileContent);
+ }
}
private static <T> List<T> readParquetFile(File file, GenericData dataModel) throws IOException {
diff --git a/flink-formats/flink-sequence-file/pom.xml b/flink-formats/flink-sequence-file/pom.xml
new file mode 100644
index 00000000000..1a2f84e4629
--- /dev/null
+++ b/flink-formats/flink-sequence-file/pom.xml
@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-formats</artifactId>
+ <version>1.8-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-sequence-file</artifactId>
+ <name>flink-sequence-file</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Hadoop is needed for SequenceFile -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-hadoop2</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+
+ <build>
+ <plugins>
+ <!-- skip dependency convergence due to Hadoop dependency -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dependency-convergence</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriter.java b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriter.java
new file mode 100644
index 00000000000..169aa4ba7cb
--- /dev/null
+++ b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriter.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.sequencefile;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.serialization.BulkWriter;
+import org.apache.flink.api.java.tuple.Tuple2;
+
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+
+import java.io.IOException;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A {@link BulkWriter} implementation that wraps a {@link SequenceFile.Writer}.
+ *
+ * @param <K> The type of key written.
+ * @param <V> The type of value written.
+ */
+@PublicEvolving
+public class SequenceFileWriter<K extends Writable, V extends Writable> implements BulkWriter<Tuple2<K, V>> {
+
+ private final SequenceFile.Writer writer;
+
+ SequenceFileWriter(SequenceFile.Writer writer) {
+ this.writer = checkNotNull(writer);
+ }
+
+ @Override
+ public void addElement(Tuple2<K, V> element) throws IOException {
+ writer.append(element.f0, element.f1);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writer.hsync();
+ }
+
+ @Override
+ public void finish() throws IOException {
+ writer.close();
+ }
+}
diff --git a/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.java b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.java
new file mode 100644
index 00000000000..d7b96f67002
--- /dev/null
+++ b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.sequencefile;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.serialization.BulkWriter;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.fs.FSDataOutputStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+
+import java.io.IOException;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A factory that creates a SequenceFile {@link BulkWriter}.
+ *
+ * @param <K> The type of key to write. It should be writable.
+ * @param <V> The type of value to write. It should be writable.
+ */
+@PublicEvolving
+public class SequenceFileWriterFactory<K extends Writable, V extends Writable> implements BulkWriter.Factory<Tuple2<K, V>> {
+
+ private static final long serialVersionUID = 1L;
+
+ /** A constant specifying that no compression is requested. */
+ public static final String NO_COMPRESSION = "NO_COMPRESSION";
+
+ private final SerializableHadoopConfiguration serializableHadoopConfig;
+ private final Class<K> keyClass;
+ private final Class<V> valueClass;
+ private final String compressionCodecName;
+ private final SequenceFile.CompressionType compressionType;
+
+ /**
+ * Creates a new SequenceFileWriterFactory using the given builder to assemble the
+ * SequenceFileWriter.
+ *
+ * @param hadoopConf The Hadoop configuration for Sequence File Writer.
+ * @param keyClass The class of key to write.
+ * @param valueClass The class of value to write.
+ */
+ public SequenceFileWriterFactory(Configuration hadoopConf, Class<K> keyClass, Class<V> valueClass) {
+ this(hadoopConf, keyClass, valueClass, NO_COMPRESSION, SequenceFile.CompressionType.BLOCK);
+ }
+
+ /**
+ * Creates a new SequenceFileWriterFactory using the given builder to assemble the
+ * SequenceFileWriter.
+ *
+ * @param hadoopConf The Hadoop configuration for Sequence File Writer.
+ * @param keyClass The class of key to write.
+ * @param valueClass The class of value to write.
+ * @param compressionCodecName The name of compression codec.
+ */
+ public SequenceFileWriterFactory(Configuration hadoopConf, Class<K> keyClass, Class<V> valueClass, String compressionCodecName) {
+ this(hadoopConf, keyClass, valueClass, compressionCodecName, SequenceFile.CompressionType.BLOCK);
+ }
+
+ /**
+ * Creates a new SequenceFileWriterFactory using the given builder to assemble the
+ * SequenceFileWriter.
+ *
+ * @param hadoopConf The Hadoop configuration for Sequence File Writer.
+ * @param keyClass The class of key to write.
+ * @param valueClass The class of value to write.
+ * @param compressionCodecName The name of compression codec.
+ * @param compressionType The type of compression level.
+ */
+ public SequenceFileWriterFactory(Configuration hadoopConf, Class<K> keyClass, Class<V> valueClass, String compressionCodecName, SequenceFile.CompressionType compressionType) {
+ this.serializableHadoopConfig = new SerializableHadoopConfiguration(checkNotNull(hadoopConf));
+ this.keyClass = checkNotNull(keyClass);
+ this.valueClass = checkNotNull(valueClass);
+ this.compressionCodecName = checkNotNull(compressionCodecName);
+ this.compressionType = checkNotNull(compressionType);
+ }
+
+ @Override
+ public SequenceFileWriter<K, V> create(FSDataOutputStream out) throws IOException {
+ org.apache.hadoop.fs.FSDataOutputStream stream = new org.apache.hadoop.fs.FSDataOutputStream(out, null);
+ CompressionCodec compressionCodec = getCompressionCodec(serializableHadoopConfig.get(), compressionCodecName);
+ SequenceFile.Writer writer = SequenceFile.createWriter(
+ serializableHadoopConfig.get(),
+ SequenceFile.Writer.stream(stream),
+ SequenceFile.Writer.keyClass(keyClass),
+ SequenceFile.Writer.valueClass(valueClass),
+ SequenceFile.Writer.compression(compressionType, compressionCodec));
+ return new SequenceFileWriter<>(writer);
+ }
+
+ private CompressionCodec getCompressionCodec(Configuration conf, String compressionCodecName) {
+ checkNotNull(conf);
+ checkNotNull(compressionCodecName);
+
+ if (compressionCodecName.equals(NO_COMPRESSION)) {
+ return null;
+ }
+
+ CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf);
+ CompressionCodec codec = codecFactory.getCodecByName(compressionCodecName);
+ if (codec == null) {
+ throw new RuntimeException("Codec " + compressionCodecName + " not found.");
+ }
+ return codec;
+ }
+}
+
diff --git a/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfiguration.java b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfiguration.java
new file mode 100644
index 00000000000..8e00e072b69
--- /dev/null
+++ b/flink-formats/flink-sequence-file/src/main/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfiguration.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.sequencefile;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+/**
+ * Wrapper class for serialization of {@link Configuration}.
+ */
+class SerializableHadoopConfiguration implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private transient Configuration hadoopConfig;
+
+ SerializableHadoopConfiguration(Configuration hadoopConfig) {
+ this.hadoopConfig = hadoopConfig;
+ }
+
+ Configuration get() {
+ return this.hadoopConfig;
+ }
+
+ // --------------------
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ this.hadoopConfig.write(out);
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException {
+ final Configuration config = new Configuration();
+ config.readFields(in);
+
+ if (this.hadoopConfig == null) {
+ this.hadoopConfig = config;
+ }
+ }
+}
diff --git a/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SequenceStreamingFileSinkITCase.java b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SequenceStreamingFileSinkITCase.java
new file mode 100644
index 00000000000..2b3f325579e
--- /dev/null
+++ b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SequenceStreamingFileSinkITCase.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.sequencefile;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
+import org.apache.flink.streaming.util.FiniteTestSource;
+import org.apache.flink.test.util.AbstractTestBase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Integration test case for writing bulk encoded files with the
+ * {@link StreamingFileSink} with SequenceFile.
+ */
+public class SequenceStreamingFileSinkITCase extends AbstractTestBase {
+
+ private final Configuration configuration = new Configuration();
+
+ private final List<Tuple2<Long, String>> testData = Arrays.asList(
+ new Tuple2<>(1L, "a"),
+ new Tuple2<>(2L, "b"),
+ new Tuple2<>(3L, "c")
+ );
+
+ @Test
+ public void testWriteSequenceFile() throws Exception {
+ final File folder = TEMPORARY_FOLDER.newFolder();
+ final Path testPath = Path.fromLocalFile(folder);
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+ env.enableCheckpointing(100);
+
+ DataStream<Tuple2<Long, String>> stream = env.addSource(
+ new FiniteTestSource<>(testData),
+ TypeInformation.of(new TypeHint<Tuple2<Long, String>>() {
+
+ })
+ );
+
+ stream.map(new MapFunction<Tuple2<Long, String>, Tuple2<LongWritable, Text>>() {
+ @Override
+ public Tuple2<LongWritable, Text> map(Tuple2<Long, String> value) throws Exception {
+ return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1));
+ }
+ }).addSink(
+ StreamingFileSink.forBulkFormat(
+ testPath,
+ new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2")
+ ).build());
+
+ env.execute();
+
+ validateResults(folder, testData);
+ }
+
+ private List<Tuple2<Long, String>> readSequenceFile(File file) throws IOException {
+ SequenceFile.Reader reader = new SequenceFile.Reader(
+ configuration, SequenceFile.Reader.file(new org.apache.hadoop.fs.Path(file.toURI())));
+ LongWritable key = new LongWritable();
+ Text val = new Text();
+ ArrayList<Tuple2<Long, String>> results = new ArrayList<>();
+ while (reader.next(key, val)) {
+ results.add(new Tuple2<>(key.get(), val.toString()));
+ }
+ reader.close();
+ return results;
+ }
+
+ private void validateResults(File folder, List<Tuple2<Long, String>> expected) throws Exception {
+ File[] buckets = folder.listFiles();
+ assertNotNull(buckets);
+ assertEquals(1, buckets.length);
+
+ final File[] partFiles = buckets[0].listFiles();
+ assertNotNull(partFiles);
+ assertEquals(2, partFiles.length);
+
+ for (File partFile : partFiles) {
+ assertTrue(partFile.length() > 0);
+
+ final List<Tuple2<Long, String>> fileContent = readSequenceFile(partFile);
+ assertEquals(expected, fileContent);
+ }
+ }
+}
diff --git a/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfigurationTest.java b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfigurationTest.java
new file mode 100644
index 00000000000..ea0fb955dc5
--- /dev/null
+++ b/flink-formats/flink-sequence-file/src/test/java/org/apache/flink/formats/sequencefile/SerializableHadoopConfigurationTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.sequencefile;
+
+import org.apache.hadoop.conf.Configuration;
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeMatcher;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+/**
+ * Tests for the {@link SerializableHadoopConfiguration}.
+ */
+public class SerializableHadoopConfigurationTest {
+
+ private static final String TEST_KEY = "test-key";
+
+ private static final String TEST_VALUE = "test-value";
+
+ private Configuration configuration;
+
+ @Before
+ public void createConfigWithCustomProperty() {
+ this.configuration = new Configuration();
+ configuration.set(TEST_KEY, TEST_VALUE);
+ }
+
+ @Test
+ public void customPropertiesSurviveSerializationDeserialization() throws IOException, ClassNotFoundException {
+ final SerializableHadoopConfiguration serializableConfigUnderTest = new SerializableHadoopConfiguration(configuration);
+ final byte[] serializedConfigUnderTest = serializeAndGetBytes(serializableConfigUnderTest);
+ final SerializableHadoopConfiguration deserializableConfigUnderTest = deserializeAndGetConfiguration(serializedConfigUnderTest);
+
+ Assert.assertThat(deserializableConfigUnderTest.get(), hasTheSamePropertiesAs(configuration));
+ }
+
+ // ---------------------------------------- Matchers ---------------------------------------- //
+
+ private static TypeSafeMatcher<Configuration> hasTheSamePropertiesAs(final Configuration expectedConfig) {
+ return new TypeSafeMatcher<Configuration>() {
+ @Override
+ protected boolean matchesSafely(Configuration actualConfig) {
+ final String value = actualConfig.get(TEST_KEY);
+ return actualConfig != expectedConfig && value != null && expectedConfig.get(TEST_KEY).equals(value);
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("a Hadoop Configuration with property: key=")
+ .appendValue(TEST_KEY)
+ .appendText(" and value=")
+ .appendValue(TEST_VALUE);
+ }
+ };
+ }
+
+ // ---------------------------------------- Helper Methods ---------------------------------------- //
+
+ private byte[] serializeAndGetBytes(SerializableHadoopConfiguration serializableConfigUnderTest) throws IOException {
+ try (
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ ObjectOutputStream out = new ObjectOutputStream(byteStream)
+ ) {
+ out.writeObject(serializableConfigUnderTest);
+ out.flush();
+ return byteStream.toByteArray();
+ }
+ }
+
+ private SerializableHadoopConfiguration deserializeAndGetConfiguration(byte[] serializedConfig) throws IOException, ClassNotFoundException {
+ try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(serializedConfig))) {
+ return (SerializableHadoopConfiguration) in.readObject();
+ }
+ }
+}
diff --git a/flink-formats/flink-sequence-file/src/test/resources/log4j-test.properties b/flink-formats/flink-sequence-file/src/test/resources/log4j-test.properties
new file mode 100644
index 00000000000..644b884d04d
--- /dev/null
+++ b/flink-formats/flink-sequence-file/src/test/resources/log4j-test.properties
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=OFF, testlogger
+log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
+log4j.appender.testlogger.target=System.err
+log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
+log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
diff --git a/flink-formats/pom.xml b/flink-formats/pom.xml
index c354402c21f..51c34660b0b 100644
--- a/flink-formats/pom.xml
+++ b/flink-formats/pom.xml
@@ -40,6 +40,7 @@ under the License.
<module>flink-json</module>
<module>flink-avro-confluent-registry</module>
<module>flink-parquet</module>
+ <module>flink-sequence-file</module>
</modules>
<!-- override these root dependencies as 'provided', so they don't end up
diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
index f4a2c9f1ef9..1161368f073 100644
--- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
@@ -80,7 +80,8 @@
Tuple2.of(MigrationVersion.v1_3, 1496532000000L),
Tuple2.of(MigrationVersion.v1_4, 1516897628000L),
Tuple2.of(MigrationVersion.v1_5, 1533639934000L),
- Tuple2.of(MigrationVersion.v1_6, 1534696817000L));
+ Tuple2.of(MigrationVersion.v1_6, 1534696817000L),
+ Tuple2.of(MigrationVersion.v1_7, 1544024599000L));
}
/**
diff --git a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1544024599000-flink1.7-snapshot b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1544024599000-flink1.7-snapshot
new file mode 100644
index 00000000000..40f0191bb3d
Binary files /dev/null and b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1544024599000-flink1.7-snapshot differ
diff --git a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.7-snapshot b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.7-snapshot
new file mode 100644
index 00000000000..fa2d68d7782
Binary files /dev/null and b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.7-snapshot differ
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index 49096009217..9afb8f01b4f 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -46,6 +46,11 @@ under the License.
<artifactId>flink-shaded-asm</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-asm-6</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ClosureCleaner.java b/flink-java/src/main/java/org/apache/flink/api/java/ClosureCleaner.java
index 3b809137e61..bdc0c6749f6 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ClosureCleaner.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ClosureCleaner.java
@@ -22,10 +22,10 @@
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.shaded.asm5.org.objectweb.asm.ClassReader;
-import org.apache.flink.shaded.asm5.org.objectweb.asm.ClassVisitor;
-import org.apache.flink.shaded.asm5.org.objectweb.asm.MethodVisitor;
-import org.apache.flink.shaded.asm5.org.objectweb.asm.Opcodes;
+import org.apache.flink.shaded.asm6.org.objectweb.asm.ClassReader;
+import org.apache.flink.shaded.asm6.org.objectweb.asm.ClassVisitor;
+import org.apache.flink.shaded.asm6.org.objectweb.asm.MethodVisitor;
+import org.apache.flink.shaded.asm6.org.objectweb.asm.Opcodes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index 22a2a93f066..beb1b65c4a5 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -73,6 +73,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -963,12 +964,16 @@ public Plan createProgramPlan(String jobName, boolean clearSinks) {
if (!config.isAutoTypeRegistrationDisabled()) {
plan.accept(new Visitor<org.apache.flink.api.common.operators.Operator<?>>() {
- private final HashSet<Class<?>> deduplicator = new HashSet<>();
+ private final Set<Class<?>> registeredTypes = new HashSet<>();
+ private final Set<org.apache.flink.api.common.operators.Operator<?>> visitedOperators = new HashSet<>();
@Override
public boolean preVisit(org.apache.flink.api.common.operators.Operator<?> visitable) {
+ if (!visitedOperators.add(visitable)) {
+ return false;
+ }
OperatorInformation<?> opInfo = visitable.getOperatorInfo();
- Serializers.recursivelyRegisterType(opInfo.getOutputType(), config, deduplicator);
+ Serializers.recursivelyRegisterType(opInfo.getOutputType(), config, registeredTypes);
return true;
}
diff --git a/flink-jepsen/README.md b/flink-jepsen/README.md
index a3e2668c26b..d329c8379ad 100644
--- a/flink-jepsen/README.md
+++ b/flink-jepsen/README.md
@@ -15,27 +15,37 @@ The faults that can be currently introduced to the Flink cluster include:
* Network partitions
There are many more properties other than job availability that could be
-verified but are not yet covered by this test suite, e.g., end-to-end exactly-once processing
+verified but are not yet fully covered by this project, e.g., end-to-end exactly-once processing
semantics.
## Usage
+
+### Setting up the Environment
See the [Jepsen documentation](https://github.com/jepsen-io/jepsen#setting-up-a-jepsen-environment)
-for how to set up the environment to run tests. The script under `scripts/run-tests.sh` documents how to invoke
-tests. The Flink job used for testing is located under
-`flink-end-to-end-tests/flink-datastream-allround-test`. You have to build the job first and copy
-the resulting jar (`DataStreamAllroundTestProgram.jar`) to the `./bin` directory of this project's
-root.
+for details on how to set up the environment required to run the tests.
+To simplify development, we have prepared Dockerfiles and a [Docker Compose](https://docs.docker.com/compose/) template
+so that you can run the tests locally in containers (see Section [Docker](#usage-docker)).
+
+### Running Tests
+This project does not comprise of only a single test that can be run but rather a parameterizable
+test template. This allows the user to specify the cluster manager that Flink should be on, the
+location of the high availability storage directory, the jobs to be submitted, etc.
+The script under `docker/run-tests.sh` shows examples on how to specify and run tests.
+By default, the example tests run the `DataStreamAllroundTestProgram`, which is located under
+`flink-end-to-end-tests/flink-datastream-allround-test` of the Flink project root.
+Before running the tests, you have to build the job first, and copy the resulting jar
+(`DataStreamAllroundTestProgram.jar`) to the `./bin` directory of this project's root.
+Also included in the examples is a more complicated scenario with two jobs that share a Kafka
+topic. See the `run-tests.sh` script for details on how to enable and run this test.
### Docker
-
-To simplify development, we have prepared Dockerfiles and a Docker Compose template
-so that you can run the tests locally in containers. To build the images
-and start the containers, simply run:
+To build the images and start the containers, simply run:
$ cd docker
$ ./up.sh
-After the containers started, open a new terminal window and run `docker exec -it jepsen-control bash`.
+This should start one control node container and three containers that will be used as DB nodes.
+After the containers have started, open a new terminal window and run `docker exec -it jepsen-control bash`.
This will allow you to run arbitrary commands on the control node.
To start the tests, you can use the `run-tests.sh` script in the `docker` directory,
which expects the number of test iterations, and a URI to a Flink distribution, e.g.,
diff --git a/flink-jepsen/docker/run-tests.sh b/flink-jepsen/docker/run-tests.sh
index 8b2b1e6d18f..b2fd195b4da 100755
--- a/flink-jepsen/docker/run-tests.sh
+++ b/flink-jepsen/docker/run-tests.sh
@@ -17,6 +17,8 @@
# limitations under the License.
################################################################################
+set -euo pipefail
+
dockerdir=$(dirname $0)
dockerdir=$(cd ${dockerdir}; pwd)
@@ -26,6 +28,47 @@ n2
n3
EOF
-common_jepsen_args+=(--nodes-file ${dockerdir}/nodes)
+common_jepsen_args+=(--ha-storage-dir hdfs:///flink
+--tarball ${2}
+--ssh-private-key ~/.ssh/id_rsa
+--nodes-file ${dockerdir}/nodes)
+
+for i in $(seq 1 ${1})
+do
+ echo "Executing run #${i} of ${1}"
+
+ # YARN session cluster
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-task-managers --test-spec "${dockerdir}/test-specs/yarn-session.edn"
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --test-spec "${dockerdir}/test-specs/yarn-session.edn"
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen fail-name-node-during-recovery --test-spec "${dockerdir}/test-specs/yarn-session.edn"
+
+ # YARN per-job cluster
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-task-managers --test-spec "${dockerdir}/test-specs/yarn-job.edn"
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --test-spec "${dockerdir}/test-specs/yarn-job.edn"
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen fail-name-node-during-recovery --test-spec "${dockerdir}/test-specs/yarn-job.edn"
+
+ # Mesos
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-task-managers --test-spec "${dockerdir}/test-specs/mesos-session.edn"
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --test-spec "${dockerdir}/test-specs/mesos-session.edn"
+
+ # Standalone
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --test-spec "${dockerdir}/test-specs/standalone-session.edn"
+ lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --client-gen cancel-jobs --test-spec "${dockerdir}/test-specs/standalone-session.edn"
+
+ # Below is a test that uses Flink's exactly-once Kafka producer/consumer.
+ # The test submits two jobs:
+ #
+ # (1) DataGeneratorJob - Publishes data to a Kafka topic
+ # (2) StateMachineJob - Consumes data from the same Kafka topic, and validates exactly-once semantics
+ #
+ # To enable the test, you first need to build the flink-state-machine-kafka job jar,
+ # and copy the artifact to flink-jepsen/bin:
+ #
+ # git clone https://github.com/igalshilman/flink-state-machine-example
+ # cd flink-state-machine-example
+ # mvn clean package -pl flink-state-machine-kafka/flink-state-machine-kafka -am
+ # cp flink-state-machine-kafka/flink-state-machine-kafka/target/flink-state-machine-kafka-1.0-SNAPSHOT.jar /path/to/flink-jepsen/bin
+ #
+ # lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-task-managers-bursts --time-limit 60 --test-spec "${dockerdir}/test-specs/standalone-session-kafka.edn" --job-running-healthy-threshold 15
-. ${dockerdir}/../scripts/run-tests.sh ${1} ${2} 1
+done
diff --git a/flink-jepsen/docker/test-specs/mesos-session.edn b/flink-jepsen/docker/test-specs/mesos-session.edn
new file mode 100644
index 00000000000..e4c01bfd733
--- /dev/null
+++ b/flink-jepsen/docker/test-specs/mesos-session.edn
@@ -0,0 +1,19 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+{:dbs [:hadoop :zookeeper :mesos :flink-mesos-session]
+ :jobs [{:job-jar "/jepsen/bin/DataStreamAllroundTestProgram.jar"
+ :job-args "--environment.parallelism 1 --state_backend.checkpoint_directory hdfs:///checkpoints --state_backend rocks --state_backend.rocks.incremental true"}]}
diff --git a/flink-jepsen/docker/test-specs/standalone-session-kafka.edn b/flink-jepsen/docker/test-specs/standalone-session-kafka.edn
new file mode 100644
index 00000000000..5f6704d53f4
--- /dev/null
+++ b/flink-jepsen/docker/test-specs/standalone-session-kafka.edn
@@ -0,0 +1,24 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+{:dbs [:hadoop :zookeeper :kafka :flink-standalone-session]
+ :jobs [{:job-jar "/jepsen/bin/flink-state-machine-kafka-1.0-SNAPSHOT.jar"
+ :job-args "--parallelism 1 --checkpointInterval 5000 --numKeys 1000 --topic kafka-test-topic --sleep 200 --semantic exactly-once --bootstrap.servers localhost:9092 --transaction.timeout.ms 600000 --checkpointDir hdfs:///flink-checkpoints"
+ :main-class "com.dataartisans.flink.example.eventpattern.DataGeneratorJob"}
+
+ {:job-jar "/jepsen/bin/flink-state-machine-kafka-1.0-SNAPSHOT.jar"
+ :job-args "--parallelism 1 --checkpointInterval 5000 --input-topic kafka-test-topic --bootstrap.servers localhost:9092 --checkpointDir hdfs:///flink-checkpoints --auto.offset.reset earliest"
+ :main-class "com.dataartisans.flink.example.eventpattern.StateMachineJob"}]}
diff --git a/flink-jepsen/docker/test-specs/standalone-session.edn b/flink-jepsen/docker/test-specs/standalone-session.edn
new file mode 100644
index 00000000000..763d592bda2
--- /dev/null
+++ b/flink-jepsen/docker/test-specs/standalone-session.edn
@@ -0,0 +1,19 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+{:dbs [:hadoop :zookeeper :flink-standalone-session]
+ :jobs [{:job-jar "/jepsen/bin/DataStreamAllroundTestProgram.jar"
+ :job-args "--environment.parallelism 1 --state_backend.checkpoint_directory hdfs:///checkpoints --state_backend rocks --state_backend.rocks.incremental true"}]}
diff --git a/flink-jepsen/docker/test-specs/yarn-job.edn b/flink-jepsen/docker/test-specs/yarn-job.edn
new file mode 100644
index 00000000000..390f21d31f7
--- /dev/null
+++ b/flink-jepsen/docker/test-specs/yarn-job.edn
@@ -0,0 +1,19 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+{:dbs [:hadoop :zookeeper :flink-yarn-job]
+ :jobs [{:job-jar "/jepsen/bin/DataStreamAllroundTestProgram.jar"
+ :job-args "--environment.parallelism 1 --state_backend.checkpoint_directory hdfs:///checkpoints --state_backend rocks --state_backend.rocks.incremental true"}]}
diff --git a/flink-jepsen/docker/test-specs/yarn-session.edn b/flink-jepsen/docker/test-specs/yarn-session.edn
new file mode 100644
index 00000000000..aa95c159405
--- /dev/null
+++ b/flink-jepsen/docker/test-specs/yarn-session.edn
@@ -0,0 +1,19 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+{:dbs [:hadoop :zookeeper :flink-yarn-session]
+ :jobs [{:job-jar "/jepsen/bin/DataStreamAllroundTestProgram.jar"
+ :job-args "--environment.parallelism 1 --state_backend.checkpoint_directory hdfs:///checkpoints --state_backend rocks --state_backend.rocks.incremental true"}]}
diff --git a/flink-jepsen/scripts/run-tests.sh b/flink-jepsen/scripts/run-tests.sh
deleted file mode 100755
index a2b256b6f6a..00000000000
--- a/flink-jepsen/scripts/run-tests.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-set -euo pipefail
-
-scripts=$(dirname $0)
-scripts=$(cd ${scripts}; pwd)
-
-parallelism=${3}
-
-common_jepsen_args+=(--ha-storage-dir hdfs:///flink
---job-jar ${scripts}/../bin/DataStreamAllroundTestProgram.jar
---tarball ${2}
---job-args "--environment.parallelism ${parallelism} --state_backend.checkpoint_directory hdfs:///checkpoints --state_backend rocks --state_backend.rocks.incremental true"
---ssh-private-key ~/.ssh/id_rsa)
-
-for i in $(seq 1 ${1})
-do
- echo "Executing run #${i} of ${1}"
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-task-managers --deployment-mode yarn-session
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --deployment-mode yarn-session
- lein run test "${common_jepsen_args[@]}" --nemesis-gen fail-name-node-during-recovery --deployment-mode yarn-session
-
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-task-managers --deployment-mode yarn-job
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --deployment-mode yarn-job
- lein run test "${common_jepsen_args[@]}" --nemesis-gen fail-name-node-during-recovery --deployment-mode yarn-job
-
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-task-managers --deployment-mode mesos-session
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --deployment-mode mesos-session
-
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --deployment-mode standalone-session
- lein run test "${common_jepsen_args[@]}" --nemesis-gen kill-job-managers --client-gen cancel-job --deployment-mode standalone-session
- echo
-done
diff --git a/flink-jepsen/src/jepsen/flink/checker.clj b/flink-jepsen/src/jepsen/flink/checker.clj
index 7e437e9d628..d5ff6e0ebb0 100644
--- a/flink-jepsen/src/jepsen/flink/checker.clj
+++ b/flink-jepsen/src/jepsen/flink/checker.clj
@@ -155,34 +155,80 @@
([job-running-healthy-threshold job-recovery-grace-period]
(job-running-within-grace-period job-running-healthy-threshold job-recovery-grace-period 10)))
-(defn get-job-running-history
+(defn- history->jobs-running?-value
[history]
(->>
history
- (remove #(= (:process %) :nemesis))
+ (filter #(= (:f %) :jobs-running?))
(remove #(= (:type %) :invoke))
- (map :value)
- (map boolean)
- (remove nil?)))
+ (map :value)))
+
+(defn- history->job-ids
+ "Extracts all job ids from a history."
+ [history]
+ (set (->> history
+ (history->jobs-running?-value)
+ (map keys)
+ (flatten)
+ (remove nil?))))
+
+(defn all-jobs-running?-history
+ [history]
+ (->>
+ history
+ (history->jobs-running?-value)
+ (map vals)
+ (map #(and
+ (not (empty? %))
+ (every? true? %)))))
(defn- healthy?
[model]
- (>= (:healthy-count model) (:healthy-threshold model)))
+ (or (>= (:healthy-count model) (:healthy-threshold model))
+ (:job-canceled? model)))
+
+(defn- jobs-running?->job-running?
+ "Rewrites history entries of the form {:f :jobs-running? :value {...}}
+
+ Example: {:type ok :f :jobs-running? :value {job-id-1 true}} -> {:type ok :f :job-running? :value true}"
+ [history-entry job-id]
+ (let [job-running?-entry (assoc history-entry :f :job-running?)
+ job-running?-entry-ok (update job-running?-entry :value #(get % job-id))]
+ (if (= (:type history-entry) :ok)
+ job-running?-entry-ok
+ job-running?-entry)))
+
+(defn- history->single-job-history
+ "Rewrites a history to one that appears to run a single Flink job."
+ [history job-id]
+ (let [transform-history-entry (fn [history-entry]
+ (case (:f history-entry)
+ :jobs-running? (jobs-running?->job-running? history-entry job-id)
+ :cancel-jobs (assoc history-entry :f :cancel-job)
+ history-entry))]
+ (map transform-history-entry history)))
+
+(defn- compute-final-model
+ [model history]
+ (let [start-time (-> history first :time)]
+ (reduce knossos.model/step
+ (assoc model :last-failure start-time)
+ history)))
(defn job-running-checker
[]
(reify
checker/Checker
(check [_ test model history _]
- (let [final (reduce model/step (assoc model :last-failure (:time (first history))) history)
- result-map (conj {}
- (find test :nemesis-gen)
- (find test :deployment-mode))]
- (if (or (model/inconsistent? final)
- (and
- (not (healthy? final))
- (not (:job-canceled? final))))
- (into result-map {:valid? false
- :final-model final})
- (into result-map {:valid? true
- :final-model final}))))))
+ (let [job-ids (history->job-ids history)
+ individual-job-histories (map (partial history->single-job-history history) job-ids)
+ final-models (map (partial compute-final-model model) individual-job-histories)
+ inconsistent-or-unhealthy (or (empty? job-ids)
+ (some model/inconsistent? final-models)
+ (some (complement healthy?) final-models))
+ result-map (select-keys test [:nemesis-gen :deployment-mode])]
+ (if inconsistent-or-unhealthy
+ (into result-map {:valid? false
+ :final-models final-models})
+ (into result-map {:valid? true
+ :final-models final-models}))))))
diff --git a/flink-jepsen/src/jepsen/flink/client.clj b/flink-jepsen/src/jepsen/flink/client.clj
index 1ab987bd704..17746ec5c6b 100644
--- a/flink-jepsen/src/jepsen/flink/client.clj
+++ b/flink-jepsen/src/jepsen/flink/client.clj
@@ -119,6 +119,14 @@
(map :status)
(every? #(= "RUNNING" %)))))))
+(defn jobs-running?
+ "Checks if multiple jobs are running. Returns a map where the keys are job ids and the values are
+ booleans indicating whether the job is running or not."
+ [base-url job-ids]
+ (let [job-running-on-current-master? (partial job-running? base-url)
+ make-job-id-running?-pair (juxt identity job-running-on-current-master?)]
+ (into {} (map make-job-id-running?-pair job-ids))))
+
(defn- cancel-job!
"Cancels the specified job. Returns true if the job could be canceled.
Returns false if the job does not exist. Throws an exception if the HTTP status
@@ -131,6 +139,10 @@
(not (http/success? response)) (throw (ex-info "Job cancellation unsuccessful" {:job-id job-id :error error}))
:else true)))
+(defn- cancel-jobs!
+ [base-url job-ids]
+ (doseq [job-id job-ids] (cancel-job! base-url job-id)))
+
(defmacro dispatch-operation
[op & body]
`(try
@@ -148,24 +160,23 @@
(System/exit 1)))))
(defn- dispatch-rest-operation!
- [rest-url job-id op]
- (assert job-id)
+ [rest-url job-ids op]
+ (assert job-ids)
(if-not rest-url
(assoc op :type :fail :error "Have not determined REST URL yet.")
(case (:f op)
- :job-running? (dispatch-operation op (fu/retry
- (partial job-running? rest-url job-id)
- :retries 3
- :fallback #(throw %)))
- :cancel-job (dispatch-operation-or-fatal op (cancel-job! rest-url job-id)))))
+ :jobs-running? (dispatch-operation op (fu/retry
+ (partial jobs-running? rest-url job-ids)
+ :retries 3
+ :fallback #(throw %)))
+ :cancel-jobs (dispatch-operation-or-fatal op (cancel-jobs! rest-url job-ids)))))
(defrecord Client
- [deploy-cluster! ; function that starts a non-standalone cluster and submits the job
- closer ; function that closes the ZK client
+ [closer ; function that closes the ZK client
rest-url ; atom storing the current rest-url
init-future ; future that completes if rest-url is set to an initial value
- job-id ; atom storing the job-id
- job-submitted?] ; Has the job already been submitted? Used to avoid re-submission if the client is re-opened.
+ job-ids ; atom storing the job-ids
+ client-setup?] ; Has the client already been setup? Used to avoid running setup! again if the client is re-opened.
client/Client
(open! [this test _]
(info "Open client.")
@@ -174,23 +185,21 @@
:rest-url rest-url-atom
:init-future init-future)))
- (setup! [_ test]
+ (setup! [_ _]
(info "Setup client.")
- (when (compare-and-set! job-submitted? false true)
- (deploy-cluster! test)
+ (when (compare-and-set! client-setup? false true)
(deref init-future)
(let [jobs (fu/retry (fn [] (list-jobs! @rest-url))
:fallback (fn [e]
(fatal e "Could not get running jobs.")
(System/exit 1)))
- num-jobs (count jobs)
- job (first jobs)]
- (assert (= 1 num-jobs) (str "Expected 1 job, was " num-jobs))
- (info "Submitted job" job)
- (reset! job-id job))))
+ num-jobs (count jobs)]
+ (assert (pos? num-jobs) (str "Expected at least 1 job, was " num-jobs))
+ (info "Submitted jobs" jobs)
+ (reset! job-ids jobs))))
(invoke! [_ _ op]
- (dispatch-rest-operation! @rest-url @job-id op))
+ (dispatch-rest-operation! @rest-url @job-ids op))
(teardown! [_ _])
(close! [_ _]
@@ -198,5 +207,5 @@
(closer)))
(defn create-client
- [deploy-cluster!]
- (Client. deploy-cluster! nil nil nil (atom nil) (atom false)))
+ []
+ (Client. nil nil nil (atom nil) (atom false)))
diff --git a/flink-jepsen/src/jepsen/flink/db.clj b/flink-jepsen/src/jepsen/flink/db.clj
index 971c69e5000..6bac982622a 100644
--- a/flink-jepsen/src/jepsen/flink/db.clj
+++ b/flink-jepsen/src/jepsen/flink/db.clj
@@ -17,13 +17,11 @@
(ns jepsen.flink.db
(:require [clj-http.client :as http]
[clojure.java.io]
- [clojure.string :as str]
[clojure.tools.logging :refer :all]
[jepsen
[control :as c]
[db :as db]
- [util :refer [meh]]
- [zookeeper :as zk]]
+ [util :refer [meh]]]
[jepsen.control.util :as cu]
[jepsen.flink.hadoop :as hadoop]
[jepsen.flink.mesos :as mesos]
@@ -36,12 +34,6 @@
(def conf-file (str install-dir "/conf/flink-conf.yaml"))
(def masters-file (str install-dir "/conf/masters"))
-(def default-flink-dist-url "https://archive.apache.org/dist/flink/flink-1.6.0/flink-1.6.0-bin-hadoop28-scala_2.11.tgz")
-(def hadoop-dist-url "https://archive.apache.org/dist/hadoop/common/hadoop-2.8.3/hadoop-2.8.3.tar.gz")
-(def deb-zookeeper-package "3.4.9-3+deb8u1")
-(def deb-mesos-package "1.5.0-2.0.2")
-(def deb-marathon-package "1.6.322")
-
(def taskmanager-slots 3)
(defn flink-configuration
@@ -69,6 +61,20 @@
;; TODO: write log4j.properties properly
(c/exec (c/lit (str "sed -i'.bak' -e '/log4j.rootLogger=/ s/=.*/=DEBUG, file/' " install-dir "/conf/log4j.properties")))))
+(defn- file-name
+ [path]
+ (.getName (clojure.java.io/file path)))
+
+(defn upload-job-jar!
+ [job-jar]
+ (c/upload job-jar upload-dir)
+ (c/exec :mv (str upload-dir "/" (file-name job-jar)) install-dir))
+
+(defn upload-job-jars!
+ [job-jars]
+ (doseq [job-jar job-jars]
+ (upload-job-jar! job-jar)))
+
(defn install-flink!
[test node]
(let [url (:tarball test)]
@@ -76,8 +82,7 @@
(cu/install-archive! url install-dir)
(info "Enable S3 FS")
(c/exec (c/lit (str "ls " install-dir "/opt/flink-s3-fs-hadoop* | xargs -I {} mv {} " install-dir "/lib")))
- (c/upload (:job-jar test) upload-dir)
- (c/exec :mv (str upload-dir "/" (.getName (clojure.java.io/file (:job-jar test)))) install-dir)
+ (upload-job-jars! (->> test :test-spec :jobs (map :job-jar)))
(write-configuration! test node)))
(defn teardown-flink!
@@ -87,36 +92,16 @@
(meh (c/exec :rm :-rf install-dir))
(meh (c/exec :rm :-rf (c/lit "/tmp/.yarn-properties*"))))
-(defn get-log-files!
- []
- (if (cu/exists? log-dir) (cu/ls-full log-dir) []))
-
-(defn flink-db
- []
- (reify db/DB
- (setup! [_ test node]
- (c/su
- (install-flink! test node)))
-
- (teardown! [_ test node]
- (c/su
- (teardown-flink!)))
-
- db/LogFiles
- (log-files [_ test node]
- (concat
- (get-log-files!)))))
-
(defn combined-db
[dbs]
(reify db/DB
(setup! [_ test node]
(c/su
- (doall (map #(db/setup! % test node) dbs))))
+ (doseq [db dbs] (db/setup! db test node))))
(teardown! [_ test node]
(c/su
(try
- (doall (map #(db/teardown! % test node) dbs))
+ (doseq [db dbs] (db/teardown! db test node))
(finally (fu/stop-all-supervised-services!)))))
db/LogFiles
(log-files [_ test node]
@@ -125,6 +110,22 @@
(map #(db/log-files % test node))
(flatten)))))
+(defn flink-db
+ [db]
+ (let [flink-base-db (reify db/DB
+ (setup! [_ test node]
+ (c/su
+ (install-flink! test node)))
+
+ (teardown! [_ _ _]
+ (c/su
+ (teardown-flink!)))
+
+ db/LogFiles
+ (log-files [_ _ _]
+ (fu/find-files! log-dir)))]
+ (combined-db [flink-base-db db])))
+
(defn- sorted-nodes
[test]
(-> test :nodes sort))
@@ -145,7 +146,7 @@
[m]
(->>
(map #(str (name (first %)) "=" (second %)) m)
- (clojure.string/join " ")
+ (apply fu/join-space)
(#(str % " "))))
(defn- hadoop-env-vars
@@ -156,28 +157,37 @@
(defn exec-flink!
[cmd args]
(c/su
- (c/exec (c/lit (str
+ (c/exec (c/lit (fu/join-space
(hadoop-env-vars)
- install-dir "/bin/flink " cmd " " args)))))
+ (str install-dir "/bin/flink")
+ cmd
+ (apply fu/join-space args))))))
(defn flink-run-cli-args
"Returns the CLI args that should be passed to 'flink run'"
- [test]
+ [job-spec]
(concat
["-d"]
- (if (:main-class test)
- [(str "-c " (:main-class test))]
+ (if (:main-class job-spec)
+ [(str "-c " (:main-class job-spec))]
[])))
(defn submit-job!
([test] (submit-job! test []))
([test cli-args]
- (exec-flink! "run" (clojure.string/join
- " "
- (concat cli-args
- (flink-run-cli-args test)
- [(str install-dir "/" (last (str/split (:job-jar test) #"/")))
- (:job-args test)])))))
+ (doseq [{:keys [job-jar job-args] :as job-spec} (-> test :test-spec :jobs)]
+ (exec-flink! "run" (concat cli-args
+ (flink-run-cli-args job-spec)
+ [(str install-dir "/" (file-name job-jar))
+ job-args])))))
+
+(defn- submit-job-with-retry!
+ ([test] (submit-job-with-retry! test []))
+ ([test cli-args] (fu/retry
+ (partial submit-job! test cli-args)
+ :fallback (fn [e] (do
+ (fatal e "Could not submit job.")
+ (System/exit 1))))))
;;; Standalone
@@ -192,122 +202,130 @@
(select-nodes test (partial drop standalone-master-count)))
(defn- start-standalone-masters!
- [test node]
- (when (some #{node} (standalone-master-nodes test))
+ []
+ (let [jobmanager-script (str install-dir "/bin/jobmanager.sh")
+ jobmanager-log (str log-dir "/jobmanager.log")]
(fu/create-supervised-service!
"flink-master"
- (str "env " (hadoop-env-vars)
- install-dir "/bin/jobmanager.sh start-foreground "
- ">> " log-dir "/jobmanager.log"))))
+ (fu/join-space "env" (hadoop-env-vars) jobmanager-script "start-foreground" ">>" jobmanager-log))))
(defn- start-standalone-taskmanagers!
- [test node]
- (when (some #{node} (standalone-taskmanager-nodes test))
+ []
+ (let [taskmanager-script (str install-dir "/bin/taskmanager.sh")
+ taskmanager-log (str log-dir "/taskmanager.log")]
(fu/create-supervised-service!
"flink-taskmanager"
- (str "env " (hadoop-env-vars)
- install-dir "/bin/taskmanager.sh start-foreground "
- ">> " log-dir "/taskmanager.log"))))
-
-(defn- start-flink-db
- []
- (reify db/DB
- (setup! [_ test node]
- (c/su
- (start-standalone-masters! test node)
- (start-standalone-taskmanagers! test node)))
-
- (teardown! [_ test node]
- (c/su
- (when (some #{node} (standalone-master-nodes test))
- (fu/stop-supervised-service! "flink-master"))
- (when (some #{node} (standalone-taskmanager-nodes test))
- (fu/stop-supervised-service! "flink-taskmanager"))))))
+ (fu/join-space "env" (hadoop-env-vars) taskmanager-script "start-foreground" ">>" taskmanager-log))))
-(defn flink-standalone-db
+(defn start-flink-db
[]
- (let [zk (zk/db deb-zookeeper-package)
- hadoop (hadoop/db hadoop-dist-url)
- flink (flink-db)
- start-flink (start-flink-db)]
- (combined-db [hadoop zk flink start-flink])))
-
-(defn submit-job-from-first-node!
- [test]
- (c/on (first-node test)
- (submit-job! test)))
+ (flink-db
+ (reify db/DB
+ (setup! [_ test node]
+ (c/su
+ (when (some #{node} (standalone-master-nodes test))
+ (start-standalone-masters!))
+ (when (some #{node} (standalone-taskmanager-nodes test))
+ (start-standalone-taskmanagers!))
+ (when (= (first-node test) node)
+ (submit-job-with-retry! test))))
+
+ (teardown! [_ test node]
+ (c/su
+ (when (some #{node} (standalone-master-nodes test))
+ (fu/stop-supervised-service! "flink-master"))
+ (when (some #{node} (standalone-taskmanager-nodes test))
+ (fu/stop-supervised-service! "flink-taskmanager")))))))
;;; YARN
-(defn flink-yarn-db
+(defn- start-yarn-session-cmd
[]
- (let [zk (zk/db deb-zookeeper-package)
- hadoop (hadoop/db hadoop-dist-url)
- flink (flink-db)]
- (combined-db [hadoop zk flink])))
+ (fu/join-space (hadoop-env-vars)
+ (str install-dir "/bin/yarn-session.sh")
+ "-d"
+ "-jm 2048m"
+ "-tm 2048m"))
-(defn start-yarn-session!
- [test]
- (let [node (first-node test)]
- (c/on node
- (info "Starting YARN session from" node)
- (c/su
- (c/exec (c/lit (str (hadoop-env-vars)
- " " install-dir "/bin/yarn-session.sh -d -jm 2048m -tm 2048m")))
- (submit-job! test)))))
-
-(defn start-yarn-job!
+(defn- start-yarn-session!
+ []
+ (info "Starting YARN session")
+ (let [exec-start-yarn-session! #(c/su (c/exec (c/lit (start-yarn-session-cmd))))
+ log-failure! (fn [exception _] (info "Starting YARN session failed due to"
+ (.getMessage exception)
+ "Retrying..."))]
+ (fu/retry exec-start-yarn-session!
+ :delay 4000
+ :on-error log-failure!)))
+
+(defn yarn-session-db
+ []
+ (flink-db (reify db/DB
+ (setup! [_ test node]
+ (when (= node (first-node test))
+ (start-yarn-session!)
+ (submit-job! test)))
+ (teardown! [_ _ _]))))
+
+(defn- start-yarn-job!
[test]
- (c/on (first-node test)
- (c/su
- (submit-job! test ["-m yarn-cluster" "-yjm 2048m" "-ytm 2048m"]))))
-
-;;; Mesos
+ (c/su
+ (submit-job-with-retry! test ["-m yarn-cluster" "-yjm 2048m" "-ytm 2048m"])))
-(defn flink-mesos-db
+(defn yarn-job-db
[]
- (let [zk (zk/db deb-zookeeper-package)
- hadoop (hadoop/db hadoop-dist-url)
- mesos (mesos/db deb-mesos-package deb-marathon-package)
- flink (flink-db)]
- (combined-db [hadoop zk mesos flink])))
+ (flink-db (reify db/DB
+ (setup! [_ test node]
+ (when (= node (first-node test))
+ (start-yarn-job! test)))
+ (teardown! [_ _ _]))))
-(defn submit-job-with-retry!
- [test]
- (fu/retry
- (partial submit-job! test)
- :fallback (fn [e] (do
- (fatal e "Could not submit job.")
- (System/exit 1)))))
+;;; Mesos
-(defn mesos-appmaster-cmd
+(defn- mesos-appmaster-cmd
"Returns the command used by Marathon to start Flink's Mesos application master."
[test]
- (str (hadoop-env-vars)
- install-dir "/bin/mesos-appmaster.sh "
- "-Dmesos.master=" (zookeeper-uri
- test
- mesos/zk-namespace) " "
- "-Djobmanager.rpc.address=$(hostname -f) "
- "-Djobmanager.heap.mb=2048 "
- "-Djobmanager.rpc.port=6123 "
- "-Dmesos.resourcemanager.tasks.mem=2048 "
- "-Dtaskmanager.heap.mb=2048 "
- "-Dmesos.resourcemanager.tasks.cpus=1 "
- "-Drest.bind-address=$(hostname -f) "))
-
-(defn start-mesos-session!
+ (fu/join-space
+ (hadoop-env-vars)
+ (str install-dir "/bin/mesos-appmaster.sh")
+ (str "-Dmesos.master=" (zookeeper-uri test mesos/zk-namespace))
+ "-Djobmanager.rpc.address=$(hostname -f)"
+ "-Djobmanager.heap.mb=2048"
+ "-Djobmanager.rpc.port=6123"
+ "-Dmesos.resourcemanager.tasks.mem=2048"
+ "-Dtaskmanager.heap.mb=2048"
+ "-Dmesos.resourcemanager.tasks.cpus=1"
+ "-Drest.bind-address=$(hostname -f)"))
+
+(defn- start-mesos-session!
[test]
(c/su
- (let [r (fu/retry (fn []
- (http/post
- (str (mesos/marathon-base-url test) "/v2/apps")
- {:form-params {:id "flink"
- :cmd (mesos-appmaster-cmd test)
- :cpus 1.0
- :mem 2048
- :maxLaunchDelaySeconds 3}
- :content-type :json})))]
- (info "Submitted Flink Application via Marathon" r)
- (c/on (-> test :nodes sort first)
- (submit-job-with-retry! test)))))
+ (let [log-submission-failure! (fn [exception _]
+ (info "Submitting Flink Application via Marathon failed due to"
+ (.getMessage exception)
+ "Retrying..."))
+ submit-flink! (fn []
+ (http/post
+ (str (mesos/marathon-base-url test) "/v2/apps")
+ {:form-params {:id "flink"
+ :cmd (mesos-appmaster-cmd test)
+ :cpus 1.0
+ :mem 2048
+ :maxLaunchDelaySeconds 3}
+ :content-type :json}))
+ marathon-response (fu/retry submit-flink!
+ :on-retry log-submission-failure!
+ :delay 4000)]
+ (info "Submitted Flink Application via Marathon" marathon-response))))
+
+(defn flink-mesos-app-master
+ []
+ (flink-db
+ (reify
+ db/DB
+ (setup! [_ test node]
+ (when (= (first-node test) node)
+ (start-mesos-session! test)
+ (submit-job-with-retry! test)))
+
+ (teardown! [_ _ _]))))
diff --git a/flink-jepsen/src/jepsen/flink/flink.clj b/flink-jepsen/src/jepsen/flink/flink.clj
index c5d0d225932..a3698abd3ff 100644
--- a/flink-jepsen/src/jepsen/flink/flink.clj
+++ b/flink-jepsen/src/jepsen/flink/flink.clj
@@ -20,32 +20,45 @@
[jepsen
[cli :as cli]
[generator :as gen]
- [tests :as tests]]
+ [tests :as tests]
+ [zookeeper :as zk]]
[jepsen.os.debian :as debian]
- [jepsen.flink.client :refer :all]
- [jepsen.flink.checker :as flink-checker]
- [jepsen.flink.db :as fdb]
- [jepsen.flink.nemesis :as fn]))
+ [jepsen.flink
+ [client :refer :all]
+ [checker :as flink-checker]
+ [db :as fdb]
+ [generator :as fg]
+ [hadoop :as hadoop]
+ [kafka :as kafka]
+ [mesos :as mesos]
+ [nemesis :as fn]]))
-(def flink-test-config
- {:yarn-session {:db (fdb/flink-yarn-db)
- :deployment-strategy fdb/start-yarn-session!}
- :yarn-job {:db (fdb/flink-yarn-db)
- :deployment-strategy fdb/start-yarn-job!}
- :mesos-session {:db (fdb/flink-mesos-db)
- :deployment-strategy fdb/start-mesos-session!}
- :standalone-session {:db (fdb/flink-standalone-db)
- :deployment-strategy fdb/submit-job-from-first-node!}})
+(def default-flink-dist-url "https://archive.apache.org/dist/flink/flink-1.6.0/flink-1.6.0-bin-hadoop28-scala_2.11.tgz")
+(def hadoop-dist-url "https://archive.apache.org/dist/hadoop/common/hadoop-2.8.3/hadoop-2.8.3.tar.gz")
+(def kafka-dist-url "http://mirror.funkfreundelandshut.de/apache/kafka/2.0.1/kafka_2.11-2.0.1.tgz")
+(def deb-zookeeper-package "3.4.9-3+deb8u1")
+(def deb-mesos-package "1.5.0-2.0.2")
+(def deb-marathon-package "1.6.322")
-(def poll-job-running {:type :invoke, :f :job-running?, :value nil})
-(def cancel-job {:type :invoke, :f :cancel-job, :value nil})
-(def poll-job-running-loop (gen/seq (cycle [poll-job-running (gen/sleep 5)])))
+(def dbs
+ {:flink-yarn-job (fdb/yarn-job-db)
+ :flink-yarn-session (fdb/yarn-session-db)
+ :flink-standalone-session (fdb/start-flink-db)
+ :flink-mesos-session (fdb/flink-mesos-app-master)
+ :hadoop (hadoop/db hadoop-dist-url)
+ :kafka (kafka/db kafka-dist-url)
+ :mesos (mesos/db deb-mesos-package deb-marathon-package)
+ :zookeeper (zk/db deb-zookeeper-package)})
+
+(def poll-jobs-running {:type :invoke, :f :jobs-running?, :value nil})
+(def cancel-jobs {:type :invoke, :f :cancel-jobs, :value nil})
+(def poll-jobs-running-loop (gen/seq (cycle [poll-jobs-running (gen/sleep 5)])))
(defn default-client-gen
"Client generator that polls for the job running status."
[]
(->
- poll-job-running-loop
+ poll-jobs-running-loop
(gen/singlethreaded)))
(defn cancelling-client-gen
@@ -53,39 +66,39 @@
[]
(->
(gen/concat (gen/time-limit 15 (default-client-gen))
- (gen/once cancel-job)
+ (gen/once cancel-jobs)
(default-client-gen))
(gen/singlethreaded)))
(def client-gens
{:poll-job-running default-client-gen
- :cancel-job cancelling-client-gen})
+ :cancel-jobs cancelling-client-gen})
(defn flink-test
[opts]
(merge tests/noop-test
- (let [{:keys [db deployment-strategy]} (-> opts :deployment-mode flink-test-config)
+ (let [dbs (->> opts :test-spec :dbs (map dbs))
{:keys [job-running-healthy-threshold job-recovery-grace-period]} opts
client-gen ((:client-gen opts) client-gens)]
{:name "Apache Flink"
:os debian/os
- :db db
+ :db (fdb/combined-db dbs)
:nemesis (fn/nemesis)
:model (flink-checker/job-running-within-grace-period
job-running-healthy-threshold
job-recovery-grace-period)
:generator (let [stop (atom nil)]
- (->> (fn/stoppable-generator stop (client-gen))
+ (->> (fg/stoppable-generator stop (client-gen))
(gen/nemesis
- (fn/stop-generator stop
+ (fg/stop-generator stop
((fn/nemesis-generator-factories (:nemesis-gen opts)) opts)
job-running-healthy-threshold
job-recovery-grace-period))))
- :client (create-client deployment-strategy)
+ :client (create-client)
:checker (flink-checker/job-running-checker)})
(assoc opts :concurrency 1)))
-(defn keys-as-allowed-values-help-text
+(defn- keys->allowed-values-help-text
"Takes a map and returns a string explaining which values are allowed.
This is a CLI helper function."
[m]
@@ -94,34 +107,34 @@
(clojure.string/join ", ")
(str "Must be one of: ")))
+(defn read-test-spec
+ [path]
+ (clojure.edn/read-string (slurp path)))
+
(defn -main
[& args]
(cli/run!
(merge
(cli/single-test-cmd
{:test-fn flink-test
- :tarball fdb/default-flink-dist-url
- :opt-spec [[nil "--ha-storage-dir DIR" "high-availability.storageDir"]
- [nil "--job-jar JAR" "Path to the job jar"]
- [nil "--job-args ARGS" "CLI arguments for the flink job"]
- [nil "--main-class CLASS" "Job main class"]
+ :tarball default-flink-dist-url
+ :opt-spec [[nil "--test-spec FILE" "Path to a test specification (.edn)"
+ :parse-fn read-test-spec
+ :validate [#(->> % :dbs (map dbs) (every? (complement nil?)))
+ (str "Invalid :dbs specification. " (keys->allowed-values-help-text dbs))]]
+ [nil "--ha-storage-dir DIR" "high-availability.storageDir"]
[nil "--nemesis-gen GEN" (str "Which nemesis should be used?"
- (keys-as-allowed-values-help-text fn/nemesis-generator-factories))
+ (keys->allowed-values-help-text fn/nemesis-generator-factories))
:parse-fn keyword
:default :kill-task-managers
- :validate [#(fn/nemesis-generator-factories (keyword %))
- (keys-as-allowed-values-help-text fn/nemesis-generator-factories)]]
+ :validate [#(fn/nemesis-generator-factories %)
+ (keys->allowed-values-help-text fn/nemesis-generator-factories)]]
[nil "--client-gen GEN" (str "Which client should be used?"
- (keys-as-allowed-values-help-text client-gens))
+ (keys->allowed-values-help-text client-gens))
:parse-fn keyword
:default :poll-job-running
- :validate [#(client-gens (keyword %))
- (keys-as-allowed-values-help-text client-gens)]]
- [nil "--deployment-mode MODE" (keys-as-allowed-values-help-text flink-test-config)
- :parse-fn keyword
- :default :yarn-session
- :validate [#(flink-test-config (keyword %))
- (keys-as-allowed-values-help-text flink-test-config)]]
+ :validate [#(client-gens %)
+ (keys->allowed-values-help-text client-gens)]]
[nil "--job-running-healthy-threshold TIMES" "Number of consecutive times the job must be running to be considered healthy."
:default 5
:parse-fn #(Long/parseLong %)
diff --git a/flink-jepsen/src/jepsen/flink/generator.clj b/flink-jepsen/src/jepsen/flink/generator.clj
index af928c4d826..053f14db61d 100644
--- a/flink-jepsen/src/jepsen/flink/generator.clj
+++ b/flink-jepsen/src/jepsen/flink/generator.clj
@@ -16,7 +16,8 @@
(ns jepsen.flink.generator
(:require [jepsen.util :as util]
- [jepsen.generator :as gen]))
+ [jepsen.generator :as gen]
+ [jepsen.flink.checker :as flink-checker]))
(gen/defgenerator TimeLimitGen
[dt source deadline-atom]
@@ -37,3 +38,61 @@
(defn time-limit
[dt source]
(TimeLimitGen. dt source (atom nil)))
+
+(defn stoppable-generator
+ "Given an atom and a source generator, returns a generator that stops emitting operations from
+ the source if the atom is set to true."
+ [stop source]
+ (reify gen/Generator
+ (op [_ test process]
+ (if @stop
+ nil
+ (gen/op source test process)))))
+
+(defn- take-last-with-default
+ [n default coll]
+ (->>
+ (cycle [default])
+ (concat (reverse coll))
+ (take n)
+ (reverse)))
+
+(defn- inc-by-factor
+ [n factor]
+ (assert (>= factor 1))
+ (int (* n factor)))
+
+(defn stop-generator
+ "Returns a generator that emits operations from a given source generator. If the source is
+ exhausted and either job-recovery-grace-period has passed or the job has been running
+ job-running-healthy-threshold times consecutively, the stop atom is set to true."
+ [stop source job-running-healthy-threshold job-recovery-grace-period]
+ (gen/concat source
+ (let [t (atom nil)]
+ (reify gen/Generator
+ (op [_ test process]
+ (when (nil? @t)
+ (compare-and-set! t nil (util/relative-time-nanos)))
+ (let [history (->>
+ (:active-histories test)
+ deref
+ first
+ deref)
+ job-running-history (->>
+ history
+ (filter (fn [op] (>= (- (:time op) @t) 0)))
+ (flink-checker/all-jobs-running?-history)
+ (take-last-with-default job-running-healthy-threshold false))]
+ (if (or
+ (every? true? job-running-history)
+ (> (util/relative-time-nanos) (+ @t
+ (util/secs->nanos
+ (inc-by-factor
+ job-recovery-grace-period
+ 1.1)))))
+ (do
+ (reset! stop true)
+ nil)
+ (do
+ (Thread/sleep 1000)
+ (recur test process)))))))))
diff --git a/flink-jepsen/src/jepsen/flink/kafka.clj b/flink-jepsen/src/jepsen/flink/kafka.clj
new file mode 100644
index 00000000000..d32114b8fa5
--- /dev/null
+++ b/flink-jepsen/src/jepsen/flink/kafka.clj
@@ -0,0 +1,99 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+(ns jepsen.flink.kafka
+ (:require [clojure.tools.logging :refer :all]
+ [jepsen
+ [db :as db]
+ [control :as c]
+ [util :refer [meh]]]
+ [jepsen.control.util :as cu]
+ [jepsen.flink.zookeeper :as fzk]
+ [jepsen.flink.utils :as fu]))
+
+(def install-dir "/opt/kafka")
+(def application-log-dir "/opt/kafka/logs")
+(def log-dirs "/opt/kafka/kafka-logs")
+(def server-properties (str install-dir "/config/server.properties"))
+(def start-script (str install-dir "/bin/kafka-server-start.sh"))
+(def topic-script (str install-dir "/bin/kafka-topics.sh"))
+(def stop-script (str install-dir "/bin/kafka-server-stop.sh"))
+
+(defn- broker-id
+ [nodes node]
+ (.indexOf (sort nodes) node))
+
+(defn- override-property
+ [name value]
+ (str "--override " name "=" value))
+
+(defn- start-server-command
+ [{:keys [nodes] :as test} node]
+ (fu/join-space
+ start-script
+ "-daemon"
+ server-properties
+ (override-property "zookeeper.connect" (fzk/zookeeper-quorum test))
+ (override-property "broker.id" (broker-id nodes node))
+ (override-property "log.dirs" log-dirs)
+ (override-property "retention.ms" "1800000")))
+
+(defn- start-server!
+ [test node]
+ (c/exec (c/lit (start-server-command test node))))
+
+(defn- stop-server!
+ []
+ (info "Stopping Kafka")
+ (cu/grepkill! "kafka"))
+
+(defn- create-topic-command
+ [{:keys [nodes] :as test}]
+ (fu/join-space
+ topic-script
+ "--create"
+ "--topic kafka-test-topic"
+ (str "--partitions " (count nodes))
+ "--replication-factor 1"
+ "--zookeeper"
+ (fzk/zookeeper-quorum test)))
+
+(defn- create-topic!
+ [test]
+ (info "Attempting to create Kafka topic")
+ (fu/retry (fn [] (c/exec (c/lit (create-topic-command test))))))
+
+(defn- delete-kafka!
+ []
+ (info "Deleting Kafka distribution and logs")
+ (c/exec :rm :-rf install-dir))
+
+(defn db
+ [kafka-dist-url]
+ (reify db/DB
+ (setup! [_ test node]
+ (c/su
+ (cu/install-archive! kafka-dist-url install-dir)
+ (start-server! test node)
+ (when (zero? (broker-id (:nodes test) node))
+ (create-topic! test))))
+ (teardown! [_ _ _]
+ (c/su
+ (stop-server!)
+ (delete-kafka!)))
+ db/LogFiles
+ (log-files [_ _ _]
+ (fu/find-files! application-log-dir))))
diff --git a/flink-jepsen/src/jepsen/flink/nemesis.clj b/flink-jepsen/src/jepsen/flink/nemesis.clj
index 5335bba874c..37e842189af 100644
--- a/flink-jepsen/src/jepsen/flink/nemesis.clj
+++ b/flink-jepsen/src/jepsen/flink/nemesis.clj
@@ -23,7 +23,6 @@
[util :as ju]]
[jepsen.control.util :as cu]
[jepsen.flink.client :refer :all]
- [jepsen.flink.checker :as flink-checker]
[jepsen.flink.generator :as fgen]
[jepsen.flink.hadoop :as fh]
[jepsen.flink.zookeeper :refer :all]))
@@ -70,59 +69,6 @@
;;; Generators
-(defn stoppable-generator
- [stop source]
- (reify gen/Generator
- (op [gen test process]
- (if @stop
- nil
- (gen/op source test process)))))
-
-(defn take-last-with-default
- [n default coll]
- (->>
- (cycle [default])
- (concat (reverse coll))
- (take n)
- (reverse)))
-
-(defn- inc-by-factor
- [n factor]
- (assert (>= factor 1))
- (int (* n factor)))
-
-(defn stop-generator
- [stop source job-running-healthy-threshold job-recovery-grace-period]
- (gen/concat source
- (let [t (atom nil)]
- (reify gen/Generator
- (op [_ test process]
- (when (nil? @t)
- (compare-and-set! t nil (ju/relative-time-nanos)))
- (let [history (->>
- (:active-histories test)
- deref
- first
- deref)
- job-running-history (->>
- history
- (filter (fn [op] (>= (- (:time op) @t) 0)))
- (flink-checker/get-job-running-history)
- (take-last-with-default job-running-healthy-threshold false))]
- (if (or
- (every? true? job-running-history)
- (> (ju/relative-time-nanos) (+ @t
- (ju/secs->nanos
- (inc-by-factor
- job-recovery-grace-period
- 1.1)))))
- (do
- (reset! stop true)
- nil)
- (do
- (Thread/sleep 1000)
- (recur test process)))))))))
-
(defn kill-taskmanagers-gen
[time-limit dt op]
(fgen/time-limit time-limit (gen/stagger dt (gen/seq (cycle [{:type :info, :f op}])))))
diff --git a/flink-jepsen/src/jepsen/flink/utils.clj b/flink-jepsen/src/jepsen/flink/utils.clj
index 1aa53efe7ae..7ccae12d661 100644
--- a/flink-jepsen/src/jepsen/flink/utils.clj
+++ b/flink-jepsen/src/jepsen/flink/utils.clj
@@ -35,7 +35,7 @@
:or {on-retry (fn [exception attempt] (warn "Retryable operation failed:"
(.getMessage exception)))
success identity
- fallback :default
+ fallback #(throw %)
retries 10
delay 2000}
:as keys}]
@@ -51,6 +51,10 @@
(recur op (assoc keys :retries (dec retries))))
(success r)))))
+(defn join-space
+ [& tokens]
+ (clojure.string/join " " tokens))
+
(defn find-files!
"Lists files recursively given a directory. If the directory does not exist, an empty collection
is returned."
diff --git a/flink-jepsen/test/jepsen/flink/checker_test.clj b/flink-jepsen/test/jepsen/flink/checker_test.clj
index c27d751e69e..d1ade1645b0 100644
--- a/flink-jepsen/test/jepsen/flink/checker_test.clj
+++ b/flink-jepsen/test/jepsen/flink/checker_test.clj
@@ -20,96 +20,99 @@
[checker :as checker]]
[jepsen.flink.checker :refer :all]))
-(deftest get-job-running-history-test
+(deftest all-jobs-running?-history-test
(let [history [{:type :info, :f :kill-random-subset-task-managers, :process :nemesis, :time 121898381144, :value '("172.31.33.170")}
- {:type :invoke, :f :job-running?, :value nil, :process 0, :time 127443701575}
- {:type :ok, :f :job-running?, :value false, :process 0, :time 127453553462}
- {:type :invoke, :f :job-running?, :value nil, :process 0, :time 127453553463}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 127453553464}
- {:type :info, :f :job-running?, :value nil, :process 0, :time 127453553465}]]
- (is (= (get-job-running-history history) [false true false]))))
+ {:type :invoke, :f :jobs-running?, :value nil, :process 0, :time 127443701575}
+ {:type :ok, :f :jobs-running?, :value {"3886d6b547969c3f15c53896bb496b8f" false}, :process 0, :time 127453553462}
+ {:type :invoke, :f :jobs-running?, :value nil, :process 0, :time 127453553463}
+ {:type :ok, :f :jobs-running?, :value {"3886d6b547969c3f15c53896bb496b8f" true}, :process 0, :time 127453553464}
+ {:type :info, :f :jobs-running?, :value nil, :process 0, :time 127453553465}]]
+ (is (= [false true false] (all-jobs-running?-history history)))))
(deftest job-running-checker-test
(let [checker (job-running-checker)
test {}
model (job-running-within-grace-period 3 60 10)
opts {}
- check (fn [history] (checker/check checker test model history opts))]
+ check (fn [history] (checker/check checker test model history opts))
+ job-running-value {"3886d6b547969c3f15c53896bb496b8f" true}
+ job-not-running-value {"3886d6b547969c3f15c53896bb496b8f" false}]
(testing "Model should be inconsistent if job is not running after grace period."
(let [result (check
[{:type :info, :f :kill-task-managers, :process :nemesis, :time 0, :value ["172.31.32.48"]}
- {:type :ok, :f :job-running?, :value false, :process 0, :time 60000000001}])]
+ {:type :ok, :f :jobs-running?, :value job-not-running-value, :process 0, :time 60000000001}])]
(is (= false (:valid? result)))
- (is (= "Job is not running." (-> result :final-model :msg)))))
+ (is (= "Job is not running." (-> result :final-models first :msg)))))
(testing "Model should be consistent if job is running after grace period."
(is (= true (:valid? (check
[{:type :info, :f :kill-task-managers, :process :nemesis, :time 0, :value ["172.31.32.48"]}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 60000000001}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 60000000002}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 60000000003}])))))
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 60000000001}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 60000000002}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 60000000003}])))))
(testing "Should tolerate non-running job during failures."
(is (= true (:valid? (check
[{:type :info, :f :partition-start, :process :nemesis, :time -1}
{:type :info, :f :partition-start, :process :nemesis, :time 0, :value "Cut off [...]"}
- {:type :ok, :f :job-running?, :value false, :process 0, :time 60000000001}
+ {:type :ok, :f :jobs-running?, :value job-not-running-value, :process 0, :time 60000000001}
{:type :info, :f :partition-stop, :process :nemesis, :time 60000000002}
{:type :info, :f :partition-stop, :process :nemesis, :time 60000000003, :value "fully connected"}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 60000000004}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 60000000005}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 60000000006}])))))
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 60000000004}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 60000000005}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 60000000006}])))))
(testing "Should not tolerate non-running job without a cause."
(let [result (check
- [{:type :ok, :f :job-running?, :value true, :process 0, :time 0}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 1}
- {:type :ok, :f :job-running?, :value false, :process 0, :time 60000000001}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 60000000002}])]
+ [{:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 0}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 1}
+ {:type :ok, :f :jobs-running?, :value job-not-running-value, :process 0, :time 60000000001}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 60000000002}])]
(is (= false (:valid? result)))
- (is (= "Job is not running." (-> result :final-model :msg)))))
+ (is (= "Job is not running." (-> result :final-models first :msg)))))
(testing "Model should be inconsistent if job submission was unsuccessful."
- (let [result (check [{:type :invoke, :f :job-running?, :value nil, :process 0, :time 239150413307}
- {:type :info, :f :job-running?, :value nil, :process 0, :time 239150751938, :error "indeterminate: Assert failed: job-id"}])]
+ (let [result (check [{:type :invoke, :f :jobs-running?, :value nil, :process 0, :time 239150413307}
+ {:type :info, :f :jobs-running?, :value nil, :process 0, :time 239150751938, :error "indeterminate: Assert failed: job-id"}])]
(is (= false (:valid? result)))))
(testing "Model should be inconsistent if the job status cannot be polled, i.e., if the cluster is unavailable."
- (let [result (check [{:type :fail, :f :job-running?, :value nil, :process 0, :time 0 :error "Error"}
- {:type :fail, :f :job-running?, :value nil, :process 0, :time 60000000001 :error "Error"}
- {:type :fail, :f :job-running?, :value nil, :process 0, :time 60000000002 :error "Error"}])]
+ (let [result (check [{:type :fail, :f :jobs-running?, :value job-running-value, :process 0, :time 0 :error "Error"}
+ {:type :fail, :f :jobs-running?, :value nil, :process 0, :time 60000000001 :error "Error"}
+ {:type :fail, :f :jobs-running?, :value nil, :process 0, :time 60000000002 :error "Error"}])]
(is (= false (:valid? result)))
- (is (= "Cluster is not running." (-> result :final-model :msg)))))
+ (is (= "Cluster is not running." (-> result :final-models first :msg)))))
(testing "Should tolerate non-running job after cancellation."
- (is (= true (:valid? (check [{:type :invoke, :f :cancel-job, :value nil, :process 0, :time 0}
- {:type :ok, :f :cancel-job, :value true, :process 0, :time 1}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 2}
- {:type :ok, :f :job-running?, :value false, :process 0, :time 3}])))))
+ (is (= true (:valid? (check [{:type :invoke, :f :cancel-jobs, :value nil, :process 0, :time 0}
+ {:type :ok, :f :cancel-jobs, :value nil, :process 0, :time 1}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 2}
+ {:type :ok, :f :jobs-running?, :value job-not-running-value, :process 0, :time 3}])))))
(testing "Model should be inconsistent if job is running after cancellation."
- (let [result (check [{:type :invoke, :f :cancel-job, :value nil, :process 0, :time 0}
- {:type :ok, :f :cancel-job, :value true, :process 0, :time 1}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 10000000002}])]
+ (let [result (check [{:type :invoke, :f :cancel-jobs, :value nil, :process 0, :time 0}
+ {:type :ok, :f :cancel-jobs, :value true, :process 0, :time 1}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 10000000002}])]
(is (= false (:valid? result)))
- (is (= "Job is running after cancellation." (-> result :final-model :msg)))))
+ (is (= "Job is running after cancellation." (-> result :final-models first :msg)))))
(testing "Model should be inconsistent if Flink cluster is not available at the end."
- (let [result (check [{:type :ok, :f :job-running?, :value true, :process 0, :time 0}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 1}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 2}
- {:type :fail, :f :job-running?, :value nil, :process 0, :time 60000000003, :error "Error"}])]
+ (let [result (check [{:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 0}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 1}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 2}
+ {:type :fail, :f :jobs-running?, :value nil, :process 0, :time 60000000003, :error "Error"}])]
(is (= false (:valid? result)))
- (is (= "Cluster is not running." (-> result :final-model :msg)))))
+ (is (= "Cluster is not running." (-> result :final-models first :msg)))))
(testing "Model should be inconsistent if Flink cluster is not available after job cancellation."
- (let [result (check [{:type :ok, :f :job-running?, :value true, :process 0, :time 0}
- {:type :invoke, :f :cancel-job, :value nil, :process 0, :time 1}
- {:type :ok, :f :cancel-job, :value true, :process 0, :time 2}
- {:type :fail, :f :job-running?, :value nil, :process 0, :time 60000000001, :error "Error"}])]
+ (let [result (check [{:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 0}
+ {:type :invoke, :f :cancel-jobs, :value nil, :process 0, :time 1}
+ {:type :ok, :f :cancel-jobs, :value job-running-value, :process 0, :time 2}
+ {:type :fail, :f :jobs-running?, :value nil, :process 0, :time 60000000001, :error "Error"}])]
(is (= false (:valid? result)))
- (is (= "Cluster is not running." (-> result :final-model :msg)))))
+ (is (= "Cluster is not running." (-> result :final-models first :msg)))))
(testing "Should throw AssertionError if job cancelling operation failed."
(is (thrown-with-msg? AssertionError
#":cancel-job must not fail"
- (check [{:type :fail, :f :cancel-job, :value nil, :process 0, :time 0}]))))
+ (check [{:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 0}
+ {:type :fail, :f :cancel-jobs, :value nil, :process 0, :time 1}]))))
(testing "Should tolerate non-running job if grace period has not passed."
- (is (= true (:valid? (check [{:type :invoke, :f :job-running?, :value nil, :process 0, :time 0}
- {:type :ok, :f :job-running?, :value false, :process 0, :time 1}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 2}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 3}
- {:type :ok, :f :job-running?, :value true, :process 0, :time 4}])))))))
+ (is (= true (:valid? (check [{:type :invoke, :f :jobs-running?, :value nil, :process 0, :time 0}
+ {:type :ok, :f :jobs-running?, :value job-not-running-value, :process 0, :time 1}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 2}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 3}
+ {:type :ok, :f :jobs-running?, :value job-running-value, :process 0, :time 4}])))))))
(deftest safe-inc-test
(is (= (safe-inc nil) 1))
diff --git a/flink-jepsen/test/jepsen/flink/utils_test.clj b/flink-jepsen/test/jepsen/flink/utils_test.clj
index 607f90d1170..17d5e5f4371 100644
--- a/flink-jepsen/test/jepsen/flink/utils_test.clj
+++ b/flink-jepsen/test/jepsen/flink/utils_test.clj
@@ -29,7 +29,7 @@
(testing "Exhaust all attempts."
(let [failing-always (fn [] (throw (Exception. "Expected")))]
- (is (nil? (retry failing-always :retries 1 :delay 0)))))
+ (is (nil? (retry failing-always :retries 1 :delay 0 :fallback :nil)))))
(testing "Propagate exception."
(let [failing-always (fn [] (throw (Exception. "Expected")))]
diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java
index bc8ef25a5a3..51c4ad0c1e1 100644
--- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java
+++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java
@@ -75,7 +75,8 @@
MigrationVersion.v1_3,
MigrationVersion.v1_4,
MigrationVersion.v1_5,
- MigrationVersion.v1_6);
+ MigrationVersion.v1_6,
+ MigrationVersion.v1_7);
}
public CEPMigrationTest(MigrationVersion migrateVersion) {
diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink1.7-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink1.7-snapshot
new file mode 100644
index 00000000000..f7466eac9b4
Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink1.7-snapshot differ
diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-conditions-flink1.7-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-conditions-flink1.7-snapshot
new file mode 100644
index 00000000000..05178bd5172
Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-conditions-flink1.7-snapshot differ
diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink1.7-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink1.7-snapshot
new file mode 100644
index 00000000000..ddaffa978ad
Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink1.7-snapshot differ
diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink1.7-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink1.7-snapshot
new file mode 100644
index 00000000000..3c09bb20eb7
Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink1.7-snapshot differ
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
new file mode 100644
index 00000000000..f318166e145
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -0,0 +1,6340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.validate;
+
+import org.apache.calcite.config.NullCollation;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.function.Function2;
+import org.apache.calcite.linq4j.function.Functions;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.prepare.Prepare;
+import org.apache.calcite.rel.type.DynamicRecordType;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rel.type.RelRecordType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexPatternFieldRef;
+import org.apache.calcite.rex.RexVisitor;
+import org.apache.calcite.runtime.CalciteContextException;
+import org.apache.calcite.runtime.CalciteException;
+import org.apache.calcite.runtime.Feature;
+import org.apache.calcite.runtime.Resources;
+import org.apache.calcite.schema.ColumnStrategy;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.impl.ModifiableViewTable;
+import org.apache.calcite.sql.JoinConditionType;
+import org.apache.calcite.sql.JoinType;
+import org.apache.calcite.sql.SqlAccessEnum;
+import org.apache.calcite.sql.SqlAccessType;
+import org.apache.calcite.sql.SqlBasicCall;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlDataTypeSpec;
+import org.apache.calcite.sql.SqlDelete;
+import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplain;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlInsert;
+import org.apache.calcite.sql.SqlIntervalLiteral;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlJoin;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlMatchRecognize;
+import org.apache.calcite.sql.SqlMerge;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlOrderBy;
+import org.apache.calcite.sql.SqlSampleSpec;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlSelectKeyword;
+import org.apache.calcite.sql.SqlSyntax;
+import org.apache.calcite.sql.SqlUnresolvedFunction;
+import org.apache.calcite.sql.SqlUpdate;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlWindow;
+import org.apache.calcite.sql.SqlWith;
+import org.apache.calcite.sql.SqlWithItem;
+import org.apache.calcite.sql.fun.SqlCase;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.AssignableOperandTypeChecker;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlOperandTypeInference;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.calcite.sql.util.SqlShuttle;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.calcite.sql2rel.InitializerContext;
+import org.apache.calcite.util.BitString;
+import org.apache.calcite.util.Bug;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.ImmutableNullableList;
+import org.apache.calcite.util.Litmus;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Static;
+import org.apache.calcite.util.Util;
+import org.apache.calcite.util.trace.CalciteTrace;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
+
+/*
+ * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT UNTIL CALCITE-2707 IS FIXED.
+ * (Added lines: 5937-5943)
+ */
+
+/**
+ * Default implementation of {@link SqlValidator}.
+ */
+public class SqlValidatorImpl implements SqlValidatorWithHints {
+ //~ Static fields/initializers ---------------------------------------------
+
+ public static final Logger TRACER = CalciteTrace.PARSER_LOGGER;
+
+ /**
+ * Alias generated for the source table when rewriting UPDATE to MERGE.
+ */
+ public static final String UPDATE_SRC_ALIAS = "SYS$SRC";
+
+ /**
+ * Alias generated for the target table when rewriting UPDATE to MERGE if no
+ * alias was specified by the user.
+ */
+ public static final String UPDATE_TGT_ALIAS = "SYS$TGT";
+
+ /**
+ * Alias prefix generated for source columns when rewriting UPDATE to MERGE.
+ */
+ public static final String UPDATE_ANON_PREFIX = "SYS$ANON";
+
+ //~ Instance fields --------------------------------------------------------
+
+ private final SqlOperatorTable opTab;
+ final SqlValidatorCatalogReader catalogReader;
+
+ /**
+ * Maps ParsePosition strings to the {@link SqlIdentifier} identifier
+ * objects at these positions
+ */
+ protected final Map<String, IdInfo> idPositions = new HashMap<>();
+
+ /**
+ * Maps {@link SqlNode query node} objects to the {@link SqlValidatorScope}
+ * scope created from them.
+ */
+ protected final Map<SqlNode, SqlValidatorScope> scopes =
+ new IdentityHashMap<>();
+
+ /**
+ * Maps a {@link SqlSelect} node to the scope used by its WHERE and HAVING
+ * clauses.
+ */
+ private final Map<SqlSelect, SqlValidatorScope> whereScopes =
+ new IdentityHashMap<>();
+
+ /**
+ * Maps a {@link SqlSelect} node to the scope used by its GROUP BY clause.
+ */
+ private final Map<SqlSelect, SqlValidatorScope> groupByScopes =
+ new IdentityHashMap<>();
+
+ /**
+ * Maps a {@link SqlSelect} node to the scope used by its SELECT and HAVING
+ * clauses.
+ */
+ private final Map<SqlSelect, SqlValidatorScope> selectScopes =
+ new IdentityHashMap<>();
+
+ /**
+ * Maps a {@link SqlSelect} node to the scope used by its ORDER BY clause.
+ */
+ private final Map<SqlSelect, SqlValidatorScope> orderScopes =
+ new IdentityHashMap<>();
+
+ /**
+ * Maps a {@link SqlSelect} node that is the argument to a CURSOR
+ * constructor to the scope of the result of that select node
+ */
+ private final Map<SqlSelect, SqlValidatorScope> cursorScopes =
+ new IdentityHashMap<>();
+
+ /**
+ * The name-resolution scope of a LATERAL TABLE clause.
+ */
+ private TableScope tableScope = null;
+
+ /**
+ * Maps a {@link SqlNode node} to the
+ * {@link SqlValidatorNamespace namespace} which describes what columns they
+ * contain.
+ */
+ protected final Map<SqlNode, SqlValidatorNamespace> namespaces =
+ new IdentityHashMap<>();
+
+ /**
+ * Set of select expressions used as cursor definitions. In standard SQL,
+ * only the top-level SELECT is a cursor; Calcite extends this with
+ * cursors as inputs to table functions.
+ */
+ private final Set<SqlNode> cursorSet = Sets.newIdentityHashSet();
+
+ /**
+ * Stack of objects that maintain information about function calls. A stack
+ * is needed to handle nested function calls. The function call currently
+ * being validated is at the top of the stack.
+ */
+ protected final Deque<FunctionParamInfo> functionCallStack =
+ new ArrayDeque<>();
+
+ private int nextGeneratedId;
+ protected final RelDataTypeFactory typeFactory;
+
+ /** The type of dynamic parameters until a type is imposed on them. */
+ protected final RelDataType unknownType;
+ private final RelDataType booleanType;
+
+ /**
+ * Map of derived RelDataType for each node. This is an IdentityHashMap
+ * since in some cases (such as null literals) we need to discriminate by
+ * instance.
+ */
+ private final Map<SqlNode, RelDataType> nodeToTypeMap =
+ new IdentityHashMap<>();
+ private final AggFinder aggFinder;
+ private final AggFinder aggOrOverFinder;
+ private final AggFinder aggOrOverOrGroupFinder;
+ private final AggFinder groupFinder;
+ private final AggFinder overFinder;
+ private final SqlConformance conformance;
+ private final Map<SqlNode, SqlNode> originalExprs = new HashMap<>();
+
+ private SqlNode top;
+
+ // REVIEW jvs 30-June-2006: subclasses may override shouldExpandIdentifiers
+ // in a way that ignores this; we should probably get rid of the protected
+ // method and always use this variable (or better, move preferences like
+ // this to a separate "parameter" class)
+ protected boolean expandIdentifiers;
+
+ protected boolean expandColumnReferences;
+
+ private boolean rewriteCalls;
+
+ private NullCollation nullCollation = NullCollation.HIGH;
+
+ // TODO jvs 11-Dec-2008: make this local to performUnconditionalRewrites
+ // if it's OK to expand the signature of that method.
+ private boolean validatingSqlMerge;
+
+ private boolean inWindow; // Allow nested aggregates
+
+ private final SqlValidatorImpl.ValidationErrorFunction validationErrorFunction =
+ new SqlValidatorImpl.ValidationErrorFunction();
+
+ //~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a validator.
+ *
+ * @param opTab Operator table
+ * @param catalogReader Catalog reader
+ * @param typeFactory Type factory
+ * @param conformance Compatibility mode
+ */
+ protected SqlValidatorImpl(
+ SqlOperatorTable opTab,
+ SqlValidatorCatalogReader catalogReader,
+ RelDataTypeFactory typeFactory,
+ SqlConformance conformance) {
+ this.opTab = Objects.requireNonNull(opTab);
+ this.catalogReader = Objects.requireNonNull(catalogReader);
+ this.typeFactory = Objects.requireNonNull(typeFactory);
+ this.conformance = Objects.requireNonNull(conformance);
+
+ unknownType = typeFactory.createUnknownType();
+ booleanType = typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+
+ rewriteCalls = true;
+ expandColumnReferences = true;
+ aggFinder = new AggFinder(opTab, false, true, false, null);
+ aggOrOverFinder = new AggFinder(opTab, true, true, false, null);
+ overFinder = new AggFinder(opTab, true, false, false, aggOrOverFinder);
+ groupFinder = new AggFinder(opTab, false, false, true, null);
+ aggOrOverOrGroupFinder = new AggFinder(opTab, true, true, true, null);
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ public SqlConformance getConformance() {
+ return conformance;
+ }
+
+ public SqlValidatorCatalogReader getCatalogReader() {
+ return catalogReader;
+ }
+
+ public SqlOperatorTable getOperatorTable() {
+ return opTab;
+ }
+
+ public RelDataTypeFactory getTypeFactory() {
+ return typeFactory;
+ }
+
+ public RelDataType getUnknownType() {
+ return unknownType;
+ }
+
+ public SqlNodeList expandStar(
+ SqlNodeList selectList,
+ SqlSelect select,
+ boolean includeSystemVars) {
+ final List<SqlNode> list = new ArrayList<>();
+ final List<Map.Entry<String, RelDataType>> types = new ArrayList<>();
+ for (int i = 0; i < selectList.size(); i++) {
+ final SqlNode selectItem = selectList.get(i);
+ final RelDataType originalType = getValidatedNodeTypeIfKnown(selectItem);
+ expandSelectItem(
+ selectItem,
+ select,
+ Util.first(originalType, unknownType),
+ list,
+ catalogReader.nameMatcher().createSet(),
+ types,
+ includeSystemVars);
+ }
+ getRawSelectScope(select).setExpandedSelectList(list);
+ return new SqlNodeList(list, SqlParserPos.ZERO);
+ }
+
+ // implement SqlValidator
+ public void declareCursor(SqlSelect select, SqlValidatorScope parentScope) {
+ cursorSet.add(select);
+
+ // add the cursor to a map that maps the cursor to its select based on
+ // the position of the cursor relative to other cursors in that call
+ FunctionParamInfo funcParamInfo = functionCallStack.peek();
+ Map<Integer, SqlSelect> cursorMap = funcParamInfo.cursorPosToSelectMap;
+ int numCursors = cursorMap.size();
+ cursorMap.put(numCursors, select);
+
+ // create a namespace associated with the result of the select
+ // that is the argument to the cursor constructor; register it
+ // with a scope corresponding to the cursor
+ SelectScope cursorScope = new SelectScope(parentScope, null, select);
+ cursorScopes.put(select, cursorScope);
+ final SelectNamespace selectNs = createSelectNamespace(select, select);
+ String alias = deriveAlias(select, nextGeneratedId++);
+ registerNamespace(cursorScope, alias, selectNs, false);
+ }
+
+ // implement SqlValidator
+ public void pushFunctionCall() {
+ FunctionParamInfo funcInfo = new FunctionParamInfo();
+ functionCallStack.push(funcInfo);
+ }
+
+ // implement SqlValidator
+ public void popFunctionCall() {
+ functionCallStack.pop();
+ }
+
+ // implement SqlValidator
+ public String getParentCursor(String columnListParamName) {
+ FunctionParamInfo funcParamInfo = functionCallStack.peek();
+ Map<String, String> parentCursorMap =
+ funcParamInfo.columnListParamToParentCursorMap;
+ return parentCursorMap.get(columnListParamName);
+ }
+
+ /**
+ * If <code>selectItem</code> is "*" or "TABLE.*", expands it and returns
+ * true; otherwise writes the unexpanded item.
+ *
+ * @param selectItem Select-list item
+ * @param select Containing select clause
+ * @param selectItems List that expanded items are written to
+ * @param aliases Set of aliases
+ * @param fields List of field names and types, in alias order
+ * @param includeSystemVars If true include system vars in lists
+ * @return Whether the node was expanded
+ */
+ private boolean expandSelectItem(
+ final SqlNode selectItem,
+ SqlSelect select,
+ RelDataType targetType,
+ List<SqlNode> selectItems,
+ Set<String> aliases,
+ List<Map.Entry<String, RelDataType>> fields,
+ final boolean includeSystemVars) {
+ final SelectScope scope = (SelectScope) getWhereScope(select);
+ if (expandStar(selectItems, aliases, fields, includeSystemVars, scope,
+ selectItem)) {
+ return true;
+ }
+
+ // Expand the select item: fully-qualify columns, and convert
+ // parentheses-free functions such as LOCALTIME into explicit function
+ // calls.
+ SqlNode expanded = expand(selectItem, scope);
+ final String alias =
+ deriveAlias(
+ selectItem,
+ aliases.size());
+
+ // If expansion has altered the natural alias, supply an explicit 'AS'.
+ final SqlValidatorScope selectScope = getSelectScope(select);
+ if (expanded != selectItem) {
+ String newAlias =
+ deriveAlias(
+ expanded,
+ aliases.size());
+ if (!newAlias.equals(alias)) {
+ expanded =
+ SqlStdOperatorTable.AS.createCall(
+ selectItem.getParserPosition(),
+ expanded,
+ new SqlIdentifier(alias, SqlParserPos.ZERO));
+ deriveTypeImpl(selectScope, expanded);
+ }
+ }
+
+ selectItems.add(expanded);
+ aliases.add(alias);
+
+ inferUnknownTypes(targetType, scope, expanded);
+ final RelDataType type = deriveType(selectScope, expanded);
+ setValidatedNodeType(expanded, type);
+ fields.add(Pair.of(alias, type));
+ return false;
+ }
+
+ private boolean expandStar(List<SqlNode> selectItems, Set<String> aliases,
+ List<Map.Entry<String, RelDataType>> fields, boolean includeSystemVars,
+ SelectScope scope, SqlNode node) {
+ if (!(node instanceof SqlIdentifier)) {
+ return false;
+ }
+ final SqlIdentifier identifier = (SqlIdentifier) node;
+ if (!identifier.isStar()) {
+ return false;
+ }
+ final SqlParserPos startPosition = identifier.getParserPosition();
+ switch (identifier.names.size()) {
+ case 1:
+ boolean hasDynamicStruct = false;
+ for (ScopeChild child : scope.children) {
+ final int before = fields.size();
+ if (child.namespace.getRowType().isDynamicStruct()) {
+ hasDynamicStruct = true;
+ // don't expand star if the underneath table is dynamic.
+ // Treat this star as a special field in validation/conversion and
+ // wait until execution time to expand this star.
+ final SqlNode exp =
+ new SqlIdentifier(
+ ImmutableList.of(child.name,
+ DynamicRecordType.DYNAMIC_STAR_PREFIX),
+ startPosition);
+ addToSelectList(
+ selectItems,
+ aliases,
+ fields,
+ exp,
+ scope,
+ includeSystemVars);
+ } else {
+ final SqlNode from = child.namespace.getNode();
+ final SqlValidatorNamespace fromNs = getNamespace(from, scope);
+ assert fromNs != null;
+ final RelDataType rowType = fromNs.getRowType();
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ String columnName = field.getName();
+
+ // TODO: do real implicit collation here
+ final SqlIdentifier exp =
+ new SqlIdentifier(
+ ImmutableList.of(child.name, columnName),
+ startPosition);
+ // Don't add expanded rolled up columns
+ if (!isRolledUpColumn(exp, scope)) {
+ addOrExpandField(
+ selectItems,
+ aliases,
+ fields,
+ includeSystemVars,
+ scope,
+ exp,
+ field);
+ }
+ }
+ }
+ if (child.nullable) {
+ for (int i = before; i < fields.size(); i++) {
+ final Map.Entry<String, RelDataType> entry = fields.get(i);
+ final RelDataType type = entry.getValue();
+ if (!type.isNullable()) {
+ fields.set(i,
+ Pair.of(entry.getKey(),
+ typeFactory.createTypeWithNullability(type, true)));
+ }
+ }
+ }
+ }
+ // If NATURAL JOIN or USING is present, move key fields to the front of
+ // the list, per standard SQL. Disabled if there are dynamic fields.
+ if (!hasDynamicStruct || Bug.CALCITE_2400_FIXED) {
+ new Permute(scope.getNode().getFrom(), 0).permute(selectItems, fields);
+ }
+ return true;
+
+ default:
+ final SqlIdentifier prefixId = identifier.skipLast(1);
+ final SqlValidatorScope.ResolvedImpl resolved =
+ new SqlValidatorScope.ResolvedImpl();
+ final SqlNameMatcher nameMatcher =
+ scope.validator.catalogReader.nameMatcher();
+ scope.resolve(prefixId.names, nameMatcher, true, resolved);
+ if (resolved.count() == 0) {
+ // e.g. "select s.t.* from e"
+ // or "select r.* from e"
+ throw newValidationError(prefixId,
+ RESOURCE.unknownIdentifier(prefixId.toString()));
+ }
+ final RelDataType rowType = resolved.only().rowType();
+ if (rowType.isDynamicStruct()) {
+ // don't expand star if the underneath table is dynamic.
+ addToSelectList(
+ selectItems,
+ aliases,
+ fields,
+ prefixId.plus(DynamicRecordType.DYNAMIC_STAR_PREFIX, startPosition),
+ scope,
+ includeSystemVars);
+ } else if (rowType.isStruct()) {
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ String columnName = field.getName();
+
+ // TODO: do real implicit collation here
+ addOrExpandField(
+ selectItems,
+ aliases,
+ fields,
+ includeSystemVars,
+ scope,
+ prefixId.plus(columnName, startPosition),
+ field);
+ }
+ } else {
+ throw newValidationError(prefixId, RESOURCE.starRequiresRecordType());
+ }
+ return true;
+ }
+ }
+
+ private SqlNode maybeCast(SqlNode node, RelDataType currentType,
+ RelDataType desiredType) {
+ return currentType.equals(desiredType)
+ || (currentType.isNullable() != desiredType.isNullable()
+ && typeFactory.createTypeWithNullability(currentType,
+ desiredType.isNullable()).equals(desiredType))
+ ? node
+ : SqlStdOperatorTable.CAST.createCall(SqlParserPos.ZERO,
+ node, SqlTypeUtil.convertTypeToSpec(desiredType));
+ }
+
+ private boolean addOrExpandField(List<SqlNode> selectItems, Set<String> aliases,
+ List<Map.Entry<String, RelDataType>> fields, boolean includeSystemVars,
+ SelectScope scope, SqlIdentifier id, RelDataTypeField field) {
+ switch (field.getType().getStructKind()) {
+ case PEEK_FIELDS:
+ case PEEK_FIELDS_DEFAULT:
+ final SqlNode starExp = id.plusStar();
+ expandStar(
+ selectItems,
+ aliases,
+ fields,
+ includeSystemVars,
+ scope,
+ starExp);
+ return true;
+
+ default:
+ addToSelectList(
+ selectItems,
+ aliases,
+ fields,
+ id,
+ scope,
+ includeSystemVars);
+ }
+
+ return false;
+ }
+
+ public SqlNode validate(SqlNode topNode) {
+ SqlValidatorScope scope = new EmptyScope(this);
+ scope = new CatalogScope(scope, ImmutableList.of("CATALOG"));
+ final SqlNode topNode2 = validateScopedExpression(topNode, scope);
+ final RelDataType type = getValidatedNodeType(topNode2);
+ Util.discard(type);
+ return topNode2;
+ }
+
+ public List<SqlMoniker> lookupHints(SqlNode topNode, SqlParserPos pos) {
+ SqlValidatorScope scope = new EmptyScope(this);
+ SqlNode outermostNode = performUnconditionalRewrites(topNode, false);
+ cursorSet.add(outermostNode);
+ if (outermostNode.isA(SqlKind.TOP_LEVEL)) {
+ registerQuery(
+ scope,
+ null,
+ outermostNode,
+ outermostNode,
+ null,
+ false);
+ }
+ final SqlValidatorNamespace ns = getNamespace(outermostNode);
+ if (ns == null) {
+ throw new AssertionError("Not a query: " + outermostNode);
+ }
+ Collection<SqlMoniker> hintList = Sets.newTreeSet(SqlMoniker.COMPARATOR);
+ lookupSelectHints(ns, pos, hintList);
+ return ImmutableList.copyOf(hintList);
+ }
+
+ public SqlMoniker lookupQualifiedName(SqlNode topNode, SqlParserPos pos) {
+ final String posString = pos.toString();
+ IdInfo info = idPositions.get(posString);
+ if (info != null) {
+ final SqlQualified qualified = info.scope.fullyQualify(info.id);
+ return new SqlIdentifierMoniker(qualified.identifier);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Looks up completion hints for a syntactically correct select SQL that has
+ * been parsed into an expression tree.
+ *
+ * @param select the Select node of the parsed expression tree
+ * @param pos indicates the position in the sql statement we want to get
+ * completion hints for
+ * @param hintList list of {@link SqlMoniker} (sql identifiers) that can
+ * fill in at the indicated position
+ */
+ void lookupSelectHints(
+ SqlSelect select,
+ SqlParserPos pos,
+ Collection<SqlMoniker> hintList) {
+ IdInfo info = idPositions.get(pos.toString());
+ if ((info == null) || (info.scope == null)) {
+ SqlNode fromNode = select.getFrom();
+ final SqlValidatorScope fromScope = getFromScope(select);
+ lookupFromHints(fromNode, fromScope, pos, hintList);
+ } else {
+ lookupNameCompletionHints(info.scope, info.id.names,
+ info.id.getParserPosition(), hintList);
+ }
+ }
+
+ private void lookupSelectHints(
+ SqlValidatorNamespace ns,
+ SqlParserPos pos,
+ Collection<SqlMoniker> hintList) {
+ final SqlNode node = ns.getNode();
+ if (node instanceof SqlSelect) {
+ lookupSelectHints((SqlSelect) node, pos, hintList);
+ }
+ }
+
+ private void lookupFromHints(
+ SqlNode node,
+ SqlValidatorScope scope,
+ SqlParserPos pos,
+ Collection<SqlMoniker> hintList) {
+ final SqlValidatorNamespace ns = getNamespace(node);
+ if (ns.isWrapperFor(IdentifierNamespace.class)) {
+ IdentifierNamespace idNs = ns.unwrap(IdentifierNamespace.class);
+ final SqlIdentifier id = idNs.getId();
+ for (int i = 0; i < id.names.size(); i++) {
+ if (pos.toString().equals(
+ id.getComponent(i).getParserPosition().toString())) {
+ final List<SqlMoniker> objNames = new ArrayList<>();
+ SqlValidatorUtil.getSchemaObjectMonikers(
+ getCatalogReader(),
+ id.names.subList(0, i + 1),
+ objNames);
+ for (SqlMoniker objName : objNames) {
+ if (objName.getType() != SqlMonikerType.FUNCTION) {
+ hintList.add(objName);
+ }
+ }
+ return;
+ }
+ }
+ }
+ switch (node.getKind()) {
+ case JOIN:
+ lookupJoinHints((SqlJoin) node, scope, pos, hintList);
+ break;
+ default:
+ lookupSelectHints(ns, pos, hintList);
+ break;
+ }
+ }
+
+ private void lookupJoinHints(
+ SqlJoin join,
+ SqlValidatorScope scope,
+ SqlParserPos pos,
+ Collection<SqlMoniker> hintList) {
+ SqlNode left = join.getLeft();
+ SqlNode right = join.getRight();
+ SqlNode condition = join.getCondition();
+ lookupFromHints(left, scope, pos, hintList);
+ if (hintList.size() > 0) {
+ return;
+ }
+ lookupFromHints(right, scope, pos, hintList);
+ if (hintList.size() > 0) {
+ return;
+ }
+ final JoinConditionType conditionType = join.getConditionType();
+ final SqlValidatorScope joinScope = scopes.get(join);
+ switch (conditionType) {
+ case ON:
+ condition.findValidOptions(this, joinScope, pos, hintList);
+ return;
+ default:
+
+ // No suggestions.
+ // Not supporting hints for other types such as 'Using' yet.
+ return;
+ }
+ }
+
+ /**
+ * Populates a list of all the valid alternatives for an identifier.
+ *
+ * @param scope Validation scope
+ * @param names Components of the identifier
+ * @param pos position
+ * @param hintList a list of valid options
+ */
+ public final void lookupNameCompletionHints(
+ SqlValidatorScope scope,
+ List<String> names,
+ SqlParserPos pos,
+ Collection<SqlMoniker> hintList) {
+ // Remove the last part of name - it is a dummy
+ List<String> subNames = Util.skipLast(names);
+
+ if (subNames.size() > 0) {
+ // If there's a prefix, resolve it to a namespace.
+ SqlValidatorNamespace ns = null;
+ for (String name : subNames) {
+ if (ns == null) {
+ final SqlValidatorScope.ResolvedImpl resolved =
+ new SqlValidatorScope.ResolvedImpl();
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ scope.resolve(ImmutableList.of(name), nameMatcher, false, resolved);
+ if (resolved.count() == 1) {
+ ns = resolved.only().namespace;
+ }
+ } else {
+ ns = ns.lookupChild(name);
+ }
+ if (ns == null) {
+ break;
+ }
+ }
+ if (ns != null) {
+ RelDataType rowType = ns.getRowType();
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ hintList.add(
+ new SqlMonikerImpl(
+ field.getName(),
+ SqlMonikerType.COLUMN));
+ }
+ }
+
+ // builtin function names are valid completion hints when the
+ // identifier has only 1 name part
+ findAllValidFunctionNames(names, this, hintList, pos);
+ } else {
+ // No prefix; use the children of the current scope (that is,
+ // the aliases in the FROM clause)
+ scope.findAliases(hintList);
+
+ // If there's only one alias, add all child columns
+ SelectScope selectScope =
+ SqlValidatorUtil.getEnclosingSelectScope(scope);
+ if ((selectScope != null)
+ && (selectScope.getChildren().size() == 1)) {
+ RelDataType rowType =
+ selectScope.getChildren().get(0).getRowType();
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ hintList.add(
+ new SqlMonikerImpl(
+ field.getName(),
+ SqlMonikerType.COLUMN));
+ }
+ }
+ }
+
+ findAllValidUdfNames(names, this, hintList);
+ }
+
+ private static void findAllValidUdfNames(
+ List<String> names,
+ SqlValidator validator,
+ Collection<SqlMoniker> result) {
+ final List<SqlMoniker> objNames = new ArrayList<>();
+ SqlValidatorUtil.getSchemaObjectMonikers(
+ validator.getCatalogReader(),
+ names,
+ objNames);
+ for (SqlMoniker objName : objNames) {
+ if (objName.getType() == SqlMonikerType.FUNCTION) {
+ result.add(objName);
+ }
+ }
+ }
+
+ private static void findAllValidFunctionNames(
+ List<String> names,
+ SqlValidator validator,
+ Collection<SqlMoniker> result,
+ SqlParserPos pos) {
+ // a function name can only be 1 part
+ if (names.size() > 1) {
+ return;
+ }
+ for (SqlOperator op : validator.getOperatorTable().getOperatorList()) {
+ SqlIdentifier curOpId =
+ new SqlIdentifier(
+ op.getName(),
+ pos);
+
+ final SqlCall call =
+ SqlUtil.makeCall(
+ validator.getOperatorTable(),
+ curOpId);
+ if (call != null) {
+ result.add(
+ new SqlMonikerImpl(
+ op.getName(),
+ SqlMonikerType.FUNCTION));
+ } else {
+ if ((op.getSyntax() == SqlSyntax.FUNCTION)
+ || (op.getSyntax() == SqlSyntax.PREFIX)) {
+ if (op.getOperandTypeChecker() != null) {
+ String sig = op.getAllowedSignatures();
+ sig = sig.replaceAll("'", "");
+ result.add(
+ new SqlMonikerImpl(
+ sig,
+ SqlMonikerType.FUNCTION));
+ continue;
+ }
+ result.add(
+ new SqlMonikerImpl(
+ op.getName(),
+ SqlMonikerType.FUNCTION));
+ }
+ }
+ }
+ }
+
+ public SqlNode validateParameterizedExpression(
+ SqlNode topNode,
+ final Map<String, RelDataType> nameToTypeMap) {
+ SqlValidatorScope scope = new ParameterScope(this, nameToTypeMap);
+ return validateScopedExpression(topNode, scope);
+ }
+
+ private SqlNode validateScopedExpression(
+ SqlNode topNode,
+ SqlValidatorScope scope) {
+ SqlNode outermostNode = performUnconditionalRewrites(topNode, false);
+ cursorSet.add(outermostNode);
+ top = outermostNode;
+ TRACER.trace("After unconditional rewrite: {}", outermostNode);
+ if (outermostNode.isA(SqlKind.TOP_LEVEL)) {
+ registerQuery(scope, null, outermostNode, outermostNode, null, false);
+ }
+ outermostNode.validate(this, scope);
+ if (!outermostNode.isA(SqlKind.TOP_LEVEL)) {
+ // force type derivation so that we can provide it to the
+ // caller later without needing the scope
+ deriveType(scope, outermostNode);
+ }
+ TRACER.trace("After validation: {}", outermostNode);
+ return outermostNode;
+ }
+
+ public void validateQuery(SqlNode node, SqlValidatorScope scope,
+ RelDataType targetRowType) {
+ final SqlValidatorNamespace ns = getNamespace(node, scope);
+ if (node.getKind() == SqlKind.TABLESAMPLE) {
+ List<SqlNode> operands = ((SqlCall) node).getOperandList();
+ SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands.get(1));
+ if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
+ validateFeature(RESOURCE.sQLFeature_T613(), node.getParserPosition());
+ } else if (sampleSpec
+ instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
+ validateFeature(RESOURCE.sQLFeatureExt_T613_Substitution(),
+ node.getParserPosition());
+ }
+ }
+
+ validateNamespace(ns, targetRowType);
+ switch (node.getKind()) {
+ case EXTEND:
+ // Until we have a dedicated namespace for EXTEND
+ deriveType(scope, node);
+ }
+ if (node == top) {
+ validateModality(node);
+ }
+ validateAccess(
+ node,
+ ns.getTable(),
+ SqlAccessEnum.SELECT);
+ }
+
+ /**
+ * Validates a namespace.
+ *
+ * @param namespace Namespace
+ * @param targetRowType Desired row type, must not be null, may be the data
+ * type 'unknown'.
+ */
+ protected void validateNamespace(final SqlValidatorNamespace namespace,
+ RelDataType targetRowType) {
+ namespace.validate(targetRowType);
+ if (namespace.getNode() != null) {
+ setValidatedNodeType(namespace.getNode(), namespace.getType());
+ }
+ }
+
+ @VisibleForTesting
+ public SqlValidatorScope getEmptyScope() {
+ return new EmptyScope(this);
+ }
+
+ public SqlValidatorScope getCursorScope(SqlSelect select) {
+ return cursorScopes.get(select);
+ }
+
+ public SqlValidatorScope getWhereScope(SqlSelect select) {
+ return whereScopes.get(select);
+ }
+
+ public SqlValidatorScope getSelectScope(SqlSelect select) {
+ return selectScopes.get(select);
+ }
+
+ public SelectScope getRawSelectScope(SqlSelect select) {
+ SqlValidatorScope scope = getSelectScope(select);
+ if (scope instanceof AggregatingSelectScope) {
+ scope = ((AggregatingSelectScope) scope).getParent();
+ }
+ return (SelectScope) scope;
+ }
+
+ public SqlValidatorScope getHavingScope(SqlSelect select) {
+ // Yes, it's the same as getSelectScope
+ return selectScopes.get(select);
+ }
+
+ public SqlValidatorScope getGroupScope(SqlSelect select) {
+ // Yes, it's the same as getWhereScope
+ return groupByScopes.get(select);
+ }
+
+ public SqlValidatorScope getFromScope(SqlSelect select) {
+ return scopes.get(select);
+ }
+
+ public SqlValidatorScope getOrderScope(SqlSelect select) {
+ return orderScopes.get(select);
+ }
+
+ public SqlValidatorScope getMatchRecognizeScope(SqlMatchRecognize node) {
+ return scopes.get(node);
+ }
+
+ public SqlValidatorScope getJoinScope(SqlNode node) {
+ return scopes.get(stripAs(node));
+ }
+
+ public SqlValidatorScope getOverScope(SqlNode node) {
+ return scopes.get(node);
+ }
+
+ private SqlValidatorNamespace getNamespace(SqlNode node,
+ SqlValidatorScope scope) {
+ if (node instanceof SqlIdentifier && scope instanceof DelegatingScope) {
+ final SqlIdentifier id = (SqlIdentifier) node;
+ final DelegatingScope idScope = (DelegatingScope) ((DelegatingScope) scope).getParent();
+ return getNamespace(id, idScope);
+ } else if (node instanceof SqlCall) {
+ // Handle extended identifiers.
+ final SqlCall call = (SqlCall) node;
+ switch (call.getOperator().getKind()) {
+ case EXTEND:
+ final SqlIdentifier id = (SqlIdentifier) call.getOperandList().get(0);
+ final DelegatingScope idScope = (DelegatingScope) scope;
+ return getNamespace(id, idScope);
+ case AS:
+ final SqlNode nested = call.getOperandList().get(0);
+ switch (nested.getKind()) {
+ case EXTEND:
+ return getNamespace(nested, scope);
+ }
+ break;
+ }
+ }
+ return getNamespace(node);
+ }
+
+ private SqlValidatorNamespace getNamespace(SqlIdentifier id, DelegatingScope scope) {
+ if (id.isSimple()) {
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ final SqlValidatorScope.ResolvedImpl resolved =
+ new SqlValidatorScope.ResolvedImpl();
+ scope.resolve(id.names, nameMatcher, false, resolved);
+ if (resolved.count() == 1) {
+ return resolved.only().namespace;
+ }
+ }
+ return getNamespace(id);
+ }
+
+ public SqlValidatorNamespace getNamespace(SqlNode node) {
+ switch (node.getKind()) {
+ case AS:
+
+ // AS has a namespace if it has a column list 'AS t (c1, c2, ...)'
+ final SqlValidatorNamespace ns = namespaces.get(node);
+ if (ns != null) {
+ return ns;
+ }
+ // fall through
+ case OVER:
+ case COLLECTION_TABLE:
+ case ORDER_BY:
+ case TABLESAMPLE:
+ return getNamespace(((SqlCall) node).operand(0));
+ default:
+ return namespaces.get(node);
+ }
+ }
+
+ private void handleOffsetFetch(SqlNode offset, SqlNode fetch) {
+ if (offset instanceof SqlDynamicParam) {
+ setValidatedNodeType(offset,
+ typeFactory.createSqlType(SqlTypeName.INTEGER));
+ }
+ if (fetch instanceof SqlDynamicParam) {
+ setValidatedNodeType(fetch,
+ typeFactory.createSqlType(SqlTypeName.INTEGER));
+ }
+ }
+
+ /**
+ * Performs expression rewrites which are always used unconditionally. These
+ * rewrites massage the expression tree into a standard form so that the
+ * rest of the validation logic can be simpler.
+ *
+ * @param node expression to be rewritten
+ * @param underFrom whether node appears directly under a FROM clause
+ * @return rewritten expression
+ */
+ protected SqlNode performUnconditionalRewrites(
+ SqlNode node,
+ boolean underFrom) {
+ if (node == null) {
+ return node;
+ }
+
+ SqlNode newOperand;
+
+ // first transform operands and invoke generic call rewrite
+ if (node instanceof SqlCall) {
+ if (node instanceof SqlMerge) {
+ validatingSqlMerge = true;
+ }
+ SqlCall call = (SqlCall) node;
+ final SqlKind kind = call.getKind();
+ final List<SqlNode> operands = call.getOperandList();
+ for (int i = 0; i < operands.size(); i++) {
+ SqlNode operand = operands.get(i);
+ boolean childUnderFrom;
+ if (kind == SqlKind.SELECT) {
+ childUnderFrom = i == SqlSelect.FROM_OPERAND;
+ } else if (kind == SqlKind.AS && (i == 0)) {
+ // for an aliased expression, it is under FROM if
+ // the AS expression is under FROM
+ childUnderFrom = underFrom;
+ } else {
+ childUnderFrom = false;
+ }
+ newOperand =
+ performUnconditionalRewrites(operand, childUnderFrom);
+ if (newOperand != null && newOperand != operand) {
+ call.setOperand(i, newOperand);
+ }
+ }
+
+ if (call.getOperator() instanceof SqlUnresolvedFunction) {
+ assert call instanceof SqlBasicCall;
+ final SqlUnresolvedFunction function =
+ (SqlUnresolvedFunction) call.getOperator();
+ // This function hasn't been resolved yet. Perform
+ // a half-hearted resolution now in case it's a
+ // builtin function requiring special casing. If it's
+ // not, we'll handle it later during overload resolution.
+ final List<SqlOperator> overloads = new ArrayList<>();
+ opTab.lookupOperatorOverloads(function.getNameAsId(),
+ function.getFunctionType(), SqlSyntax.FUNCTION, overloads);
+ if (overloads.size() == 1) {
+ ((SqlBasicCall) call).setOperator(overloads.get(0));
+ }
+ }
+ if (rewriteCalls) {
+ node = call.getOperator().rewriteCall(this, call);
+ }
+ } else if (node instanceof SqlNodeList) {
+ SqlNodeList list = (SqlNodeList) node;
+ for (int i = 0, count = list.size(); i < count; i++) {
+ SqlNode operand = list.get(i);
+ newOperand =
+ performUnconditionalRewrites(
+ operand,
+ false);
+ if (newOperand != null) {
+ list.getList().set(i, newOperand);
+ }
+ }
+ }
+
+ // now transform node itself
+ final SqlKind kind = node.getKind();
+ switch (kind) {
+ case VALUES:
+ // CHECKSTYLE: IGNORE 1
+ if (underFrom || true) {
+ // leave FROM (VALUES(...)) [ AS alias ] clauses alone,
+ // otherwise they grow cancerously if this rewrite is invoked
+ // over and over
+ return node;
+ } else {
+ final SqlNodeList selectList =
+ new SqlNodeList(SqlParserPos.ZERO);
+ selectList.add(SqlIdentifier.star(SqlParserPos.ZERO));
+ return new SqlSelect(node.getParserPosition(), null, selectList, node,
+ null, null, null, null, null, null, null);
+ }
+
+ case ORDER_BY: {
+ SqlOrderBy orderBy = (SqlOrderBy) node;
+ handleOffsetFetch(orderBy.offset, orderBy.fetch);
+ if (orderBy.query instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) orderBy.query;
+
+ // Don't clobber existing ORDER BY. It may be needed for
+ // an order-sensitive function like RANK.
+ if (select.getOrderList() == null) {
+ // push ORDER BY into existing select
+ select.setOrderBy(orderBy.orderList);
+ select.setOffset(orderBy.offset);
+ select.setFetch(orderBy.fetch);
+ return select;
+ }
+ }
+ if (orderBy.query instanceof SqlWith
+ && ((SqlWith) orderBy.query).body instanceof SqlSelect) {
+ SqlWith with = (SqlWith) orderBy.query;
+ SqlSelect select = (SqlSelect) with.body;
+
+ // Don't clobber existing ORDER BY. It may be needed for
+ // an order-sensitive function like RANK.
+ if (select.getOrderList() == null) {
+ // push ORDER BY into existing select
+ select.setOrderBy(orderBy.orderList);
+ select.setOffset(orderBy.offset);
+ select.setFetch(orderBy.fetch);
+ return with;
+ }
+ }
+ final SqlNodeList selectList = new SqlNodeList(SqlParserPos.ZERO);
+ selectList.add(SqlIdentifier.star(SqlParserPos.ZERO));
+ final SqlNodeList orderList;
+ if (getInnerSelect(node) != null && isAggregate(getInnerSelect(node))) {
+ orderList = SqlNode.clone(orderBy.orderList);
+ // We assume that ORDER BY item does not have ASC etc.
+ // We assume that ORDER BY item is present in SELECT list.
+ for (int i = 0; i < orderList.size(); i++) {
+ SqlNode sqlNode = orderList.get(i);
+ SqlNodeList selectList2 = getInnerSelect(node).getSelectList();
+ for (Ord<SqlNode> sel : Ord.zip(selectList2)) {
+ if (stripAs(sel.e).equalsDeep(sqlNode, Litmus.IGNORE)) {
+ orderList.set(i,
+ SqlLiteral.createExactNumeric(Integer.toString(sel.i + 1),
+ SqlParserPos.ZERO));
+ }
+ }
+ }
+ } else {
+ orderList = orderBy.orderList;
+ }
+ return new SqlSelect(SqlParserPos.ZERO, null, selectList, orderBy.query,
+ null, null, null, null, orderList, orderBy.offset,
+ orderBy.fetch);
+ }
+
+ case EXPLICIT_TABLE: {
+ // (TABLE t) is equivalent to (SELECT * FROM t)
+ SqlCall call = (SqlCall) node;
+ final SqlNodeList selectList = new SqlNodeList(SqlParserPos.ZERO);
+ selectList.add(SqlIdentifier.star(SqlParserPos.ZERO));
+ return new SqlSelect(SqlParserPos.ZERO, null, selectList, call.operand(0),
+ null, null, null, null, null, null, null);
+ }
+
+ case DELETE: {
+ SqlDelete call = (SqlDelete) node;
+ SqlSelect select = createSourceSelectForDelete(call);
+ call.setSourceSelect(select);
+ break;
+ }
+
+ case UPDATE: {
+ SqlUpdate call = (SqlUpdate) node;
+ SqlSelect select = createSourceSelectForUpdate(call);
+ call.setSourceSelect(select);
+
+ // See if we're supposed to rewrite UPDATE to MERGE
+ // (unless this is the UPDATE clause of a MERGE,
+ // in which case leave it alone).
+ if (!validatingSqlMerge) {
+ SqlNode selfJoinSrcExpr =
+ getSelfJoinExprForUpdate(
+ call.getTargetTable(),
+ UPDATE_SRC_ALIAS);
+ if (selfJoinSrcExpr != null) {
+ node = rewriteUpdateToMerge(call, selfJoinSrcExpr);
+ }
+ }
+ break;
+ }
+
+ case MERGE: {
+ SqlMerge call = (SqlMerge) node;
+ rewriteMerge(call);
+ break;
+ }
+ }
+ return node;
+ }
+
+ private SqlSelect getInnerSelect(SqlNode node) {
+ for (;;) {
+ if (node instanceof SqlSelect) {
+ return (SqlSelect) node;
+ } else if (node instanceof SqlOrderBy) {
+ node = ((SqlOrderBy) node).query;
+ } else if (node instanceof SqlWith) {
+ node = ((SqlWith) node).body;
+ } else {
+ return null;
+ }
+ }
+ }
+
+ private void rewriteMerge(SqlMerge call) {
+ SqlNodeList selectList;
+ SqlUpdate updateStmt = call.getUpdateCall();
+ if (updateStmt != null) {
+ // if we have an update statement, just clone the select list
+ // from the update statement's source since it's the same as
+ // what we want for the select list of the merge source -- '*'
+ // followed by the update set expressions
+ selectList = SqlNode.clone(updateStmt.getSourceSelect().getSelectList());
+ } else {
+ // otherwise, just use select *
+ selectList = new SqlNodeList(SqlParserPos.ZERO);
+ selectList.add(SqlIdentifier.star(SqlParserPos.ZERO));
+ }
+ SqlNode targetTable = call.getTargetTable();
+ if (call.getAlias() != null) {
+ targetTable =
+ SqlValidatorUtil.addAlias(
+ targetTable,
+ call.getAlias().getSimple());
+ }
+
+ // Provided there is an insert substatement, the source select for
+ // the merge is a left outer join between the source in the USING
+ // clause and the target table; otherwise, the join is just an
+ // inner join. Need to clone the source table reference in order
+ // for validation to work
+ SqlNode sourceTableRef = call.getSourceTableRef();
+ SqlInsert insertCall = call.getInsertCall();
+ JoinType joinType = (insertCall == null) ? JoinType.INNER : JoinType.LEFT;
+ final SqlNode leftJoinTerm = SqlNode.clone(sourceTableRef);
+ SqlNode outerJoin =
+ new SqlJoin(SqlParserPos.ZERO,
+ leftJoinTerm,
+ SqlLiteral.createBoolean(false, SqlParserPos.ZERO),
+ joinType.symbol(SqlParserPos.ZERO),
+ targetTable,
+ JoinConditionType.ON.symbol(SqlParserPos.ZERO),
+ call.getCondition());
+ SqlSelect select =
+ new SqlSelect(SqlParserPos.ZERO, null, selectList, outerJoin, null,
+ null, null, null, null, null, null);
+ call.setSourceSelect(select);
+
+ // Source for the insert call is a select of the source table
+ // reference with the select list being the value expressions;
+ // note that the values clause has already been converted to a
+ // select on the values row constructor; so we need to extract
+ // that via the from clause on the select
+ if (insertCall != null) {
+ SqlCall valuesCall = (SqlCall) insertCall.getSource();
+ SqlCall rowCall = valuesCall.operand(0);
+ selectList =
+ new SqlNodeList(
+ rowCall.getOperandList(),
+ SqlParserPos.ZERO);
+ final SqlNode insertSource = SqlNode.clone(sourceTableRef);
+ select =
+ new SqlSelect(SqlParserPos.ZERO, null, selectList, insertSource, null,
+ null, null, null, null, null, null);
+ insertCall.setSource(select);
+ }
+ }
+
+ private SqlNode rewriteUpdateToMerge(
+ SqlUpdate updateCall,
+ SqlNode selfJoinSrcExpr) {
+ // Make sure target has an alias.
+ if (updateCall.getAlias() == null) {
+ updateCall.setAlias(
+ new SqlIdentifier(UPDATE_TGT_ALIAS, SqlParserPos.ZERO));
+ }
+ SqlNode selfJoinTgtExpr =
+ getSelfJoinExprForUpdate(
+ updateCall.getTargetTable(),
+ updateCall.getAlias().getSimple());
+ assert selfJoinTgtExpr != null;
+
+ // Create join condition between source and target exprs,
+ // creating a conjunction with the user-level WHERE
+ // clause if one was supplied
+ SqlNode condition = updateCall.getCondition();
+ SqlNode selfJoinCond =
+ SqlStdOperatorTable.EQUALS.createCall(
+ SqlParserPos.ZERO,
+ selfJoinSrcExpr,
+ selfJoinTgtExpr);
+ if (condition == null) {
+ condition = selfJoinCond;
+ } else {
+ condition =
+ SqlStdOperatorTable.AND.createCall(
+ SqlParserPos.ZERO,
+ selfJoinCond,
+ condition);
+ }
+ SqlNode target =
+ updateCall.getTargetTable().clone(SqlParserPos.ZERO);
+
+ // For the source, we need to anonymize the fields, so
+ // that for a statement like UPDATE T SET I = I + 1,
+ // there's no ambiguity for the "I" in "I + 1";
+ // this is OK because the source and target have
+ // identical values due to the self-join.
+ // Note that we anonymize the source rather than the
+ // target because downstream, the optimizer rules
+ // don't want to see any projection on top of the target.
+ IdentifierNamespace ns =
+ new IdentifierNamespace(this, target, null, null);
+ RelDataType rowType = ns.getRowType();
+ SqlNode source = updateCall.getTargetTable().clone(SqlParserPos.ZERO);
+ final SqlNodeList selectList = new SqlNodeList(SqlParserPos.ZERO);
+ int i = 1;
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ SqlIdentifier col =
+ new SqlIdentifier(
+ field.getName(),
+ SqlParserPos.ZERO);
+ selectList.add(
+ SqlValidatorUtil.addAlias(col, UPDATE_ANON_PREFIX + i));
+ ++i;
+ }
+ source =
+ new SqlSelect(SqlParserPos.ZERO, null, selectList, source, null, null,
+ null, null, null, null, null);
+ source = SqlValidatorUtil.addAlias(source, UPDATE_SRC_ALIAS);
+ SqlMerge mergeCall =
+ new SqlMerge(updateCall.getParserPosition(), target, condition, source,
+ updateCall, null, null, updateCall.getAlias());
+ rewriteMerge(mergeCall);
+ return mergeCall;
+ }
+
+ /**
+ * Allows a subclass to provide information about how to convert an UPDATE
+ * into a MERGE via self-join. If this method returns null, then no such
+ * conversion takes place. Otherwise, this method should return a suitable
+ * unique identifier expression for the given table.
+ *
+ * @param table identifier for table being updated
+ * @param alias alias to use for qualifying columns in expression, or null
+ * for unqualified references; if this is equal to
+ * {@value #UPDATE_SRC_ALIAS}, then column references have been
+ * anonymized to "SYS$ANONx", where x is the 1-based column
+ * number.
+ * @return expression for unique identifier, or null to prevent conversion
+ */
+ protected SqlNode getSelfJoinExprForUpdate(
+ SqlNode table,
+ String alias) {
+ return null;
+ }
+
+ /**
+ * Creates the SELECT statement that putatively feeds rows into an UPDATE
+ * statement to be updated.
+ *
+ * @param call Call to the UPDATE operator
+ * @return select statement
+ */
+ protected SqlSelect createSourceSelectForUpdate(SqlUpdate call) {
+ final SqlNodeList selectList = new SqlNodeList(SqlParserPos.ZERO);
+ selectList.add(SqlIdentifier.star(SqlParserPos.ZERO));
+ int ordinal = 0;
+ for (SqlNode exp : call.getSourceExpressionList()) {
+ // Force unique aliases to avoid a duplicate for Y with
+ // SET X=Y
+ String alias = SqlUtil.deriveAliasFromOrdinal(ordinal);
+ selectList.add(SqlValidatorUtil.addAlias(exp, alias));
+ ++ordinal;
+ }
+ SqlNode sourceTable = call.getTargetTable();
+ if (call.getAlias() != null) {
+ sourceTable =
+ SqlValidatorUtil.addAlias(
+ sourceTable,
+ call.getAlias().getSimple());
+ }
+ return new SqlSelect(SqlParserPos.ZERO, null, selectList, sourceTable,
+ call.getCondition(), null, null, null, null, null, null);
+ }
+
+ /**
+ * Creates the SELECT statement that putatively feeds rows into a DELETE
+ * statement to be deleted.
+ *
+ * @param call Call to the DELETE operator
+ * @return select statement
+ */
+ protected SqlSelect createSourceSelectForDelete(SqlDelete call) {
+ final SqlNodeList selectList = new SqlNodeList(SqlParserPos.ZERO);
+ selectList.add(SqlIdentifier.star(SqlParserPos.ZERO));
+ SqlNode sourceTable = call.getTargetTable();
+ if (call.getAlias() != null) {
+ sourceTable =
+ SqlValidatorUtil.addAlias(
+ sourceTable,
+ call.getAlias().getSimple());
+ }
+ return new SqlSelect(SqlParserPos.ZERO, null, selectList, sourceTable,
+ call.getCondition(), null, null, null, null, null, null);
+ }
+
+ /**
+ * Returns null if there is no common type. E.g. if the rows have a
+ * different number of columns.
+ */
+ RelDataType getTableConstructorRowType(
+ SqlCall values,
+ SqlValidatorScope scope) {
+ final List<SqlNode> rows = values.getOperandList();
+ assert rows.size() >= 1;
+ final List<RelDataType> rowTypes = new ArrayList<>();
+ for (final SqlNode row : rows) {
+ assert row.getKind() == SqlKind.ROW;
+ SqlCall rowConstructor = (SqlCall) row;
+
+ // REVIEW jvs 10-Sept-2003: Once we support single-row queries as
+ // rows, need to infer aliases from there.
+ final List<String> aliasList = new ArrayList<>();
+ final List<RelDataType> typeList = new ArrayList<>();
+ for (Ord<SqlNode> column : Ord.zip(rowConstructor.getOperandList())) {
+ final String alias = deriveAlias(column.e, column.i);
+ aliasList.add(alias);
+ final RelDataType type = deriveType(scope, column.e);
+ typeList.add(type);
+ }
+ rowTypes.add(typeFactory.createStructType(typeList, aliasList));
+ }
+ if (rows.size() == 1) {
+ // TODO jvs 10-Oct-2005: get rid of this workaround once
+ // leastRestrictive can handle all cases
+ return rowTypes.get(0);
+ }
+ return typeFactory.leastRestrictive(rowTypes);
+ }
+
+ public RelDataType getValidatedNodeType(SqlNode node) {
+ RelDataType type = getValidatedNodeTypeIfKnown(node);
+ if (type == null) {
+ throw Util.needToImplement(node);
+ } else {
+ return type;
+ }
+ }
+
+ public RelDataType getValidatedNodeTypeIfKnown(SqlNode node) {
+ final RelDataType type = nodeToTypeMap.get(node);
+ if (type != null) {
+ return type;
+ }
+ final SqlValidatorNamespace ns = getNamespace(node);
+ if (ns != null) {
+ return ns.getType();
+ }
+ final SqlNode original = originalExprs.get(node);
+ if (original != null && original != node) {
+ return getValidatedNodeType(original);
+ }
+ if (node instanceof SqlIdentifier) {
+ return getCatalogReader().getNamedType((SqlIdentifier) node);
+ }
+ return null;
+ }
+
+ /**
+ * Saves the type of a {@link SqlNode}, now that it has been validated.
+ *
+ * <p>Unlike the base class method, this method is not deprecated.
+ * It is available from within Calcite, but is not part of the public API.
+ *
+ * @param node A SQL parse tree node, never null
+ * @param type Its type; must not be null
+ */
+ @SuppressWarnings("deprecation")
+ public final void setValidatedNodeType(SqlNode node, RelDataType type) {
+ Objects.requireNonNull(type);
+ Objects.requireNonNull(node);
+ if (type.equals(unknownType)) {
+ // don't set anything until we know what it is, and don't overwrite
+ // a known type with the unknown type
+ return;
+ }
+ nodeToTypeMap.put(node, type);
+ }
+
+ public void removeValidatedNodeType(SqlNode node) {
+ nodeToTypeMap.remove(node);
+ }
+
+ public RelDataType deriveType(
+ SqlValidatorScope scope,
+ SqlNode expr) {
+ Objects.requireNonNull(scope);
+ Objects.requireNonNull(expr);
+
+ // if we already know the type, no need to re-derive
+ RelDataType type = nodeToTypeMap.get(expr);
+ if (type != null) {
+ return type;
+ }
+ final SqlValidatorNamespace ns = getNamespace(expr);
+ if (ns != null) {
+ return ns.getType();
+ }
+ type = deriveTypeImpl(scope, expr);
+ Preconditions.checkArgument(
+ type != null,
+ "SqlValidator.deriveTypeInternal returned null");
+ setValidatedNodeType(expr, type);
+ return type;
+ }
+
+ /**
+ * Derives the type of a node, never null.
+ */
+ RelDataType deriveTypeImpl(
+ SqlValidatorScope scope,
+ SqlNode operand) {
+ DeriveTypeVisitor v = new DeriveTypeVisitor(scope);
+ final RelDataType type = operand.accept(v);
+ return Objects.requireNonNull(scope.nullifyType(operand, type));
+ }
+
+ public RelDataType deriveConstructorType(
+ SqlValidatorScope scope,
+ SqlCall call,
+ SqlFunction unresolvedConstructor,
+ SqlFunction resolvedConstructor,
+ List<RelDataType> argTypes) {
+ SqlIdentifier sqlIdentifier = unresolvedConstructor.getSqlIdentifier();
+ assert sqlIdentifier != null;
+ RelDataType type = catalogReader.getNamedType(sqlIdentifier);
+ if (type == null) {
+ // TODO jvs 12-Feb-2005: proper type name formatting
+ throw newValidationError(sqlIdentifier,
+ RESOURCE.unknownDatatypeName(sqlIdentifier.toString()));
+ }
+
+ if (resolvedConstructor == null) {
+ if (call.operandCount() > 0) {
+ // This is not a default constructor invocation, and
+ // no user-defined constructor could be found
+ throw handleUnresolvedFunction(call, unresolvedConstructor, argTypes,
+ null);
+ }
+ } else {
+ SqlCall testCall =
+ resolvedConstructor.createCall(
+ call.getParserPosition(),
+ call.getOperandList());
+ RelDataType returnType =
+ resolvedConstructor.validateOperands(
+ this,
+ scope,
+ testCall);
+ assert type == returnType;
+ }
+
+ if (shouldExpandIdentifiers()) {
+ if (resolvedConstructor != null) {
+ ((SqlBasicCall) call).setOperator(resolvedConstructor);
+ } else {
+ // fake a fully-qualified call to the default constructor
+ ((SqlBasicCall) call).setOperator(
+ new SqlFunction(
+ type.getSqlIdentifier(),
+ ReturnTypes.explicit(type),
+ null,
+ null,
+ null,
+ SqlFunctionCategory.USER_DEFINED_CONSTRUCTOR));
+ }
+ }
+ return type;
+ }
+
+ public CalciteException handleUnresolvedFunction(SqlCall call,
+ SqlFunction unresolvedFunction, List<RelDataType> argTypes,
+ List<String> argNames) {
+ // For builtins, we can give a better error message
+ final List<SqlOperator> overloads = new ArrayList<>();
+ opTab.lookupOperatorOverloads(unresolvedFunction.getNameAsId(), null,
+ SqlSyntax.FUNCTION, overloads);
+ if (overloads.size() == 1) {
+ SqlFunction fun = (SqlFunction) overloads.get(0);
+ if ((fun.getSqlIdentifier() == null)
+ && (fun.getSyntax() != SqlSyntax.FUNCTION_ID)) {
+ final int expectedArgCount =
+ fun.getOperandCountRange().getMin();
+ throw newValidationError(call,
+ RESOURCE.invalidArgCount(call.getOperator().getName(),
+ expectedArgCount));
+ }
+ }
+
+ AssignableOperandTypeChecker typeChecking =
+ new AssignableOperandTypeChecker(argTypes, argNames);
+ String signature =
+ typeChecking.getAllowedSignatures(
+ unresolvedFunction,
+ unresolvedFunction.getName());
+ throw newValidationError(call,
+ RESOURCE.validatorUnknownFunction(signature));
+ }
+
+ protected void inferUnknownTypes(
+ RelDataType inferredType,
+ SqlValidatorScope scope,
+ SqlNode node) {
+ final SqlValidatorScope newScope = scopes.get(node);
+ if (newScope != null) {
+ scope = newScope;
+ }
+ boolean isNullLiteral = SqlUtil.isNullLiteral(node, false);
+ if ((node instanceof SqlDynamicParam) || isNullLiteral) {
+ if (inferredType.equals(unknownType)) {
+ if (isNullLiteral) {
+ throw newValidationError(node, RESOURCE.nullIllegal());
+ } else {
+ throw newValidationError(node, RESOURCE.dynamicParamIllegal());
+ }
+ }
+
+ // REVIEW: should dynamic parameter types always be nullable?
+ RelDataType newInferredType =
+ typeFactory.createTypeWithNullability(inferredType, true);
+ if (SqlTypeUtil.inCharFamily(inferredType)) {
+ newInferredType =
+ typeFactory.createTypeWithCharsetAndCollation(
+ newInferredType,
+ inferredType.getCharset(),
+ inferredType.getCollation());
+ }
+ setValidatedNodeType(node, newInferredType);
+ } else if (node instanceof SqlNodeList) {
+ SqlNodeList nodeList = (SqlNodeList) node;
+ if (inferredType.isStruct()) {
+ if (inferredType.getFieldCount() != nodeList.size()) {
+ // this can happen when we're validating an INSERT
+ // where the source and target degrees are different;
+ // bust out, and the error will be detected higher up
+ return;
+ }
+ }
+ int i = 0;
+ for (SqlNode child : nodeList) {
+ RelDataType type;
+ if (inferredType.isStruct()) {
+ type = inferredType.getFieldList().get(i).getType();
+ ++i;
+ } else {
+ type = inferredType;
+ }
+ inferUnknownTypes(type, scope, child);
+ }
+ } else if (node instanceof SqlCase) {
+ final SqlCase caseCall = (SqlCase) node;
+
+ final RelDataType whenType =
+ caseCall.getValueOperand() == null ? booleanType : unknownType;
+ for (SqlNode sqlNode : caseCall.getWhenOperands().getList()) {
+ inferUnknownTypes(whenType, scope, sqlNode);
+ }
+ RelDataType returnType = deriveType(scope, node);
+ for (SqlNode sqlNode : caseCall.getThenOperands().getList()) {
+ inferUnknownTypes(returnType, scope, sqlNode);
+ }
+
+ if (!SqlUtil.isNullLiteral(caseCall.getElseOperand(), false)) {
+ inferUnknownTypes(
+ returnType,
+ scope,
+ caseCall.getElseOperand());
+ } else {
+ setValidatedNodeType(caseCall.getElseOperand(), returnType);
+ }
+ } else if (node instanceof SqlCall) {
+ final SqlCall call = (SqlCall) node;
+ final SqlOperandTypeInference operandTypeInference =
+ call.getOperator().getOperandTypeInference();
+ final SqlCallBinding callBinding = new SqlCallBinding(this, scope, call);
+ final List<SqlNode> operands = callBinding.operands();
+ final RelDataType[] operandTypes = new RelDataType[operands.size()];
+ if (operandTypeInference == null) {
+ // TODO: eventually should assert(operandTypeInference != null)
+ // instead; for now just eat it
+ Arrays.fill(operandTypes, unknownType);
+ } else {
+ operandTypeInference.inferOperandTypes(
+ callBinding,
+ inferredType,
+ operandTypes);
+ }
+ for (int i = 0; i < operands.size(); ++i) {
+ inferUnknownTypes(operandTypes[i], scope, operands.get(i));
+ }
+ }
+ }
+
+ /**
+ * Adds an expression to a select list, ensuring that its alias does not
+ * clash with any existing expressions on the list.
+ */
+ protected void addToSelectList(
+ List<SqlNode> list,
+ Set<String> aliases,
+ List<Map.Entry<String, RelDataType>> fieldList,
+ SqlNode exp,
+ SqlValidatorScope scope,
+ final boolean includeSystemVars) {
+ String alias = SqlValidatorUtil.getAlias(exp, -1);
+ String uniqueAlias =
+ SqlValidatorUtil.uniquify(
+ alias, aliases, SqlValidatorUtil.EXPR_SUGGESTER);
+ if (!alias.equals(uniqueAlias)) {
+ exp = SqlValidatorUtil.addAlias(exp, uniqueAlias);
+ }
+ fieldList.add(Pair.of(uniqueAlias, deriveType(scope, exp)));
+ list.add(exp);
+ }
+
+ public String deriveAlias(
+ SqlNode node,
+ int ordinal) {
+ return SqlValidatorUtil.getAlias(node, ordinal);
+ }
+
+ // implement SqlValidator
+ public void setIdentifierExpansion(boolean expandIdentifiers) {
+ this.expandIdentifiers = expandIdentifiers;
+ }
+
+ // implement SqlValidator
+ public void setColumnReferenceExpansion(
+ boolean expandColumnReferences) {
+ this.expandColumnReferences = expandColumnReferences;
+ }
+
+ // implement SqlValidator
+ public boolean getColumnReferenceExpansion() {
+ return expandColumnReferences;
+ }
+
+ public void setDefaultNullCollation(NullCollation nullCollation) {
+ this.nullCollation = Objects.requireNonNull(nullCollation);
+ }
+
+ public NullCollation getDefaultNullCollation() {
+ return nullCollation;
+ }
+
+ // implement SqlValidator
+ public void setCallRewrite(boolean rewriteCalls) {
+ this.rewriteCalls = rewriteCalls;
+ }
+
+ public boolean shouldExpandIdentifiers() {
+ return expandIdentifiers;
+ }
+
+ protected boolean shouldAllowIntermediateOrderBy() {
+ return true;
+ }
+
+ private void registerMatchRecognize(
+ SqlValidatorScope parentScope,
+ SqlValidatorScope usingScope,
+ SqlMatchRecognize call,
+ SqlNode enclosingNode,
+ String alias,
+ boolean forceNullable) {
+
+ final MatchRecognizeNamespace matchRecognizeNamespace =
+ createMatchRecognizeNameSpace(call, enclosingNode);
+ registerNamespace(usingScope, alias, matchRecognizeNamespace, forceNullable);
+
+ final MatchRecognizeScope matchRecognizeScope =
+ new MatchRecognizeScope(parentScope, call);
+ scopes.put(call, matchRecognizeScope);
+
+ // parse input query
+ SqlNode expr = call.getTableRef();
+ SqlNode newExpr = registerFrom(usingScope, matchRecognizeScope, true, expr,
+ expr, null, null, forceNullable, false);
+ if (expr != newExpr) {
+ call.setOperand(0, newExpr);
+ }
+ }
+
+ protected MatchRecognizeNamespace createMatchRecognizeNameSpace(
+ SqlMatchRecognize call,
+ SqlNode enclosingNode) {
+ return new MatchRecognizeNamespace(this, call, enclosingNode);
+ }
+
+ /**
+ * Registers a new namespace, and adds it as a child of its parent scope.
+ * Derived class can override this method to tinker with namespaces as they
+ * are created.
+ *
+ * @param usingScope Parent scope (which will want to look for things in
+ * this namespace)
+ * @param alias Alias by which parent will refer to this namespace
+ * @param ns Namespace
+ * @param forceNullable Whether to force the type of namespace to be nullable
+ */
+ protected void registerNamespace(
+ SqlValidatorScope usingScope,
+ String alias,
+ SqlValidatorNamespace ns,
+ boolean forceNullable) {
+ namespaces.put(ns.getNode(), ns);
+ if (usingScope != null) {
+ usingScope.addChild(ns, alias, forceNullable);
+ }
+ }
+
+ /**
+ * Registers scopes and namespaces implied a relational expression in the
+ * FROM clause.
+ *
+ * <p>{@code parentScope} and {@code usingScope} are often the same. They
+ * differ when the namespace are not visible within the parent. (Example
+ * needed.)
+ *
+ * <p>Likewise, {@code enclosingNode} and {@code node} are often the same.
+ * {@code enclosingNode} is the topmost node within the FROM clause, from
+ * which any decorations like an alias (<code>AS alias</code>) or a table
+ * sample clause are stripped away to get {@code node}. Both are recorded in
+ * the namespace.
+ *
+ * @param parentScope Parent scope which this scope turns to in order to
+ * resolve objects
+ * @param usingScope Scope whose child list this scope should add itself to
+ * @param register Whether to register this scope as a child of
+ * {@code usingScope}
+ * @param node Node which namespace is based on
+ * @param enclosingNode Outermost node for namespace, including decorations
+ * such as alias and sample clause
+ * @param alias Alias
+ * @param extendList Definitions of extended columns
+ * @param forceNullable Whether to force the type of namespace to be
+ * nullable because it is in an outer join
+ * @param lateral Whether LATERAL is specified, so that items to the
+ * left of this in the JOIN tree are visible in the
+ * scope
+ * @return registered node, usually the same as {@code node}
+ */
+ private SqlNode registerFrom(
+ SqlValidatorScope parentScope,
+ SqlValidatorScope usingScope,
+ boolean register,
+ final SqlNode node,
+ SqlNode enclosingNode,
+ String alias,
+ SqlNodeList extendList,
+ boolean forceNullable,
+ final boolean lateral) {
+ final SqlKind kind = node.getKind();
+
+ SqlNode expr;
+ SqlNode newExpr;
+
+ // Add an alias if necessary.
+ SqlNode newNode = node;
+ if (alias == null) {
+ switch (kind) {
+ case IDENTIFIER:
+ case OVER:
+ alias = deriveAlias(node, -1);
+ if (alias == null) {
+ alias = deriveAlias(node, nextGeneratedId++);
+ }
+ if (shouldExpandIdentifiers()) {
+ newNode = SqlValidatorUtil.addAlias(node, alias);
+ }
+ break;
+
+ case SELECT:
+ case UNION:
+ case INTERSECT:
+ case EXCEPT:
+ case VALUES:
+ case UNNEST:
+ case OTHER_FUNCTION:
+ case COLLECTION_TABLE:
+ case MATCH_RECOGNIZE:
+
+ // give this anonymous construct a name since later
+ // query processing stages rely on it
+ alias = deriveAlias(node, nextGeneratedId++);
+ if (shouldExpandIdentifiers()) {
+ // Since we're expanding identifiers, we should make the
+ // aliases explicit too, otherwise the expanded query
+ // will not be consistent if we convert back to SQL, e.g.
+ // "select EXPR$1.EXPR$2 from values (1)".
+ newNode = SqlValidatorUtil.addAlias(node, alias);
+ }
+ break;
+ }
+ }
+
+ if (lateral) {
+ SqlValidatorScope s = usingScope;
+ while (s instanceof JoinScope) {
+ s = ((JoinScope) s).getUsingScope();
+ }
+ final SqlNode node2 = s != null ? s.getNode() : node;
+ final TableScope tableScope = new TableScope(parentScope, node2);
+ if (usingScope instanceof ListScope) {
+ for (ScopeChild child : ((ListScope) usingScope).children) {
+ tableScope.addChild(child.namespace, child.name, child.nullable);
+ }
+ }
+ parentScope = tableScope;
+ }
+
+ SqlCall call;
+ SqlNode operand;
+ SqlNode newOperand;
+
+ switch (kind) {
+ case AS:
+ call = (SqlCall) node;
+ if (alias == null) {
+ alias = call.operand(1).toString();
+ }
+ final boolean needAlias = call.operandCount() > 2;
+ expr = call.operand(0);
+ newExpr =
+ registerFrom(
+ parentScope,
+ usingScope,
+ !needAlias,
+ expr,
+ enclosingNode,
+ alias,
+ extendList,
+ forceNullable,
+ lateral);
+ if (newExpr != expr) {
+ call.setOperand(0, newExpr);
+ }
+
+ // If alias has a column list, introduce a namespace to translate
+ // column names. We skipped registering it just now.
+ if (needAlias) {
+ registerNamespace(
+ usingScope,
+ alias,
+ new AliasNamespace(this, call, enclosingNode),
+ forceNullable);
+ }
+ return node;
+ case MATCH_RECOGNIZE:
+ registerMatchRecognize(parentScope, usingScope,
+ (SqlMatchRecognize) node, enclosingNode, alias, forceNullable);
+ return node;
+ case TABLESAMPLE:
+ call = (SqlCall) node;
+ expr = call.operand(0);
+ newExpr =
+ registerFrom(
+ parentScope,
+ usingScope,
+ true,
+ expr,
+ enclosingNode,
+ alias,
+ extendList,
+ forceNullable,
+ lateral);
+ if (newExpr != expr) {
+ call.setOperand(0, newExpr);
+ }
+ return node;
+
+ case JOIN:
+ final SqlJoin join = (SqlJoin) node;
+ final JoinScope joinScope =
+ new JoinScope(parentScope, usingScope, join);
+ scopes.put(join, joinScope);
+ final SqlNode left = join.getLeft();
+ final SqlNode right = join.getRight();
+ final boolean rightIsLateral = isLateral(right);
+ boolean forceLeftNullable = forceNullable;
+ boolean forceRightNullable = forceNullable;
+ switch (join.getJoinType()) {
+ case LEFT:
+ forceRightNullable = true;
+ break;
+ case RIGHT:
+ forceLeftNullable = true;
+ break;
+ case FULL:
+ forceLeftNullable = true;
+ forceRightNullable = true;
+ break;
+ }
+ final SqlNode newLeft =
+ registerFrom(
+ parentScope,
+ joinScope,
+ true,
+ left,
+ left,
+ null,
+ null,
+ forceLeftNullable,
+ lateral);
+ if (newLeft != left) {
+ join.setLeft(newLeft);
+ }
+ final SqlNode newRight =
+ registerFrom(
+ parentScope,
+ joinScope,
+ true,
+ right,
+ right,
+ null,
+ null,
+ forceRightNullable,
+ lateral);
+ if (newRight != right) {
+ join.setRight(newRight);
+ }
+ registerSubQueries(joinScope, join.getCondition());
+ final JoinNamespace joinNamespace = new JoinNamespace(this, join);
+ registerNamespace(null, null, joinNamespace, forceNullable);
+ return join;
+
+ case IDENTIFIER:
+ final SqlIdentifier id = (SqlIdentifier) node;
+ final IdentifierNamespace newNs =
+ new IdentifierNamespace(
+ this, id, extendList, enclosingNode,
+ parentScope);
+ registerNamespace(register ? usingScope : null, alias, newNs,
+ forceNullable);
+ if (tableScope == null) {
+ tableScope = new TableScope(parentScope, node);
+ }
+ tableScope.addChild(newNs, alias, forceNullable);
+ if (extendList != null && extendList.size() != 0) {
+ return enclosingNode;
+ }
+ return newNode;
+
+ case LATERAL:
+ return registerFrom(
+ parentScope,
+ usingScope,
+ register,
+ ((SqlCall) node).operand(0),
+ enclosingNode,
+ alias,
+ extendList,
+ forceNullable,
+ true);
+
+ case COLLECTION_TABLE:
+ call = (SqlCall) node;
+ operand = call.operand(0);
+ newOperand =
+ registerFrom(
+ parentScope,
+ usingScope,
+ register,
+ operand,
+ enclosingNode,
+ alias,
+ extendList,
+ forceNullable, lateral);
+ if (newOperand != operand) {
+ call.setOperand(0, newOperand);
+ }
+ scopes.put(node, parentScope);
+ return newNode;
+
+ case UNNEST:
+ if (!lateral) {
+ return registerFrom(parentScope, usingScope, register, node,
+ enclosingNode, alias, extendList, forceNullable, true);
+ }
+ // fall through
+ case SELECT:
+ case UNION:
+ case INTERSECT:
+ case EXCEPT:
+ case VALUES:
+ case WITH:
+ case OTHER_FUNCTION:
+ if (alias == null) {
+ alias = deriveAlias(node, nextGeneratedId++);
+ }
+ registerQuery(
+ parentScope,
+ register ? usingScope : null,
+ node,
+ enclosingNode,
+ alias,
+ forceNullable);
+ return newNode;
+
+ case OVER:
+ if (!shouldAllowOverRelation()) {
+ throw Util.unexpected(kind);
+ }
+ call = (SqlCall) node;
+ final OverScope overScope = new OverScope(usingScope, call);
+ scopes.put(call, overScope);
+ operand = call.operand(0);
+ newOperand =
+ registerFrom(
+ parentScope,
+ overScope,
+ true,
+ operand,
+ enclosingNode,
+ alias,
+ extendList,
+ forceNullable,
+ lateral);
+ if (newOperand != operand) {
+ call.setOperand(0, newOperand);
+ }
+
+ for (ScopeChild child : overScope.children) {
+ registerNamespace(register ? usingScope : null, child.name,
+ child.namespace, forceNullable);
+ }
+
+ return newNode;
+
+ case EXTEND:
+ final SqlCall extend = (SqlCall) node;
+ return registerFrom(parentScope,
+ usingScope,
+ true,
+ extend.getOperandList().get(0),
+ extend,
+ alias,
+ (SqlNodeList) extend.getOperandList().get(1),
+ forceNullable,
+ lateral);
+
+ default:
+ throw Util.unexpected(kind);
+ }
+ }
+
+ private static boolean isLateral(SqlNode node) {
+ switch (node.getKind()) {
+ case LATERAL:
+ case UNNEST:
+ // Per SQL std, UNNEST is implicitly LATERAL.
+ return true;
+ case AS:
+ return isLateral(((SqlCall) node).operand(0));
+ default:
+ return false;
+ }
+ }
+
+ protected boolean shouldAllowOverRelation() {
+ return false;
+ }
+
+ /**
+ * Creates a namespace for a <code>SELECT</code> node. Derived class may
+ * override this factory method.
+ *
+ * @param select Select node
+ * @param enclosingNode Enclosing node
+ * @return Select namespace
+ */
+ protected SelectNamespace createSelectNamespace(
+ SqlSelect select,
+ SqlNode enclosingNode) {
+ return new SelectNamespace(this, select, enclosingNode);
+ }
+
+ /**
+ * Creates a namespace for a set operation (<code>UNION</code>, <code>
+ * INTERSECT</code>, or <code>EXCEPT</code>). Derived class may override
+ * this factory method.
+ *
+ * @param call Call to set operation
+ * @param enclosingNode Enclosing node
+ * @return Set operation namespace
+ */
+ protected SetopNamespace createSetopNamespace(
+ SqlCall call,
+ SqlNode enclosingNode) {
+ return new SetopNamespace(this, call, enclosingNode);
+ }
+
+ /**
+ * Registers a query in a parent scope.
+ *
+ * @param parentScope Parent scope which this scope turns to in order to
+ * resolve objects
+ * @param usingScope Scope whose child list this scope should add itself to
+ * @param node Query node
+ * @param alias Name of this query within its parent. Must be specified
+ * if usingScope != null
+ */
+ private void registerQuery(
+ SqlValidatorScope parentScope,
+ SqlValidatorScope usingScope,
+ SqlNode node,
+ SqlNode enclosingNode,
+ String alias,
+ boolean forceNullable) {
+ Preconditions.checkArgument(usingScope == null || alias != null);
+ registerQuery(
+ parentScope,
+ usingScope,
+ node,
+ enclosingNode,
+ alias,
+ forceNullable,
+ true);
+ }
+
+ /**
+ * Registers a query in a parent scope.
+ *
+ * @param parentScope Parent scope which this scope turns to in order to
+ * resolve objects
+ * @param usingScope Scope whose child list this scope should add itself to
+ * @param node Query node
+ * @param alias Name of this query within its parent. Must be specified
+ * if usingScope != null
+ * @param checkUpdate if true, validate that the update feature is supported
+ * if validating the update statement
+ */
+ private void registerQuery(
+ SqlValidatorScope parentScope,
+ SqlValidatorScope usingScope,
+ SqlNode node,
+ SqlNode enclosingNode,
+ String alias,
+ boolean forceNullable,
+ boolean checkUpdate) {
+ Objects.requireNonNull(node);
+ Objects.requireNonNull(enclosingNode);
+ Preconditions.checkArgument(usingScope == null || alias != null);
+
+ SqlCall call;
+ List<SqlNode> operands;
+ switch (node.getKind()) {
+ case SELECT:
+ final SqlSelect select = (SqlSelect) node;
+ final SelectNamespace selectNs =
+ createSelectNamespace(select, enclosingNode);
+ registerNamespace(usingScope, alias, selectNs, forceNullable);
+ final SqlValidatorScope windowParentScope =
+ (usingScope != null) ? usingScope : parentScope;
+ SelectScope selectScope =
+ new SelectScope(parentScope, windowParentScope, select);
+ scopes.put(select, selectScope);
+
+ // Start by registering the WHERE clause
+ whereScopes.put(select, selectScope);
+ registerOperandSubQueries(
+ selectScope,
+ select,
+ SqlSelect.WHERE_OPERAND);
+
+ // Register FROM with the inherited scope 'parentScope', not
+ // 'selectScope', otherwise tables in the FROM clause would be
+ // able to see each other.
+ final SqlNode from = select.getFrom();
+ if (from != null) {
+ final SqlNode newFrom =
+ registerFrom(
+ parentScope,
+ selectScope,
+ true,
+ from,
+ from,
+ null,
+ null,
+ false,
+ false);
+ if (newFrom != from) {
+ select.setFrom(newFrom);
+ }
+ }
+
+ // If this is an aggregating query, the SELECT list and HAVING
+ // clause use a different scope, where you can only reference
+ // columns which are in the GROUP BY clause.
+ SqlValidatorScope aggScope = selectScope;
+ if (isAggregate(select)) {
+ aggScope =
+ new AggregatingSelectScope(selectScope, select, false);
+ selectScopes.put(select, aggScope);
+ } else {
+ selectScopes.put(select, selectScope);
+ }
+ if (select.getGroup() != null) {
+ GroupByScope groupByScope =
+ new GroupByScope(selectScope, select.getGroup(), select);
+ groupByScopes.put(select, groupByScope);
+ registerSubQueries(groupByScope, select.getGroup());
+ }
+ registerOperandSubQueries(
+ aggScope,
+ select,
+ SqlSelect.HAVING_OPERAND);
+ registerSubQueries(aggScope, select.getSelectList());
+ final SqlNodeList orderList = select.getOrderList();
+ if (orderList != null) {
+ // If the query is 'SELECT DISTINCT', restrict the columns
+ // available to the ORDER BY clause.
+ if (select.isDistinct()) {
+ aggScope =
+ new AggregatingSelectScope(selectScope, select, true);
+ }
+ OrderByScope orderScope =
+ new OrderByScope(aggScope, orderList, select);
+ orderScopes.put(select, orderScope);
+ registerSubQueries(orderScope, orderList);
+
+ if (!isAggregate(select)) {
+ // Since this is not an aggregating query,
+ // there cannot be any aggregates in the ORDER BY clause.
+ SqlNode agg = aggFinder.findAgg(orderList);
+ if (agg != null) {
+ throw newValidationError(agg, RESOURCE.aggregateIllegalInOrderBy());
+ }
+ }
+ }
+ break;
+
+ case INTERSECT:
+ validateFeature(RESOURCE.sQLFeature_F302(), node.getParserPosition());
+ registerSetop(
+ parentScope,
+ usingScope,
+ node,
+ node,
+ alias,
+ forceNullable);
+ break;
+
+ case EXCEPT:
+ validateFeature(RESOURCE.sQLFeature_E071_03(), node.getParserPosition());
+ registerSetop(
+ parentScope,
+ usingScope,
+ node,
+ node,
+ alias,
+ forceNullable);
+ break;
+
+ case UNION:
+ registerSetop(
+ parentScope,
+ usingScope,
+ node,
+ node,
+ alias,
+ forceNullable);
+ break;
+
+ case WITH:
+ registerWith(parentScope, usingScope, (SqlWith) node, enclosingNode,
+ alias, forceNullable, checkUpdate);
+ break;
+
+ case VALUES:
+ call = (SqlCall) node;
+ scopes.put(call, parentScope);
+ final TableConstructorNamespace tableConstructorNamespace =
+ new TableConstructorNamespace(
+ this,
+ call,
+ parentScope,
+ enclosingNode);
+ registerNamespace(
+ usingScope,
+ alias,
+ tableConstructorNamespace,
+ forceNullable);
+ operands = call.getOperandList();
+ for (int i = 0; i < operands.size(); ++i) {
+ assert operands.get(i).getKind() == SqlKind.ROW;
+
+ // FIXME jvs 9-Feb-2005: Correlation should
+ // be illegal in these sub-queries. Same goes for
+ // any non-lateral SELECT in the FROM list.
+ registerOperandSubQueries(parentScope, call, i);
+ }
+ break;
+
+ case INSERT:
+ SqlInsert insertCall = (SqlInsert) node;
+ InsertNamespace insertNs =
+ new InsertNamespace(
+ this,
+ insertCall,
+ enclosingNode,
+ parentScope);
+ registerNamespace(usingScope, null, insertNs, forceNullable);
+ registerQuery(
+ parentScope,
+ usingScope,
+ insertCall.getSource(),
+ enclosingNode,
+ null,
+ false);
+ break;
+
+ case DELETE:
+ SqlDelete deleteCall = (SqlDelete) node;
+ DeleteNamespace deleteNs =
+ new DeleteNamespace(
+ this,
+ deleteCall,
+ enclosingNode,
+ parentScope);
+ registerNamespace(usingScope, null, deleteNs, forceNullable);
+ registerQuery(
+ parentScope,
+ usingScope,
+ deleteCall.getSourceSelect(),
+ enclosingNode,
+ null,
+ false);
+ break;
+
+ case UPDATE:
+ if (checkUpdate) {
+ validateFeature(RESOURCE.sQLFeature_E101_03(),
+ node.getParserPosition());
+ }
+ SqlUpdate updateCall = (SqlUpdate) node;
+ UpdateNamespace updateNs =
+ new UpdateNamespace(
+ this,
+ updateCall,
+ enclosingNode,
+ parentScope);
+ registerNamespace(usingScope, null, updateNs, forceNullable);
+ registerQuery(
+ parentScope,
+ usingScope,
+ updateCall.getSourceSelect(),
+ enclosingNode,
+ null,
+ false);
+ break;
+
+ case MERGE:
+ validateFeature(RESOURCE.sQLFeature_F312(), node.getParserPosition());
+ SqlMerge mergeCall = (SqlMerge) node;
+ MergeNamespace mergeNs =
+ new MergeNamespace(
+ this,
+ mergeCall,
+ enclosingNode,
+ parentScope);
+ registerNamespace(usingScope, null, mergeNs, forceNullable);
+ registerQuery(
+ parentScope,
+ usingScope,
+ mergeCall.getSourceSelect(),
+ enclosingNode,
+ null,
+ false);
+
+ // update call can reference either the source table reference
+ // or the target table, so set its parent scope to the merge's
+ // source select; when validating the update, skip the feature
+ // validation check
+ if (mergeCall.getUpdateCall() != null) {
+ registerQuery(
+ whereScopes.get(mergeCall.getSourceSelect()),
+ null,
+ mergeCall.getUpdateCall(),
+ enclosingNode,
+ null,
+ false,
+ false);
+ }
+ if (mergeCall.getInsertCall() != null) {
+ registerQuery(
+ parentScope,
+ null,
+ mergeCall.getInsertCall(),
+ enclosingNode,
+ null,
+ false);
+ }
+ break;
+
+ case UNNEST:
+ call = (SqlCall) node;
+ final UnnestNamespace unnestNs =
+ new UnnestNamespace(this, call, parentScope, enclosingNode);
+ registerNamespace(
+ usingScope,
+ alias,
+ unnestNs,
+ forceNullable);
+ registerOperandSubQueries(parentScope, call, 0);
+ scopes.put(node, parentScope);
+ break;
+
+ case OTHER_FUNCTION:
+ call = (SqlCall) node;
+ ProcedureNamespace procNs =
+ new ProcedureNamespace(
+ this,
+ parentScope,
+ call,
+ enclosingNode);
+ registerNamespace(
+ usingScope,
+ alias,
+ procNs,
+ forceNullable);
+ registerSubQueries(parentScope, call);
+ break;
+
+ case MULTISET_QUERY_CONSTRUCTOR:
+ case MULTISET_VALUE_CONSTRUCTOR:
+ validateFeature(RESOURCE.sQLFeature_S271(), node.getParserPosition());
+ call = (SqlCall) node;
+ CollectScope cs = new CollectScope(parentScope, usingScope, call);
+ final CollectNamespace tableConstructorNs =
+ new CollectNamespace(call, cs, enclosingNode);
+ final String alias2 = deriveAlias(node, nextGeneratedId++);
+ registerNamespace(
+ usingScope,
+ alias2,
+ tableConstructorNs,
+ forceNullable);
+ operands = call.getOperandList();
+ for (int i = 0; i < operands.size(); i++) {
+ registerOperandSubQueries(parentScope, call, i);
+ }
+ break;
+
+ default:
+ throw Util.unexpected(node.getKind());
+ }
+ }
+
+ private void registerSetop(
+ SqlValidatorScope parentScope,
+ SqlValidatorScope usingScope,
+ SqlNode node,
+ SqlNode enclosingNode,
+ String alias,
+ boolean forceNullable) {
+ SqlCall call = (SqlCall) node;
+ final SetopNamespace setopNamespace =
+ createSetopNamespace(call, enclosingNode);
+ registerNamespace(usingScope, alias, setopNamespace, forceNullable);
+
+ // A setop is in the same scope as its parent.
+ scopes.put(call, parentScope);
+ for (SqlNode operand : call.getOperandList()) {
+ registerQuery(
+ parentScope,
+ null,
+ operand,
+ operand,
+ null,
+ false);
+ }
+ }
+
+ private void registerWith(
+ SqlValidatorScope parentScope,
+ SqlValidatorScope usingScope,
+ SqlWith with,
+ SqlNode enclosingNode,
+ String alias,
+ boolean forceNullable,
+ boolean checkUpdate) {
+ final WithNamespace withNamespace =
+ new WithNamespace(this, with, enclosingNode);
+ registerNamespace(usingScope, alias, withNamespace, forceNullable);
+
+ SqlValidatorScope scope = parentScope;
+ for (SqlNode withItem_ : with.withList) {
+ final SqlWithItem withItem = (SqlWithItem) withItem_;
+ final WithScope withScope = new WithScope(scope, withItem);
+ scopes.put(withItem, withScope);
+
+ registerQuery(scope, null, withItem.query, with,
+ withItem.name.getSimple(), false);
+ registerNamespace(null, alias,
+ new WithItemNamespace(this, withItem, enclosingNode),
+ false);
+ scope = withScope;
+ }
+
+ registerQuery(scope, null, with.body, enclosingNode, alias, forceNullable,
+ checkUpdate);
+ }
+
+ public boolean isAggregate(SqlSelect select) {
+ if (getAggregate(select) != null) {
+ return true;
+ }
+ // Also when nested window aggregates are present
+ for (SqlCall call : overFinder.findAll(select.getSelectList())) {
+ assert call.getKind() == SqlKind.OVER;
+ if (isNestedAggregateWindow(call.operand(0))) {
+ return true;
+ }
+ if (isOverAggregateWindow(call.operand(1))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ protected boolean isNestedAggregateWindow(SqlNode node) {
+ AggFinder nestedAggFinder =
+ new AggFinder(opTab, false, false, false, aggFinder);
+ return nestedAggFinder.findAgg(node) != null;
+ }
+
+ protected boolean isOverAggregateWindow(SqlNode node) {
+ return aggFinder.findAgg(node) != null;
+ }
+
+ /** Returns the parse tree node (GROUP BY, HAVING, or an aggregate function
+ * call) that causes {@code select} to be an aggregate query, or null if it
+ * is not an aggregate query.
+ *
+ * <p>The node is useful context for error messages,
+ * but you cannot assume that the node is the only aggregate function. */
+ protected SqlNode getAggregate(SqlSelect select) {
+ SqlNode node = select.getGroup();
+ if (node != null) {
+ return node;
+ }
+ node = select.getHaving();
+ if (node != null) {
+ return node;
+ }
+ return getAgg(select);
+ }
+
+ /** If there is at least one call to an aggregate function, returns the
+ * first. */
+ private SqlNode getAgg(SqlSelect select) {
+ final SelectScope selectScope = getRawSelectScope(select);
+ if (selectScope != null) {
+ final List<SqlNode> selectList = selectScope.getExpandedSelectList();
+ if (selectList != null) {
+ return aggFinder.findAgg(selectList);
+ }
+ }
+ return aggFinder.findAgg(select.getSelectList());
+ }
+
+ @SuppressWarnings("deprecation")
+ public boolean isAggregate(SqlNode selectNode) {
+ return aggFinder.findAgg(selectNode) != null;
+ }
+
+ private void validateNodeFeature(SqlNode node) {
+ switch (node.getKind()) {
+ case MULTISET_VALUE_CONSTRUCTOR:
+ validateFeature(RESOURCE.sQLFeature_S271(), node.getParserPosition());
+ break;
+ }
+ }
+
+ private void registerSubQueries(
+ SqlValidatorScope parentScope,
+ SqlNode node) {
+ if (node == null) {
+ return;
+ }
+ if (node.getKind().belongsTo(SqlKind.QUERY)
+ || node.getKind() == SqlKind.MULTISET_QUERY_CONSTRUCTOR
+ || node.getKind() == SqlKind.MULTISET_VALUE_CONSTRUCTOR) {
+ registerQuery(parentScope, null, node, node, null, false);
+ } else if (node instanceof SqlCall) {
+ validateNodeFeature(node);
+ SqlCall call = (SqlCall) node;
+ for (int i = 0; i < call.operandCount(); i++) {
+ registerOperandSubQueries(parentScope, call, i);
+ }
+ } else if (node instanceof SqlNodeList) {
+ SqlNodeList list = (SqlNodeList) node;
+ for (int i = 0, count = list.size(); i < count; i++) {
+ SqlNode listNode = list.get(i);
+ if (listNode.getKind().belongsTo(SqlKind.QUERY)) {
+ listNode =
+ SqlStdOperatorTable.SCALAR_QUERY.createCall(
+ listNode.getParserPosition(),
+ listNode);
+ list.set(i, listNode);
+ }
+ registerSubQueries(parentScope, listNode);
+ }
+ } else {
+ // atomic node -- can be ignored
+ }
+ }
+
+ /**
+ * Registers any sub-queries inside a given call operand, and converts the
+ * operand to a scalar sub-query if the operator requires it.
+ *
+ * @param parentScope Parent scope
+ * @param call Call
+ * @param operandOrdinal Ordinal of operand within call
+ * @see SqlOperator#argumentMustBeScalar(int)
+ */
+ private void registerOperandSubQueries(
+ SqlValidatorScope parentScope,
+ SqlCall call,
+ int operandOrdinal) {
+ SqlNode operand = call.operand(operandOrdinal);
+ if (operand == null) {
+ return;
+ }
+ if (operand.getKind().belongsTo(SqlKind.QUERY)
+ && call.getOperator().argumentMustBeScalar(operandOrdinal)) {
+ operand =
+ SqlStdOperatorTable.SCALAR_QUERY.createCall(
+ operand.getParserPosition(),
+ operand);
+ call.setOperand(operandOrdinal, operand);
+ }
+ registerSubQueries(parentScope, operand);
+ }
+
+ public void validateIdentifier(SqlIdentifier id, SqlValidatorScope scope) {
+ final SqlQualified fqId = scope.fullyQualify(id);
+ if (expandColumnReferences) {
+ // NOTE jvs 9-Apr-2007: this doesn't cover ORDER BY, which has its
+ // own ideas about qualification.
+ id.assignNamesFrom(fqId.identifier);
+ } else {
+ Util.discard(fqId);
+ }
+ }
+
+ public void validateLiteral(SqlLiteral literal) {
+ switch (literal.getTypeName()) {
+ case DECIMAL:
+ // Decimal and long have the same precision (as 64-bit integers), so
+ // the unscaled value of a decimal must fit into a long.
+
+ // REVIEW jvs 4-Aug-2004: This should probably be calling over to
+ // the available calculator implementations to see what they
+ // support. For now use ESP instead.
+ //
+ // jhyde 2006/12/21: I think the limits should be baked into the
+ // type system, not dependent on the calculator implementation.
+ BigDecimal bd = (BigDecimal) literal.getValue();
+ BigInteger unscaled = bd.unscaledValue();
+ long longValue = unscaled.longValue();
+ if (!BigInteger.valueOf(longValue).equals(unscaled)) {
+ // overflow
+ throw newValidationError(literal,
+ RESOURCE.numberLiteralOutOfRange(bd.toString()));
+ }
+ break;
+
+ case DOUBLE:
+ validateLiteralAsDouble(literal);
+ break;
+
+ case BINARY:
+ final BitString bitString = (BitString) literal.getValue();
+ if ((bitString.getBitCount() % 8) != 0) {
+ throw newValidationError(literal, RESOURCE.binaryLiteralOdd());
+ }
+ break;
+
+ case DATE:
+ case TIME:
+ case TIMESTAMP:
+ Calendar calendar = literal.getValueAs(Calendar.class);
+ final int year = calendar.get(Calendar.YEAR);
+ final int era = calendar.get(Calendar.ERA);
+ if (year < 1 || era == GregorianCalendar.BC || year > 9999) {
+ throw newValidationError(literal,
+ RESOURCE.dateLiteralOutOfRange(literal.toString()));
+ }
+ break;
+
+ case INTERVAL_YEAR:
+ case INTERVAL_YEAR_MONTH:
+ case INTERVAL_MONTH:
+ case INTERVAL_DAY:
+ case INTERVAL_DAY_HOUR:
+ case INTERVAL_DAY_MINUTE:
+ case INTERVAL_DAY_SECOND:
+ case INTERVAL_HOUR:
+ case INTERVAL_HOUR_MINUTE:
+ case INTERVAL_HOUR_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_MINUTE_SECOND:
+ case INTERVAL_SECOND:
+ if (literal instanceof SqlIntervalLiteral) {
+ SqlIntervalLiteral.IntervalValue interval =
+ (SqlIntervalLiteral.IntervalValue)
+ literal.getValue();
+ SqlIntervalQualifier intervalQualifier =
+ interval.getIntervalQualifier();
+
+ // ensure qualifier is good before attempting to validate literal
+ validateIntervalQualifier(intervalQualifier);
+ String intervalStr = interval.getIntervalLiteral();
+ // throws CalciteContextException if string is invalid
+ int[] values = intervalQualifier.evaluateIntervalLiteral(intervalStr,
+ literal.getParserPosition(), typeFactory.getTypeSystem());
+ Util.discard(values);
+ }
+ break;
+ default:
+ // default is to do nothing
+ }
+ }
+
+ private void validateLiteralAsDouble(SqlLiteral literal) {
+ BigDecimal bd = (BigDecimal) literal.getValue();
+ double d = bd.doubleValue();
+ if (Double.isInfinite(d) || Double.isNaN(d)) {
+ // overflow
+ throw newValidationError(literal,
+ RESOURCE.numberLiteralOutOfRange(Util.toScientificNotation(bd)));
+ }
+
+ // REVIEW jvs 4-Aug-2004: what about underflow?
+ }
+
+ public void validateIntervalQualifier(SqlIntervalQualifier qualifier) {
+ assert qualifier != null;
+ boolean startPrecisionOutOfRange = false;
+ boolean fractionalSecondPrecisionOutOfRange = false;
+ final RelDataTypeSystem typeSystem = typeFactory.getTypeSystem();
+
+ final int startPrecision = qualifier.getStartPrecision(typeSystem);
+ final int fracPrecision =
+ qualifier.getFractionalSecondPrecision(typeSystem);
+ final int maxPrecision = typeSystem.getMaxPrecision(qualifier.typeName());
+ final int minPrecision = qualifier.typeName().getMinPrecision();
+ final int minScale = qualifier.typeName().getMinScale();
+ final int maxScale = typeSystem.getMaxScale(qualifier.typeName());
+ if (qualifier.isYearMonth()) {
+ if (startPrecision < minPrecision || startPrecision > maxPrecision) {
+ startPrecisionOutOfRange = true;
+ } else {
+ if (fracPrecision < minScale || fracPrecision > maxScale) {
+ fractionalSecondPrecisionOutOfRange = true;
+ }
+ }
+ } else {
+ if (startPrecision < minPrecision || startPrecision > maxPrecision) {
+ startPrecisionOutOfRange = true;
+ } else {
+ if (fracPrecision < minScale || fracPrecision > maxScale) {
+ fractionalSecondPrecisionOutOfRange = true;
+ }
+ }
+ }
+
+ if (startPrecisionOutOfRange) {
+ throw newValidationError(qualifier,
+ RESOURCE.intervalStartPrecisionOutOfRange(startPrecision,
+ "INTERVAL " + qualifier));
+ } else if (fractionalSecondPrecisionOutOfRange) {
+ throw newValidationError(qualifier,
+ RESOURCE.intervalFractionalSecondPrecisionOutOfRange(
+ fracPrecision,
+ "INTERVAL " + qualifier));
+ }
+ }
+
+ /**
+ * Validates the FROM clause of a query, or (recursively) a child node of
+ * the FROM clause: AS, OVER, JOIN, VALUES, or sub-query.
+ *
+ * @param node Node in FROM clause, typically a table or derived
+ * table
+ * @param targetRowType Desired row type of this expression, or
+ * {@link #unknownType} if not fussy. Must not be null.
+ * @param scope Scope
+ */
+ protected void validateFrom(
+ SqlNode node,
+ RelDataType targetRowType,
+ SqlValidatorScope scope) {
+ Objects.requireNonNull(targetRowType);
+ switch (node.getKind()) {
+ case AS:
+ validateFrom(
+ ((SqlCall) node).operand(0),
+ targetRowType,
+ scope);
+ break;
+ case VALUES:
+ validateValues((SqlCall) node, targetRowType, scope);
+ break;
+ case JOIN:
+ validateJoin((SqlJoin) node, scope);
+ break;
+ case OVER:
+ validateOver((SqlCall) node, scope);
+ break;
+ default:
+ validateQuery(node, scope, targetRowType);
+ break;
+ }
+
+ // Validate the namespace representation of the node, just in case the
+ // validation did not occur implicitly.
+ getNamespace(node, scope).validate(targetRowType);
+ }
+
+ protected void validateOver(SqlCall call, SqlValidatorScope scope) {
+ throw new AssertionError("OVER unexpected in this context");
+ }
+
+ private void checkRollUpInUsing(SqlIdentifier identifier, SqlNode leftOrRight) {
+ leftOrRight = stripAs(leftOrRight);
+ // if it's not a SqlIdentifier then that's fine, it'll be validated somewhere else.
+ if (leftOrRight instanceof SqlIdentifier) {
+ SqlIdentifier from = (SqlIdentifier) leftOrRight;
+ Table table = findTable(catalogReader.getRootSchema(), Util.last(from.names),
+ catalogReader.nameMatcher().isCaseSensitive());
+ String name = Util.last(identifier.names);
+
+ if (table != null && table.isRolledUp(name)) {
+ throw newValidationError(identifier, RESOURCE.rolledUpNotAllowed(name, "USING"));
+ }
+ }
+ }
+
+ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) {
+ SqlNode left = join.getLeft();
+ SqlNode right = join.getRight();
+ SqlNode condition = join.getCondition();
+ boolean natural = join.isNatural();
+ final JoinType joinType = join.getJoinType();
+ final JoinConditionType conditionType = join.getConditionType();
+ final SqlValidatorScope joinScope = scopes.get(join);
+ validateFrom(left, unknownType, joinScope);
+ validateFrom(right, unknownType, joinScope);
+
+ // Validate condition.
+ switch (conditionType) {
+ case NONE:
+ Preconditions.checkArgument(condition == null);
+ break;
+ case ON:
+ Preconditions.checkArgument(condition != null);
+ SqlNode expandedCondition = expand(condition, joinScope);
+ join.setOperand(5, expandedCondition);
+ condition = join.getCondition();
+ validateWhereOrOn(joinScope, condition, "ON");
+ checkRollUp(null, join, condition, joinScope, "ON");
+ break;
+ case USING:
+ SqlNodeList list = (SqlNodeList) condition;
+
+ // Parser ensures that using clause is not empty.
+ Preconditions.checkArgument(list.size() > 0, "Empty USING clause");
+ for (SqlNode node : list) {
+ SqlIdentifier id = (SqlIdentifier) node;
+ final RelDataType leftColType = validateUsingCol(id, left);
+ final RelDataType rightColType = validateUsingCol(id, right);
+ if (!SqlTypeUtil.isComparable(leftColType, rightColType)) {
+ throw newValidationError(id,
+ RESOURCE.naturalOrUsingColumnNotCompatible(id.getSimple(),
+ leftColType.toString(), rightColType.toString()));
+ }
+ checkRollUpInUsing(id, left);
+ checkRollUpInUsing(id, right);
+ }
+ break;
+ default:
+ throw Util.unexpected(conditionType);
+ }
+
+ // Validate NATURAL.
+ if (natural) {
+ if (condition != null) {
+ throw newValidationError(condition,
+ RESOURCE.naturalDisallowsOnOrUsing());
+ }
+
+ // Join on fields that occur exactly once on each side. Ignore
+ // fields that occur more than once on either side.
+ final RelDataType leftRowType = getNamespace(left).getRowType();
+ final RelDataType rightRowType = getNamespace(right).getRowType();
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ List<String> naturalColumnNames =
+ SqlValidatorUtil.deriveNaturalJoinColumnList(nameMatcher,
+ leftRowType, rightRowType);
+
+ // Check compatibility of the chosen columns.
+ for (String name : naturalColumnNames) {
+ final RelDataType leftColType =
+ nameMatcher.field(leftRowType, name).getType();
+ final RelDataType rightColType =
+ nameMatcher.field(rightRowType, name).getType();
+ if (!SqlTypeUtil.isComparable(leftColType, rightColType)) {
+ throw newValidationError(join,
+ RESOURCE.naturalOrUsingColumnNotCompatible(name,
+ leftColType.toString(), rightColType.toString()));
+ }
+ }
+ }
+
+ // Which join types require/allow a ON/USING condition, or allow
+ // a NATURAL keyword?
+ switch (joinType) {
+ case LEFT_SEMI_JOIN:
+ if (!conformance.isLiberal()) {
+ throw newValidationError(join.getJoinTypeNode(),
+ RESOURCE.dialectDoesNotSupportFeature("LEFT SEMI JOIN"));
+ }
+ // fall through
+ case INNER:
+ case LEFT:
+ case RIGHT:
+ case FULL:
+ if ((condition == null) && !natural) {
+ throw newValidationError(join, RESOURCE.joinRequiresCondition());
+ }
+ break;
+ case COMMA:
+ case CROSS:
+ if (condition != null) {
+ throw newValidationError(join.getConditionTypeNode(),
+ RESOURCE.crossJoinDisallowsCondition());
+ }
+ if (natural) {
+ throw newValidationError(join.getConditionTypeNode(),
+ RESOURCE.crossJoinDisallowsCondition());
+ }
+ break;
+ default:
+ throw Util.unexpected(joinType);
+ }
+ }
+
+ /**
+ * Throws an error if there is an aggregate or windowed aggregate in the
+ * given clause.
+ *
+ * @param aggFinder Finder for the particular kind(s) of aggregate function
+ * @param node Parse tree
+ * @param clause Name of clause: "WHERE", "GROUP BY", "ON"
+ */
+ private void validateNoAggs(AggFinder aggFinder, SqlNode node,
+ String clause) {
+ final SqlCall agg = aggFinder.findAgg(node);
+ if (agg == null) {
+ return;
+ }
+ final SqlOperator op = agg.getOperator();
+ if (op == SqlStdOperatorTable.OVER) {
+ throw newValidationError(agg,
+ RESOURCE.windowedAggregateIllegalInClause(clause));
+ } else if (op.isGroup() || op.isGroupAuxiliary()) {
+ throw newValidationError(agg,
+ RESOURCE.groupFunctionMustAppearInGroupByClause(op.getName()));
+ } else {
+ throw newValidationError(agg,
+ RESOURCE.aggregateIllegalInClause(clause));
+ }
+ }
+
+ private RelDataType validateUsingCol(SqlIdentifier id, SqlNode leftOrRight) {
+ if (id.names.size() == 1) {
+ String name = id.names.get(0);
+ final SqlValidatorNamespace namespace = getNamespace(leftOrRight);
+ final RelDataType rowType = namespace.getRowType();
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ final RelDataTypeField field = nameMatcher.field(rowType, name);
+ if (field != null) {
+ if (nameMatcher.frequency(rowType.getFieldNames(), name) > 1) {
+ throw newValidationError(id,
+ RESOURCE.columnInUsingNotUnique(id.toString()));
+ }
+ return field.getType();
+ }
+ }
+ throw newValidationError(id, RESOURCE.columnNotFound(id.toString()));
+ }
+
+ /**
+ * Validates a SELECT statement.
+ *
+ * @param select Select statement
+ * @param targetRowType Desired row type, must not be null, may be the data
+ * type 'unknown'.
+ */
+ protected void validateSelect(
+ SqlSelect select,
+ RelDataType targetRowType) {
+ assert targetRowType != null;
+ // Namespace is either a select namespace or a wrapper around one.
+ final SelectNamespace ns =
+ getNamespace(select).unwrap(SelectNamespace.class);
+
+ // Its rowtype is null, meaning it hasn't been validated yet.
+ // This is important, because we need to take the targetRowType into
+ // account.
+ assert ns.rowType == null;
+
+ if (select.isDistinct()) {
+ validateFeature(RESOURCE.sQLFeature_E051_01(),
+ select.getModifierNode(SqlSelectKeyword.DISTINCT)
+ .getParserPosition());
+ }
+
+ final SqlNodeList selectItems = select.getSelectList();
+ RelDataType fromType = unknownType;
+ if (selectItems.size() == 1) {
+ final SqlNode selectItem = selectItems.get(0);
+ if (selectItem instanceof SqlIdentifier) {
+ SqlIdentifier id = (SqlIdentifier) selectItem;
+ if (id.isStar() && (id.names.size() == 1)) {
+ // Special case: for INSERT ... VALUES(?,?), the SQL
+ // standard says we're supposed to propagate the target
+ // types down. So iff the select list is an unqualified
+ // star (as it will be after an INSERT ... VALUES has been
+ // expanded), then propagate.
+ fromType = targetRowType;
+ }
+ }
+ }
+
+ // Make sure that items in FROM clause have distinct aliases.
+ final SelectScope fromScope = (SelectScope) getFromScope(select);
+ List<String> names = fromScope.getChildNames();
+ if (!catalogReader.nameMatcher().isCaseSensitive()) {
+ names = Lists.transform(names, s -> s.toUpperCase(Locale.ROOT));
+ }
+ final int duplicateAliasOrdinal = Util.firstDuplicate(names);
+ if (duplicateAliasOrdinal >= 0) {
+ final ScopeChild child =
+ fromScope.children.get(duplicateAliasOrdinal);
+ throw newValidationError(child.namespace.getEnclosingNode(),
+ RESOURCE.fromAliasDuplicate(child.name));
+ }
+
+ if (select.getFrom() == null) {
+ if (conformance.isFromRequired()) {
+ throw newValidationError(select, RESOURCE.selectMissingFrom());
+ }
+ } else {
+ validateFrom(select.getFrom(), fromType, fromScope);
+ }
+
+ validateWhereClause(select);
+ validateGroupClause(select);
+ validateHavingClause(select);
+ validateWindowClause(select);
+ handleOffsetFetch(select.getOffset(), select.getFetch());
+
+ // Validate the SELECT clause late, because a select item might
+ // depend on the GROUP BY list, or the window function might reference
+ // window name in the WINDOW clause etc.
+ final RelDataType rowType =
+ validateSelectList(selectItems, select, targetRowType);
+ ns.setType(rowType);
+
+ // Validate ORDER BY after we have set ns.rowType because in some
+ // dialects you can refer to columns of the select list, e.g.
+ // "SELECT empno AS x FROM emp ORDER BY x"
+ validateOrderList(select);
+
+ if (shouldCheckForRollUp(select.getFrom())) {
+ checkRollUpInSelectList(select);
+ checkRollUp(null, select, select.getWhere(), getWhereScope(select));
+ checkRollUp(null, select, select.getHaving(), getHavingScope(select));
+ checkRollUpInWindowDecl(select);
+ checkRollUpInGroupBy(select);
+ checkRollUpInOrderBy(select);
+ }
+ }
+
+ private void checkRollUpInSelectList(SqlSelect select) {
+ SqlValidatorScope scope = getSelectScope(select);
+ for (SqlNode item : select.getSelectList()) {
+ checkRollUp(null, select, item, scope);
+ }
+ }
+
+ private void checkRollUpInGroupBy(SqlSelect select) {
+ SqlNodeList group = select.getGroup();
+ if (group != null) {
+ for (SqlNode node : group) {
+ checkRollUp(null, select, node, getGroupScope(select), "GROUP BY");
+ }
+ }
+ }
+
+ private void checkRollUpInOrderBy(SqlSelect select) {
+ SqlNodeList orderList = select.getOrderList();
+ if (orderList != null) {
+ for (SqlNode node : orderList) {
+ checkRollUp(null, select, node, getOrderScope(select), "ORDER BY");
+ }
+ }
+ }
+
+ private void checkRollUpInWindow(SqlWindow window, SqlValidatorScope scope) {
+ if (window != null) {
+ for (SqlNode node : window.getPartitionList()) {
+ checkRollUp(null, window, node, scope, "PARTITION BY");
+ }
+
+ for (SqlNode node : window.getOrderList()) {
+ checkRollUp(null, window, node, scope, "ORDER BY");
+ }
+ }
+ }
+
+ private void checkRollUpInWindowDecl(SqlSelect select) {
+ for (SqlNode decl : select.getWindowList()) {
+ checkRollUpInWindow((SqlWindow) decl, getSelectScope(select));
+ }
+ }
+
+ private SqlNode stripDot(SqlNode node) {
+ if (node != null && node.getKind() == SqlKind.DOT) {
+ return stripDot(((SqlCall) node).operand(0));
+ }
+ return node;
+ }
+
+ private void checkRollUp(SqlNode grandParent, SqlNode parent,
+ SqlNode current, SqlValidatorScope scope, String optionalClause) {
+ current = stripAs(current);
+ if (current instanceof SqlCall && !(current instanceof SqlSelect)) {
+ // Validate OVER separately
+ checkRollUpInWindow(getWindowInOver(current), scope);
+ current = stripOver(current);
+
+ List<SqlNode> children = ((SqlCall) stripDot(current)).getOperandList();
+ for (SqlNode child : children) {
+ checkRollUp(parent, current, child, scope, optionalClause);
+ }
+ } else if (current instanceof SqlIdentifier) {
+ SqlIdentifier id = (SqlIdentifier) current;
+ if (!id.isStar() && isRolledUpColumn(id, scope)) {
+ if (!isAggregation(parent.getKind())
+ || !isRolledUpColumnAllowedInAgg(id, scope, (SqlCall) parent, grandParent)) {
+ String context = optionalClause != null ? optionalClause : parent.getKind().toString();
+ throw newValidationError(id,
+ RESOURCE.rolledUpNotAllowed(deriveAlias(id, 0), context));
+ }
+ }
+ }
+ }
+
+ private void checkRollUp(SqlNode grandParent, SqlNode parent,
+ SqlNode current, SqlValidatorScope scope) {
+ checkRollUp(grandParent, parent, current, scope, null);
+ }
+
+ private SqlWindow getWindowInOver(SqlNode over) {
+ if (over.getKind() == SqlKind.OVER) {
+ SqlNode window = ((SqlCall) over).getOperandList().get(1);
+ if (window instanceof SqlWindow) {
+ return (SqlWindow) window;
+ }
+ // SqlIdentifier, gets validated elsewhere
+ return null;
+ }
+ return null;
+ }
+
+ private static SqlNode stripOver(SqlNode node) {
+ switch (node.getKind()) {
+ case OVER:
+ return ((SqlCall) node).getOperandList().get(0);
+ default:
+ return node;
+ }
+ }
+
+ private Pair<String, String> findTableColumnPair(SqlIdentifier identifier,
+ SqlValidatorScope scope) {
+ SqlCall call = SqlUtil.makeCall(getOperatorTable(), identifier);
+ if (call != null) {
+ return null;
+ }
+ SqlQualified qualified = scope.fullyQualify(identifier);
+ List<String> names = qualified.identifier.names;
+
+ if (names.size() < 2) {
+ return null;
+ }
+
+ return new Pair<>(names.get(names.size() - 2), Util.last(names));
+ }
+
+ // Returns true iff the given column is valid inside the given aggCall.
+ private boolean isRolledUpColumnAllowedInAgg(SqlIdentifier identifier, SqlValidatorScope scope,
+ SqlCall aggCall, SqlNode parent) {
+ Pair<String, String> pair = findTableColumnPair(identifier, scope);
+
+ if (pair == null) {
+ return true;
+ }
+
+ String tableAlias = pair.left;
+ String columnName = pair.right;
+
+ Table table = findTable(tableAlias);
+ if (table != null) {
+ return table.rolledUpColumnValidInsideAgg(columnName, aggCall, parent,
+ catalogReader.getConfig());
+ }
+ return true;
+ }
+
+
+ // Returns true iff the given column is actually rolled up.
+ private boolean isRolledUpColumn(SqlIdentifier identifier, SqlValidatorScope scope) {
+ Pair<String, String> pair = findTableColumnPair(identifier, scope);
+
+ if (pair == null) {
+ return false;
+ }
+
+ String tableAlias = pair.left;
+ String columnName = pair.right;
+
+ Table table = findTable(tableAlias);
+ if (table != null) {
+ return table.isRolledUp(columnName);
+ }
+ return false;
+ }
+
+ private Table findTable(CalciteSchema schema, String tableName, boolean caseSensitive) {
+ CalciteSchema.TableEntry entry = schema.getTable(tableName, caseSensitive);
+ if (entry != null) {
+ return entry.getTable();
+ }
+
+ // Check sub schemas
+ for (CalciteSchema subSchema : schema.getSubSchemaMap().values()) {
+ Table table = findTable(subSchema, tableName, caseSensitive);
+ if (table != null) {
+ return table;
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Given a table alias, find the corresponding {@link Table} associated with it
+ * */
+ private Table findTable(String alias) {
+ List<String> names = null;
+ if (tableScope == null) {
+ // no tables to find
+ return null;
+ }
+
+ for (ScopeChild child : tableScope.children) {
+ if (catalogReader.nameMatcher().matches(child.name, alias)) {
+ names = ((SqlIdentifier) child.namespace.getNode()).names;
+ break;
+ }
+ }
+ if (names == null || names.size() == 0) {
+ return null;
+ } else if (names.size() == 1) {
+ return findTable(catalogReader.getRootSchema(), names.get(0),
+ catalogReader.nameMatcher().isCaseSensitive());
+ }
+
+ CalciteSchema.TableEntry entry =
+ SqlValidatorUtil.getTableEntry(catalogReader, names);
+
+ return entry == null ? null : entry.getTable();
+ }
+
+ private boolean shouldCheckForRollUp(SqlNode from) {
+ if (from != null) {
+ SqlKind kind = stripAs(from).getKind();
+ return kind != SqlKind.VALUES && kind != SqlKind.SELECT;
+ }
+ return false;
+ }
+
+ /** Validates that a query can deliver the modality it promises. Only called
+ * on the top-most SELECT or set operator in the tree. */
+ private void validateModality(SqlNode query) {
+ final SqlModality modality = deduceModality(query);
+ if (query instanceof SqlSelect) {
+ final SqlSelect select = (SqlSelect) query;
+ validateModality(select, modality, true);
+ } else if (query.getKind() == SqlKind.VALUES) {
+ switch (modality) {
+ case STREAM:
+ throw newValidationError(query, Static.RESOURCE.cannotStreamValues());
+ }
+ } else {
+ assert query.isA(SqlKind.SET_QUERY);
+ final SqlCall call = (SqlCall) query;
+ for (SqlNode operand : call.getOperandList()) {
+ if (deduceModality(operand) != modality) {
+ throw newValidationError(operand,
+ Static.RESOURCE.streamSetOpInconsistentInputs());
+ }
+ validateModality(operand);
+ }
+ }
+ }
+
+ /** Return the intended modality of a SELECT or set-op. */
+ private SqlModality deduceModality(SqlNode query) {
+ if (query instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) query;
+ return select.getModifierNode(SqlSelectKeyword.STREAM) != null
+ ? SqlModality.STREAM
+ : SqlModality.RELATION;
+ } else if (query.getKind() == SqlKind.VALUES) {
+ return SqlModality.RELATION;
+ } else {
+ assert query.isA(SqlKind.SET_QUERY);
+ final SqlCall call = (SqlCall) query;
+ return deduceModality(call.getOperandList().get(0));
+ }
+ }
+
+ public boolean validateModality(SqlSelect select, SqlModality modality,
+ boolean fail) {
+ final SelectScope scope = getRawSelectScope(select);
+
+ switch (modality) {
+ case STREAM:
+ if (scope.children.size() == 1) {
+ for (ScopeChild child : scope.children) {
+ if (!child.namespace.supportsModality(modality)) {
+ if (fail) {
+ throw newValidationError(child.namespace.getNode(),
+ Static.RESOURCE.cannotConvertToStream(child.name));
+ } else {
+ return false;
+ }
+ }
+ }
+ } else {
+ int supportsModalityCount = 0;
+ for (ScopeChild child : scope.children) {
+ if (child.namespace.supportsModality(modality)) {
+ ++supportsModalityCount;
+ }
+ }
+
+ if (supportsModalityCount == 0) {
+ if (fail) {
+ String inputs = String.join(", ", scope.getChildNames());
+ throw newValidationError(select,
+ Static.RESOURCE.cannotStreamResultsForNonStreamingInputs(inputs));
+ } else {
+ return false;
+ }
+ }
+ }
+ break;
+ default:
+ for (ScopeChild child : scope.children) {
+ if (!child.namespace.supportsModality(modality)) {
+ if (fail) {
+ throw newValidationError(child.namespace.getNode(),
+ Static.RESOURCE.cannotConvertToRelation(child.name));
+ } else {
+ return false;
+ }
+ }
+ }
+ }
+
+ // Make sure that aggregation is possible.
+ final SqlNode aggregateNode = getAggregate(select);
+ if (aggregateNode != null) {
+ switch (modality) {
+ case STREAM:
+ SqlNodeList groupList = select.getGroup();
+ if (groupList == null
+ || !SqlValidatorUtil.containsMonotonic(scope, groupList)) {
+ if (fail) {
+ throw newValidationError(aggregateNode,
+ Static.RESOURCE.streamMustGroupByMonotonic());
+ } else {
+ return false;
+ }
+ }
+ }
+ }
+
+ // Make sure that ORDER BY is possible.
+ final SqlNodeList orderList = select.getOrderList();
+ if (orderList != null && orderList.size() > 0) {
+ switch (modality) {
+ case STREAM:
+ if (!hasSortedPrefix(scope, orderList)) {
+ if (fail) {
+ throw newValidationError(orderList.get(0),
+ Static.RESOURCE.streamMustOrderByMonotonic());
+ } else {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ /** Returns whether the prefix is sorted. */
+ private boolean hasSortedPrefix(SelectScope scope, SqlNodeList orderList) {
+ return isSortCompatible(scope, orderList.get(0), false);
+ }
+
+ private boolean isSortCompatible(SelectScope scope, SqlNode node,
+ boolean descending) {
+ switch (node.getKind()) {
+ case DESCENDING:
+ return isSortCompatible(scope, ((SqlCall) node).getOperandList().get(0),
+ true);
+ }
+ final SqlMonotonicity monotonicity = scope.getMonotonicity(node);
+ switch (monotonicity) {
+ case INCREASING:
+ case STRICTLY_INCREASING:
+ return !descending;
+ case DECREASING:
+ case STRICTLY_DECREASING:
+ return descending;
+ default:
+ return false;
+ }
+ }
+
+ protected void validateWindowClause(SqlSelect select) {
+ final SqlNodeList windowList = select.getWindowList();
+ @SuppressWarnings("unchecked") final List<SqlWindow> windows =
+ (List) windowList.getList();
+ if (windows.isEmpty()) {
+ return;
+ }
+
+ final SelectScope windowScope = (SelectScope) getFromScope(select);
+ assert windowScope != null;
+
+ // 1. ensure window names are simple
+ // 2. ensure they are unique within this scope
+ for (SqlWindow window : windows) {
+ SqlIdentifier declName = window.getDeclName();
+ if (!declName.isSimple()) {
+ throw newValidationError(declName, RESOURCE.windowNameMustBeSimple());
+ }
+
+ if (windowScope.existingWindowName(declName.toString())) {
+ throw newValidationError(declName, RESOURCE.duplicateWindowName());
+ } else {
+ windowScope.addWindowName(declName.toString());
+ }
+ }
+
+ // 7.10 rule 2
+ // Check for pairs of windows which are equivalent.
+ for (int i = 0; i < windows.size(); i++) {
+ SqlNode window1 = windows.get(i);
+ for (int j = i + 1; j < windows.size(); j++) {
+ SqlNode window2 = windows.get(j);
+ if (window1.equalsDeep(window2, Litmus.IGNORE)) {
+ throw newValidationError(window2, RESOURCE.dupWindowSpec());
+ }
+ }
+ }
+
+ for (SqlWindow window : windows) {
+ final SqlNodeList expandedOrderList =
+ (SqlNodeList) expand(window.getOrderList(), windowScope);
+ window.setOrderList(expandedOrderList);
+ expandedOrderList.validate(this, windowScope);
+
+ final SqlNodeList expandedPartitionList =
+ (SqlNodeList) expand(window.getPartitionList(), windowScope);
+ window.setPartitionList(expandedPartitionList);
+ expandedPartitionList.validate(this, windowScope);
+ }
+
+ // Hand off to validate window spec components
+ windowList.validate(this, windowScope);
+ }
+
+ public void validateWith(SqlWith with, SqlValidatorScope scope) {
+ final SqlValidatorNamespace namespace = getNamespace(with);
+ validateNamespace(namespace, unknownType);
+ }
+
+ public void validateWithItem(SqlWithItem withItem) {
+ if (withItem.columnList != null) {
+ final RelDataType rowType = getValidatedNodeType(withItem.query);
+ final int fieldCount = rowType.getFieldCount();
+ if (withItem.columnList.size() != fieldCount) {
+ throw newValidationError(withItem.columnList,
+ RESOURCE.columnCountMismatch());
+ }
+ SqlValidatorUtil.checkIdentifierListForDuplicates(
+ withItem.columnList.getList(), validationErrorFunction);
+ } else {
+ // Luckily, field names have not been make unique yet.
+ final List<String> fieldNames =
+ getValidatedNodeType(withItem.query).getFieldNames();
+ final int i = Util.firstDuplicate(fieldNames);
+ if (i >= 0) {
+ throw newValidationError(withItem.query,
+ RESOURCE.duplicateColumnAndNoColumnList(fieldNames.get(i)));
+ }
+ }
+ }
+
+ public void validateSequenceValue(SqlValidatorScope scope, SqlIdentifier id) {
+ // Resolve identifier as a table.
+ final SqlValidatorScope.ResolvedImpl resolved =
+ new SqlValidatorScope.ResolvedImpl();
+ scope.resolveTable(id.names, catalogReader.nameMatcher(),
+ SqlValidatorScope.Path.EMPTY, resolved);
+ if (resolved.count() != 1) {
+ throw newValidationError(id, RESOURCE.tableNameNotFound(id.toString()));
+ }
+ // We've found a table. But is it a sequence?
+ final SqlValidatorNamespace ns = resolved.only().namespace;
+ if (ns instanceof TableNamespace) {
+ final Table table = ns.getTable().unwrap(Table.class);
+ switch (table.getJdbcTableType()) {
+ case SEQUENCE:
+ case TEMPORARY_SEQUENCE:
+ return;
+ }
+ }
+ throw newValidationError(id, RESOURCE.notASequence(id.toString()));
+ }
+
+ public SqlValidatorScope getWithScope(SqlNode withItem) {
+ assert withItem.getKind() == SqlKind.WITH_ITEM;
+ return scopes.get(withItem);
+ }
+
+ /**
+ * Validates the ORDER BY clause of a SELECT statement.
+ *
+ * @param select Select statement
+ */
+ protected void validateOrderList(SqlSelect select) {
+ // ORDER BY is validated in a scope where aliases in the SELECT clause
+ // are visible. For example, "SELECT empno AS x FROM emp ORDER BY x"
+ // is valid.
+ SqlNodeList orderList = select.getOrderList();
+ if (orderList == null) {
+ return;
+ }
+ if (!shouldAllowIntermediateOrderBy()) {
+ if (!cursorSet.contains(select)) {
+ throw newValidationError(select, RESOURCE.invalidOrderByPos());
+ }
+ }
+ final SqlValidatorScope orderScope = getOrderScope(select);
+ Objects.requireNonNull(orderScope);
+
+ List<SqlNode> expandList = new ArrayList<>();
+ for (SqlNode orderItem : orderList) {
+ SqlNode expandedOrderItem = expand(orderItem, orderScope);
+ expandList.add(expandedOrderItem);
+ }
+
+ SqlNodeList expandedOrderList = new SqlNodeList(
+ expandList,
+ orderList.getParserPosition());
+ select.setOrderBy(expandedOrderList);
+
+ for (SqlNode orderItem : expandedOrderList) {
+ validateOrderItem(select, orderItem);
+ }
+ }
+
+ /**
+ * Validates an item in the GROUP BY clause of a SELECT statement.
+ *
+ * @param select Select statement
+ * @param groupByItem GROUP BY clause item
+ */
+ private void validateGroupByItem(SqlSelect select, SqlNode groupByItem) {
+ final SqlValidatorScope groupByScope = getGroupScope(select);
+ groupByScope.validateExpr(groupByItem);
+ }
+
+ /**
+ * Validates an item in the ORDER BY clause of a SELECT statement.
+ *
+ * @param select Select statement
+ * @param orderItem ORDER BY clause item
+ */
+ private void validateOrderItem(SqlSelect select, SqlNode orderItem) {
+ switch (orderItem.getKind()) {
+ case DESCENDING:
+ validateFeature(RESOURCE.sQLConformance_OrderByDesc(),
+ orderItem.getParserPosition());
+ validateOrderItem(select,
+ ((SqlCall) orderItem).operand(0));
+ return;
+ }
+
+ final SqlValidatorScope orderScope = getOrderScope(select);
+ validateExpr(orderItem, orderScope);
+ }
+
+ public SqlNode expandOrderExpr(SqlSelect select, SqlNode orderExpr) {
+ final SqlNode newSqlNode =
+ new OrderExpressionExpander(select, orderExpr).go();
+ if (newSqlNode != orderExpr) {
+ final SqlValidatorScope scope = getOrderScope(select);
+ inferUnknownTypes(unknownType, scope, newSqlNode);
+ final RelDataType type = deriveType(scope, newSqlNode);
+ setValidatedNodeType(newSqlNode, type);
+ }
+ return newSqlNode;
+ }
+
+ /**
+ * Validates the GROUP BY clause of a SELECT statement. This method is
+ * called even if no GROUP BY clause is present.
+ */
+ protected void validateGroupClause(SqlSelect select) {
+ SqlNodeList groupList = select.getGroup();
+ if (groupList == null) {
+ return;
+ }
+ final String clause = "GROUP BY";
+ validateNoAggs(aggOrOverFinder, groupList, clause);
+ final SqlValidatorScope groupScope = getGroupScope(select);
+ inferUnknownTypes(unknownType, groupScope, groupList);
+
+ // expand the expression in group list.
+ List<SqlNode> expandedList = new ArrayList<>();
+ for (SqlNode groupItem : groupList) {
+ SqlNode expandedItem = expandGroupByOrHavingExpr(groupItem, groupScope, select, false);
+ expandedList.add(expandedItem);
+ }
+ groupList = new SqlNodeList(expandedList, groupList.getParserPosition());
+ select.setGroupBy(groupList);
+ for (SqlNode groupItem : expandedList) {
+ validateGroupByItem(select, groupItem);
+ }
+
+ // Nodes in the GROUP BY clause are expressions except if they are calls
+ // to the GROUPING SETS, ROLLUP or CUBE operators; this operators are not
+ // expressions, because they do not have a type.
+ for (SqlNode node : groupList) {
+ switch (node.getKind()) {
+ case GROUPING_SETS:
+ case ROLLUP:
+ case CUBE:
+ node.validate(this, groupScope);
+ break;
+ default:
+ node.validateExpr(this, groupScope);
+ }
+ }
+
+ // Derive the type of each GROUP BY item. We don't need the type, but
+ // it resolves functions, and that is necessary for deducing
+ // monotonicity.
+ final SqlValidatorScope selectScope = getSelectScope(select);
+ AggregatingSelectScope aggregatingScope = null;
+ if (selectScope instanceof AggregatingSelectScope) {
+ aggregatingScope = (AggregatingSelectScope) selectScope;
+ }
+ for (SqlNode groupItem : groupList) {
+ if (groupItem instanceof SqlNodeList
+ && ((SqlNodeList) groupItem).size() == 0) {
+ continue;
+ }
+ validateGroupItem(groupScope, aggregatingScope, groupItem);
+ }
+
+ SqlNode agg = aggFinder.findAgg(groupList);
+ if (agg != null) {
+ throw newValidationError(agg, RESOURCE.aggregateIllegalInClause(clause));
+ }
+ }
+
+ private void validateGroupItem(SqlValidatorScope groupScope,
+ AggregatingSelectScope aggregatingScope,
+ SqlNode groupItem) {
+ switch (groupItem.getKind()) {
+ case GROUPING_SETS:
+ case ROLLUP:
+ case CUBE:
+ validateGroupingSets(groupScope, aggregatingScope, (SqlCall) groupItem);
+ break;
+ default:
+ if (groupItem instanceof SqlNodeList) {
+ break;
+ }
+ final RelDataType type = deriveType(groupScope, groupItem);
+ setValidatedNodeType(groupItem, type);
+ }
+ }
+
+ private void validateGroupingSets(SqlValidatorScope groupScope,
+ AggregatingSelectScope aggregatingScope, SqlCall groupItem) {
+ for (SqlNode node : groupItem.getOperandList()) {
+ validateGroupItem(groupScope, aggregatingScope, node);
+ }
+ }
+
+ protected void validateWhereClause(SqlSelect select) {
+ // validate WHERE clause
+ final SqlNode where = select.getWhere();
+ if (where == null) {
+ return;
+ }
+ final SqlValidatorScope whereScope = getWhereScope(select);
+ final SqlNode expandedWhere = expand(where, whereScope);
+ select.setWhere(expandedWhere);
+ validateWhereOrOn(whereScope, expandedWhere, "WHERE");
+ }
+
+ protected void validateWhereOrOn(
+ SqlValidatorScope scope,
+ SqlNode condition,
+ String clause) {
+ validateNoAggs(aggOrOverOrGroupFinder, condition, clause);
+ inferUnknownTypes(
+ booleanType,
+ scope,
+ condition);
+ condition.validate(this, scope);
+
+ final RelDataType type = deriveType(scope, condition);
+ if (!SqlTypeUtil.inBooleanFamily(type)) {
+ throw newValidationError(condition, RESOURCE.condMustBeBoolean(clause));
+ }
+ }
+
+ protected void validateHavingClause(SqlSelect select) {
+ // HAVING is validated in the scope after groups have been created.
+ // For example, in "SELECT empno FROM emp WHERE empno = 10 GROUP BY
+ // deptno HAVING empno = 10", the reference to 'empno' in the HAVING
+ // clause is illegal.
+ SqlNode having = select.getHaving();
+ if (having == null) {
+ return;
+ }
+ final AggregatingScope havingScope =
+ (AggregatingScope) getSelectScope(select);
+ if (getConformance().isHavingAlias()) {
+ SqlNode newExpr = expandGroupByOrHavingExpr(having, havingScope, select, true);
+ if (having != newExpr) {
+ having = newExpr;
+ select.setHaving(newExpr);
+ }
+ }
+ havingScope.checkAggregateExpr(having, true);
+ inferUnknownTypes(
+ booleanType,
+ havingScope,
+ having);
+ having.validate(this, havingScope);
+ final RelDataType type = deriveType(havingScope, having);
+ if (!SqlTypeUtil.inBooleanFamily(type)) {
+ throw newValidationError(having, RESOURCE.havingMustBeBoolean());
+ }
+ }
+
+ protected RelDataType validateSelectList(
+ final SqlNodeList selectItems,
+ SqlSelect select,
+ RelDataType targetRowType) {
+ // First pass, ensure that aliases are unique. "*" and "TABLE.*" items
+ // are ignored.
+
+ // Validate SELECT list. Expand terms of the form "*" or "TABLE.*".
+ final SqlValidatorScope selectScope = getSelectScope(select);
+ final List<SqlNode> expandedSelectItems = new ArrayList<>();
+ final Set<String> aliases = new HashSet<>();
+ final List<Map.Entry<String, RelDataType>> fieldList = new ArrayList<>();
+
+ for (int i = 0; i < selectItems.size(); i++) {
+ SqlNode selectItem = selectItems.get(i);
+ if (selectItem instanceof SqlSelect) {
+ handleScalarSubQuery(
+ select,
+ (SqlSelect) selectItem,
+ expandedSelectItems,
+ aliases,
+ fieldList);
+ } else {
+ expandSelectItem(
+ selectItem,
+ select,
+ targetRowType.isStruct()
+ && targetRowType.getFieldCount() >= i
+ ? targetRowType.getFieldList().get(i).getType()
+ : unknownType,
+ expandedSelectItems,
+ aliases,
+ fieldList,
+ false);
+ }
+ }
+
+ // Create the new select list with expanded items. Pass through
+ // the original parser position so that any overall failures can
+ // still reference the original input text.
+ SqlNodeList newSelectList =
+ new SqlNodeList(
+ expandedSelectItems,
+ selectItems.getParserPosition());
+ if (shouldExpandIdentifiers()) {
+ select.setSelectList(newSelectList);
+ }
+ getRawSelectScope(select).setExpandedSelectList(expandedSelectItems);
+
+ // TODO: when SELECT appears as a value sub-query, should be using
+ // something other than unknownType for targetRowType
+ inferUnknownTypes(targetRowType, selectScope, newSelectList);
+
+ for (SqlNode selectItem : expandedSelectItems) {
+ validateNoAggs(groupFinder, selectItem, "SELECT");
+ validateExpr(selectItem, selectScope);
+ }
+
+ return typeFactory.createStructType(fieldList);
+ }
+
+ /**
+ * Validates an expression.
+ *
+ * @param expr Expression
+ * @param scope Scope in which expression occurs
+ */
+ private void validateExpr(SqlNode expr, SqlValidatorScope scope) {
+ if (expr instanceof SqlCall) {
+ final SqlOperator op = ((SqlCall) expr).getOperator();
+ if (op.isAggregator() && op.requiresOver()) {
+ throw newValidationError(expr,
+ RESOURCE.absentOverClause());
+ }
+ }
+
+ // Call on the expression to validate itself.
+ expr.validateExpr(this, scope);
+
+ // Perform any validation specific to the scope. For example, an
+ // aggregating scope requires that expressions are valid aggregations.
+ scope.validateExpr(expr);
+ }
+
+ /**
+ * Processes SubQuery found in Select list. Checks that is actually Scalar
+ * sub-query and makes proper entries in each of the 3 lists used to create
+ * the final rowType entry.
+ *
+ * @param parentSelect base SqlSelect item
+ * @param selectItem child SqlSelect from select list
+ * @param expandedSelectItems Select items after processing
+ * @param aliasList built from user or system values
+ * @param fieldList Built up entries for each select list entry
+ */
+ private void handleScalarSubQuery(
+ SqlSelect parentSelect,
+ SqlSelect selectItem,
+ List<SqlNode> expandedSelectItems,
+ Set<String> aliasList,
+ List<Map.Entry<String, RelDataType>> fieldList) {
+ // A scalar sub-query only has one output column.
+ if (1 != selectItem.getSelectList().size()) {
+ throw newValidationError(selectItem,
+ RESOURCE.onlyScalarSubQueryAllowed());
+ }
+
+ // No expansion in this routine just append to list.
+ expandedSelectItems.add(selectItem);
+
+ // Get or generate alias and add to list.
+ final String alias =
+ deriveAlias(
+ selectItem,
+ aliasList.size());
+ aliasList.add(alias);
+
+ final SelectScope scope = (SelectScope) getWhereScope(parentSelect);
+ final RelDataType type = deriveType(scope, selectItem);
+ setValidatedNodeType(selectItem, type);
+
+ // we do not want to pass on the RelRecordType returned
+ // by the sub query. Just the type of the single expression
+ // in the sub-query select list.
+ assert type instanceof RelRecordType;
+ RelRecordType rec = (RelRecordType) type;
+
+ RelDataType nodeType = rec.getFieldList().get(0).getType();
+ nodeType = typeFactory.createTypeWithNullability(nodeType, true);
+ fieldList.add(Pair.of(alias, nodeType));
+ }
+
+ /**
+ * Derives a row-type for INSERT and UPDATE operations.
+ *
+ * @param table Target table for INSERT/UPDATE
+ * @param targetColumnList List of target columns, or null if not specified
+ * @param append Whether to append fields to those in <code>
+ * baseRowType</code>
+ * @return Rowtype
+ */
+ protected RelDataType createTargetRowType(
+ SqlValidatorTable table,
+ SqlNodeList targetColumnList,
+ boolean append) {
+ RelDataType baseRowType = table.getRowType();
+ if (targetColumnList == null) {
+ return baseRowType;
+ }
+ List<RelDataTypeField> targetFields = baseRowType.getFieldList();
+ final List<Map.Entry<String, RelDataType>> fields = new ArrayList<>();
+ if (append) {
+ for (RelDataTypeField targetField : targetFields) {
+ fields.add(
+ Pair.of(SqlUtil.deriveAliasFromOrdinal(fields.size()),
+ targetField.getType()));
+ }
+ }
+ final Set<Integer> assignedFields = new HashSet<>();
+ final RelOptTable relOptTable = table instanceof RelOptTable
+ ? ((RelOptTable) table) : null;
+ for (SqlNode node : targetColumnList) {
+ SqlIdentifier id = (SqlIdentifier) node;
+ RelDataTypeField targetField =
+ SqlValidatorUtil.getTargetField(
+ baseRowType, typeFactory, id, catalogReader, relOptTable);
+ if (targetField == null) {
+ throw newValidationError(id,
+ RESOURCE.unknownTargetColumn(id.toString()));
+ }
+ if (!assignedFields.add(targetField.getIndex())) {
+ throw newValidationError(id,
+ RESOURCE.duplicateTargetColumn(targetField.getName()));
+ }
+ fields.add(targetField);
+ }
+ return typeFactory.createStructType(fields);
+ }
+
+ public void validateInsert(SqlInsert insert) {
+ final SqlValidatorNamespace targetNamespace = getNamespace(insert);
+ validateNamespace(targetNamespace, unknownType);
+ final RelOptTable relOptTable = SqlValidatorUtil.getRelOptTable(
+ targetNamespace, catalogReader.unwrap(Prepare.CatalogReader.class), null, null);
+ final SqlValidatorTable table = relOptTable == null
+ ? targetNamespace.getTable()
+ : relOptTable.unwrap(SqlValidatorTable.class);
+
+ // INSERT has an optional column name list. If present then
+ // reduce the rowtype to the columns specified. If not present
+ // then the entire target rowtype is used.
+ final RelDataType targetRowType =
+ createTargetRowType(
+ table,
+ insert.getTargetColumnList(),
+ false);
+
+ final SqlNode source = insert.getSource();
+ if (source instanceof SqlSelect) {
+ final SqlSelect sqlSelect = (SqlSelect) source;
+ validateSelect(sqlSelect, targetRowType);
+ } else {
+ final SqlValidatorScope scope = scopes.get(source);
+ validateQuery(source, scope, targetRowType);
+ }
+
+ // REVIEW jvs 4-Dec-2008: In FRG-365, this namespace row type is
+ // discarding the type inferred by inferUnknownTypes (which was invoked
+ // from validateSelect above). It would be better if that information
+ // were used here so that we never saw any untyped nulls during
+ // checkTypeAssignment.
+ final RelDataType sourceRowType = getNamespace(source).getRowType();
+ final RelDataType logicalTargetRowType =
+ getLogicalTargetRowType(targetRowType, insert);
+ setValidatedNodeType(insert, logicalTargetRowType);
+ final RelDataType logicalSourceRowType =
+ getLogicalSourceRowType(sourceRowType, insert);
+
+ checkFieldCount(insert.getTargetTable(), table, source,
+ logicalSourceRowType, logicalTargetRowType);
+
+ checkTypeAssignment(logicalSourceRowType, logicalTargetRowType, insert);
+
+ checkConstraint(table, source, logicalTargetRowType);
+
+ validateAccess(insert.getTargetTable(), table, SqlAccessEnum.INSERT);
+ }
+
+ /**
+ * Validates insert values against the constraint of a modifiable view.
+ *
+ * @param validatorTable Table that may wrap a ModifiableViewTable
+ * @param source The values being inserted
+ * @param targetRowType The target type for the view
+ */
+ private void checkConstraint(
+ SqlValidatorTable validatorTable,
+ SqlNode source,
+ RelDataType targetRowType) {
+ final ModifiableViewTable modifiableViewTable =
+ validatorTable.unwrap(ModifiableViewTable.class);
+ if (modifiableViewTable != null && source instanceof SqlCall) {
+ final Table table = modifiableViewTable.unwrap(Table.class);
+ final RelDataType tableRowType = table.getRowType(typeFactory);
+ final List<RelDataTypeField> tableFields = tableRowType.getFieldList();
+
+ // Get the mapping from column indexes of the underlying table
+ // to the target columns and view constraints.
+ final Map<Integer, RelDataTypeField> tableIndexToTargetField =
+ SqlValidatorUtil.getIndexToFieldMap(tableFields, targetRowType);
+ final Map<Integer, RexNode> projectMap =
+ RelOptUtil.getColumnConstraints(modifiableViewTable, targetRowType, typeFactory);
+
+ // Determine columns (indexed to the underlying table) that need
+ // to be validated against the view constraint.
+ final ImmutableBitSet targetColumns =
+ ImmutableBitSet.of(tableIndexToTargetField.keySet());
+ final ImmutableBitSet constrainedColumns =
+ ImmutableBitSet.of(projectMap.keySet());
+ final ImmutableBitSet constrainedTargetColumns =
+ targetColumns.intersect(constrainedColumns);
+
+ // Validate insert values against the view constraint.
+ final List<SqlNode> values = ((SqlCall) source).getOperandList();
+ for (final int colIndex : constrainedTargetColumns.asList()) {
+ final String colName = tableFields.get(colIndex).getName();
+ final RelDataTypeField targetField = tableIndexToTargetField.get(colIndex);
+ for (SqlNode row : values) {
+ final SqlCall call = (SqlCall) row;
+ final SqlNode sourceValue = call.operand(targetField.getIndex());
+ final ValidationError validationError =
+ new ValidationError(sourceValue,
+ RESOURCE.viewConstraintNotSatisfied(colName,
+ Util.last(validatorTable.getQualifiedName())));
+ RelOptUtil.validateValueAgainstConstraint(sourceValue,
+ projectMap.get(colIndex), validationError);
+ }
+ }
+ }
+ }
+
+ /**
+ * Validates updates against the constraint of a modifiable view.
+ *
+ * @param validatorTable A {@link SqlValidatorTable} that may wrap a
+ * ModifiableViewTable
+ * @param update The UPDATE parse tree node
+ * @param targetRowType The target type
+ */
+ private void checkConstraint(
+ SqlValidatorTable validatorTable,
+ SqlUpdate update,
+ RelDataType targetRowType) {
+ final ModifiableViewTable modifiableViewTable =
+ validatorTable.unwrap(ModifiableViewTable.class);
+ if (modifiableViewTable != null) {
+ final Table table = modifiableViewTable.unwrap(Table.class);
+ final RelDataType tableRowType = table.getRowType(typeFactory);
+
+ final Map<Integer, RexNode> projectMap =
+ RelOptUtil.getColumnConstraints(modifiableViewTable, targetRowType,
+ typeFactory);
+ final Map<String, Integer> nameToIndex =
+ SqlValidatorUtil.mapNameToIndex(tableRowType.getFieldList());
+
+ // Validate update values against the view constraint.
+ final List<SqlNode> targets = update.getTargetColumnList().getList();
+ final List<SqlNode> sources = update.getSourceExpressionList().getList();
+ for (final Pair<SqlNode, SqlNode> column : Pair.zip(targets, sources)) {
+ final String columnName = ((SqlIdentifier) column.left).getSimple();
+ final Integer columnIndex = nameToIndex.get(columnName);
+ if (projectMap.containsKey(columnIndex)) {
+ final RexNode columnConstraint = projectMap.get(columnIndex);
+ final ValidationError validationError =
+ new ValidationError(column.right,
+ RESOURCE.viewConstraintNotSatisfied(columnName,
+ Util.last(validatorTable.getQualifiedName())));
+ RelOptUtil.validateValueAgainstConstraint(column.right,
+ columnConstraint, validationError);
+ }
+ }
+ }
+ }
+
+ private void checkFieldCount(SqlNode node, SqlValidatorTable table,
+ SqlNode source, RelDataType logicalSourceRowType,
+ RelDataType logicalTargetRowType) {
+ final int sourceFieldCount = logicalSourceRowType.getFieldCount();
+ final int targetFieldCount = logicalTargetRowType.getFieldCount();
+ if (sourceFieldCount != targetFieldCount) {
+ throw newValidationError(node,
+ RESOURCE.unmatchInsertColumn(targetFieldCount, sourceFieldCount));
+ }
+ // Ensure that non-nullable fields are targeted.
+ final InitializerContext rexBuilder =
+ new InitializerContext() {
+ public RexBuilder getRexBuilder() {
+ return new RexBuilder(typeFactory);
+ }
+
+ public RexNode convertExpression(SqlNode e) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ final List<ColumnStrategy> strategies =
+ table.unwrap(RelOptTable.class).getColumnStrategies();
+ for (final RelDataTypeField field : table.getRowType().getFieldList()) {
+ final RelDataTypeField targetField =
+ logicalTargetRowType.getField(field.getName(), true, false);
+ switch (strategies.get(field.getIndex())) {
+ case NOT_NULLABLE:
+ assert !field.getType().isNullable();
+ if (targetField == null) {
+ throw newValidationError(node,
+ RESOURCE.columnNotNullable(field.getName()));
+ }
+ break;
+ case NULLABLE:
+ assert field.getType().isNullable();
+ break;
+ case VIRTUAL:
+ case STORED:
+ if (targetField != null
+ && !isValuesWithDefault(source, targetField.getIndex())) {
+ throw newValidationError(node,
+ RESOURCE.insertIntoAlwaysGenerated(field.getName()));
+ }
+ }
+ }
+ }
+
+ /** Returns whether a query uses {@code DEFAULT} to populate a given
+ * column. */
+ private boolean isValuesWithDefault(SqlNode source, int column) {
+ switch (source.getKind()) {
+ case VALUES:
+ for (SqlNode operand : ((SqlCall) source).getOperandList()) {
+ if (!isRowWithDefault(operand, column)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private boolean isRowWithDefault(SqlNode operand, int column) {
+ switch (operand.getKind()) {
+ case ROW:
+ final SqlCall row = (SqlCall) operand;
+ return row.getOperandList().size() >= column
+ && row.getOperandList().get(column).getKind() == SqlKind.DEFAULT;
+ }
+ return false;
+ }
+
+ protected RelDataType getLogicalTargetRowType(
+ RelDataType targetRowType,
+ SqlInsert insert) {
+ if (insert.getTargetColumnList() == null
+ && conformance.isInsertSubsetColumnsAllowed()) {
+ // Target an implicit subset of columns.
+ final SqlNode source = insert.getSource();
+ final RelDataType sourceRowType = getNamespace(source).getRowType();
+ final RelDataType logicalSourceRowType =
+ getLogicalSourceRowType(sourceRowType, insert);
+ final RelDataType implicitTargetRowType =
+ typeFactory.createStructType(
+ targetRowType.getFieldList()
+ .subList(0, logicalSourceRowType.getFieldCount()));
+ final SqlValidatorNamespace targetNamespace = getNamespace(insert);
+ validateNamespace(targetNamespace, implicitTargetRowType);
+ return implicitTargetRowType;
+ } else {
+ // Either the set of columns are explicitly targeted, or target the full
+ // set of columns.
+ return targetRowType;
+ }
+ }
+
+ protected RelDataType getLogicalSourceRowType(
+ RelDataType sourceRowType,
+ SqlInsert insert) {
+ return sourceRowType;
+ }
+
+ protected void checkTypeAssignment(
+ RelDataType sourceRowType,
+ RelDataType targetRowType,
+ final SqlNode query) {
+ // NOTE jvs 23-Feb-2006: subclasses may allow for extra targets
+ // representing system-maintained columns, so stop after all sources
+ // matched
+ List<RelDataTypeField> sourceFields = sourceRowType.getFieldList();
+ List<RelDataTypeField> targetFields = targetRowType.getFieldList();
+ final int sourceCount = sourceFields.size();
+ for (int i = 0; i < sourceCount; ++i) {
+ RelDataType sourceType = sourceFields.get(i).getType();
+ RelDataType targetType = targetFields.get(i).getType();
+ if (!SqlTypeUtil.canAssignFrom(targetType, sourceType)) {
+ // FRG-255: account for UPDATE rewrite; there's
+ // probably a better way to do this.
+ int iAdjusted = i;
+ if (query instanceof SqlUpdate) {
+ int nUpdateColumns =
+ ((SqlUpdate) query).getTargetColumnList().size();
+ assert sourceFields.size() >= nUpdateColumns;
+ iAdjusted -= sourceFields.size() - nUpdateColumns;
+ }
+ SqlNode node = getNthExpr(query, iAdjusted, sourceCount);
+ String targetTypeString;
+ String sourceTypeString;
+ if (SqlTypeUtil.areCharacterSetsMismatched(
+ sourceType,
+ targetType)) {
+ sourceTypeString = sourceType.getFullTypeString();
+ targetTypeString = targetType.getFullTypeString();
+ } else {
+ sourceTypeString = sourceType.toString();
+ targetTypeString = targetType.toString();
+ }
+ throw newValidationError(node,
+ RESOURCE.typeNotAssignable(
+ targetFields.get(i).getName(), targetTypeString,
+ sourceFields.get(i).getName(), sourceTypeString));
+ }
+ }
+ }
+
+ /**
+ * Locates the n'th expression in an INSERT or UPDATE query.
+ *
+ * @param query Query
+ * @param ordinal Ordinal of expression
+ * @param sourceCount Number of expressions
+ * @return Ordinal'th expression, never null
+ */
+ private SqlNode getNthExpr(SqlNode query, int ordinal, int sourceCount) {
+ if (query instanceof SqlInsert) {
+ SqlInsert insert = (SqlInsert) query;
+ if (insert.getTargetColumnList() != null) {
+ return insert.getTargetColumnList().get(ordinal);
+ } else {
+ return getNthExpr(
+ insert.getSource(),
+ ordinal,
+ sourceCount);
+ }
+ } else if (query instanceof SqlUpdate) {
+ SqlUpdate update = (SqlUpdate) query;
+ if (update.getTargetColumnList() != null) {
+ return update.getTargetColumnList().get(ordinal);
+ } else if (update.getSourceExpressionList() != null) {
+ return update.getSourceExpressionList().get(ordinal);
+ } else {
+ return getNthExpr(
+ update.getSourceSelect(),
+ ordinal,
+ sourceCount);
+ }
+ } else if (query instanceof SqlSelect) {
+ SqlSelect select = (SqlSelect) query;
+ if (select.getSelectList().size() == sourceCount) {
+ return select.getSelectList().get(ordinal);
+ } else {
+ return query; // give up
+ }
+ } else {
+ return query; // give up
+ }
+ }
+
+ public void validateDelete(SqlDelete call) {
+ final SqlSelect sqlSelect = call.getSourceSelect();
+ validateSelect(sqlSelect, unknownType);
+
+ final SqlValidatorNamespace targetNamespace = getNamespace(call);
+ validateNamespace(targetNamespace, unknownType);
+ final SqlValidatorTable table = targetNamespace.getTable();
+
+ validateAccess(call.getTargetTable(), table, SqlAccessEnum.DELETE);
+ }
+
+ public void validateUpdate(SqlUpdate call) {
+ final SqlValidatorNamespace targetNamespace = getNamespace(call);
+ validateNamespace(targetNamespace, unknownType);
+ final RelOptTable relOptTable = SqlValidatorUtil.getRelOptTable(
+ targetNamespace, catalogReader.unwrap(Prepare.CatalogReader.class), null, null);
+ final SqlValidatorTable table = relOptTable == null
+ ? targetNamespace.getTable()
+ : relOptTable.unwrap(SqlValidatorTable.class);
+
+ final RelDataType targetRowType =
+ createTargetRowType(
+ table,
+ call.getTargetColumnList(),
+ true);
+
+ final SqlSelect select = call.getSourceSelect();
+ validateSelect(select, targetRowType);
+
+ final RelDataType sourceRowType = getNamespace(call).getRowType();
+ checkTypeAssignment(sourceRowType, targetRowType, call);
+
+ checkConstraint(table, call, targetRowType);
+
+ validateAccess(call.getTargetTable(), table, SqlAccessEnum.UPDATE);
+ }
+
+ public void validateMerge(SqlMerge call) {
+ SqlSelect sqlSelect = call.getSourceSelect();
+ // REVIEW zfong 5/25/06 - Does an actual type have to be passed into
+ // validateSelect()?
+
+ // REVIEW jvs 6-June-2006: In general, passing unknownType like
+ // this means we won't be able to correctly infer the types
+ // for dynamic parameter markers (SET x = ?). But
+ // maybe validateUpdate and validateInsert below will do
+ // the job?
+
+ // REVIEW ksecretan 15-July-2011: They didn't get a chance to
+ // since validateSelect() would bail.
+ // Let's use the update/insert targetRowType when available.
+ IdentifierNamespace targetNamespace =
+ (IdentifierNamespace) getNamespace(call.getTargetTable());
+ validateNamespace(targetNamespace, unknownType);
+
+ SqlValidatorTable table = targetNamespace.getTable();
+ validateAccess(call.getTargetTable(), table, SqlAccessEnum.UPDATE);
+
+ RelDataType targetRowType = unknownType;
+
+ if (call.getUpdateCall() != null) {
+ targetRowType = createTargetRowType(
+ table,
+ call.getUpdateCall().getTargetColumnList(),
+ true);
+ }
+ if (call.getInsertCall() != null) {
+ targetRowType = createTargetRowType(
+ table,
+ call.getInsertCall().getTargetColumnList(),
+ false);
+ }
+
+ validateSelect(sqlSelect, targetRowType);
+
+ if (call.getUpdateCall() != null) {
+ validateUpdate(call.getUpdateCall());
+ }
+ if (call.getInsertCall() != null) {
+ validateInsert(call.getInsertCall());
+ }
+ }
+
+ /**
+ * Validates access to a table.
+ *
+ * @param table Table
+ * @param requiredAccess Access requested on table
+ */
+ private void validateAccess(
+ SqlNode node,
+ SqlValidatorTable table,
+ SqlAccessEnum requiredAccess) {
+ if (table != null) {
+ SqlAccessType access = table.getAllowedAccess();
+ if (!access.allowsAccess(requiredAccess)) {
+ throw newValidationError(node,
+ RESOURCE.accessNotAllowed(requiredAccess.name(),
+ table.getQualifiedName().toString()));
+ }
+ }
+ }
+
+ /**
+ * Validates a VALUES clause.
+ *
+ * @param node Values clause
+ * @param targetRowType Row type which expression must conform to
+ * @param scope Scope within which clause occurs
+ */
+ protected void validateValues(
+ SqlCall node,
+ RelDataType targetRowType,
+ final SqlValidatorScope scope) {
+ assert node.getKind() == SqlKind.VALUES;
+
+ final List<SqlNode> operands = node.getOperandList();
+ for (SqlNode operand : operands) {
+ if (!(operand.getKind() == SqlKind.ROW)) {
+ throw Util.needToImplement(
+ "Values function where operands are scalars");
+ }
+
+ SqlCall rowConstructor = (SqlCall) operand;
+ if (conformance.isInsertSubsetColumnsAllowed() && targetRowType.isStruct()
+ && rowConstructor.operandCount() < targetRowType.getFieldCount()) {
+ targetRowType =
+ typeFactory.createStructType(
+ targetRowType.getFieldList()
+ .subList(0, rowConstructor.operandCount()));
+ } else if (targetRowType.isStruct()
+ && rowConstructor.operandCount() != targetRowType.getFieldCount()) {
+ return;
+ }
+
+ inferUnknownTypes(
+ targetRowType,
+ scope,
+ rowConstructor);
+
+ if (targetRowType.isStruct()) {
+ for (Pair<SqlNode, RelDataTypeField> pair
+ : Pair.zip(rowConstructor.getOperandList(),
+ targetRowType.getFieldList())) {
+ if (!pair.right.getType().isNullable()
+ && SqlUtil.isNullLiteral(pair.left, false)) {
+ throw newValidationError(node,
+ RESOURCE.columnNotNullable(pair.right.getName()));
+ }
+ }
+ }
+ }
+
+ for (SqlNode operand : operands) {
+ operand.validate(this, scope);
+ }
+
+ // validate that all row types have the same number of columns
+ // and that expressions in each column are compatible.
+ // A values expression is turned into something that looks like
+ // ROW(type00, type01,...), ROW(type11,...),...
+ final int rowCount = operands.size();
+ if (rowCount >= 2) {
+ SqlCall firstRow = (SqlCall) operands.get(0);
+ final int columnCount = firstRow.operandCount();
+
+ // 1. check that all rows have the same cols length
+ for (SqlNode operand : operands) {
+ SqlCall thisRow = (SqlCall) operand;
+ if (columnCount != thisRow.operandCount()) {
+ throw newValidationError(node,
+ RESOURCE.incompatibleValueType(
+ SqlStdOperatorTable.VALUES.getName()));
+ }
+ }
+
+ // 2. check if types at i:th position in each row are compatible
+ for (int col = 0; col < columnCount; col++) {
+ final int c = col;
+ final RelDataType type =
+ typeFactory.leastRestrictive(
+ new AbstractList<RelDataType>() {
+ public RelDataType get(int row) {
+ SqlCall thisRow = (SqlCall) operands.get(row);
+ return deriveType(scope, thisRow.operand(c));
+ }
+
+ public int size() {
+ return rowCount;
+ }
+ });
+
+ if (null == type) {
+ throw newValidationError(node,
+ RESOURCE.incompatibleValueType(
+ SqlStdOperatorTable.VALUES.getName()));
+ }
+ }
+ }
+ }
+
+ public void validateDataType(SqlDataTypeSpec dataType) {
+ }
+
+ public void validateDynamicParam(SqlDynamicParam dynamicParam) {
+ }
+
+ /**
+ * Throws a validator exception with access to the validator context.
+ * The exception is determined when an instance is created.
+ */
+ private class ValidationError implements Supplier<CalciteContextException> {
+ private final SqlNode sqlNode;
+ private final Resources.ExInst<SqlValidatorException> validatorException;
+
+ ValidationError(SqlNode sqlNode,
+ Resources.ExInst<SqlValidatorException> validatorException) {
+ this.sqlNode = sqlNode;
+ this.validatorException = validatorException;
+ }
+
+ public CalciteContextException get() {
+ return newValidationError(sqlNode, validatorException);
+ }
+ }
+
+ /**
+ * Throws a validator exception with access to the validator context.
+ * The exception is determined when the function is applied.
+ */
+ class ValidationErrorFunction
+ implements Function2<SqlNode, Resources.ExInst<SqlValidatorException>,
+ CalciteContextException> {
+ @Override public CalciteContextException apply(
+ SqlNode v0, Resources.ExInst<SqlValidatorException> v1) {
+ return newValidationError(v0, v1);
+ }
+ }
+
+ public ValidationErrorFunction getValidationErrorFunction() {
+ return validationErrorFunction;
+ }
+
+ public CalciteContextException newValidationError(SqlNode node,
+ Resources.ExInst<SqlValidatorException> e) {
+ assert node != null;
+ final SqlParserPos pos = node.getParserPosition();
+ return SqlUtil.newContextException(pos, e);
+ }
+
+ protected SqlWindow getWindowByName(
+ SqlIdentifier id,
+ SqlValidatorScope scope) {
+ SqlWindow window = null;
+ if (id.isSimple()) {
+ final String name = id.getSimple();
+ window = scope.lookupWindow(name);
+ }
+ if (window == null) {
+ throw newValidationError(id, RESOURCE.windowNotFound(id.toString()));
+ }
+ return window;
+ }
+
+ public SqlWindow resolveWindow(
+ SqlNode windowOrRef,
+ SqlValidatorScope scope,
+ boolean populateBounds) {
+ SqlWindow window;
+ if (windowOrRef instanceof SqlIdentifier) {
+ window = getWindowByName((SqlIdentifier) windowOrRef, scope);
+ } else {
+ window = (SqlWindow) windowOrRef;
+ }
+ while (true) {
+ final SqlIdentifier refId = window.getRefName();
+ if (refId == null) {
+ break;
+ }
+ final String refName = refId.getSimple();
+ SqlWindow refWindow = scope.lookupWindow(refName);
+ if (refWindow == null) {
+ throw newValidationError(refId, RESOURCE.windowNotFound(refName));
+ }
+ window = window.overlay(refWindow, this);
+ }
+
+ if (populateBounds) {
+ window.populateBounds();
+ }
+ return window;
+ }
+
+ public SqlNode getOriginal(SqlNode expr) {
+ SqlNode original = originalExprs.get(expr);
+ if (original == null) {
+ original = expr;
+ }
+ return original;
+ }
+
+ public void setOriginal(SqlNode expr, SqlNode original) {
+ // Don't overwrite the original original.
+ if (originalExprs.get(expr) == null) {
+ originalExprs.put(expr, original);
+ }
+ }
+
+ SqlValidatorNamespace lookupFieldNamespace(RelDataType rowType, String name) {
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ final RelDataTypeField field = nameMatcher.field(rowType, name);
+ return new FieldNamespace(this, field.getType());
+ }
+
+ public void validateWindow(
+ SqlNode windowOrId,
+ SqlValidatorScope scope,
+ SqlCall call) {
+ // Enable nested aggregates with window aggregates (OVER operator)
+ inWindow = true;
+
+ final SqlWindow targetWindow;
+ switch (windowOrId.getKind()) {
+ case IDENTIFIER:
+ // Just verify the window exists in this query. It will validate
+ // when the definition is processed
+ targetWindow = getWindowByName((SqlIdentifier) windowOrId, scope);
+ break;
+ case WINDOW:
+ targetWindow = (SqlWindow) windowOrId;
+ break;
+ default:
+ throw Util.unexpected(windowOrId.getKind());
+ }
+
+ assert targetWindow.getWindowCall() == null;
+ targetWindow.setWindowCall(call);
+ targetWindow.validate(this, scope);
+ targetWindow.setWindowCall(null);
+ call.validate(this, scope);
+
+ validateAggregateParams(call, null, scope);
+
+ // Disable nested aggregates post validation
+ inWindow = false;
+ }
+
+ @Override public void validateMatchRecognize(SqlCall call) {
+ final SqlMatchRecognize matchRecognize = (SqlMatchRecognize) call;
+ final MatchRecognizeScope scope =
+ (MatchRecognizeScope) getMatchRecognizeScope(matchRecognize);
+
+ final MatchRecognizeNamespace ns =
+ getNamespace(call).unwrap(MatchRecognizeNamespace.class);
+ assert ns.rowType == null;
+
+ // rows per match
+ final SqlLiteral rowsPerMatch = matchRecognize.getRowsPerMatch();
+ final boolean allRows = rowsPerMatch != null
+ && rowsPerMatch.getValue()
+ == SqlMatchRecognize.RowsPerMatchOption.ALL_ROWS;
+
+ final RelDataTypeFactory.Builder typeBuilder = typeFactory.builder();
+
+ // parse PARTITION BY column
+ SqlNodeList partitionBy = matchRecognize.getPartitionList();
+ if (partitionBy != null) {
+ for (SqlNode node : partitionBy) {
+ SqlIdentifier identifier = (SqlIdentifier) node;
+ identifier.validate(this, scope);
+ RelDataType type = deriveType(scope, identifier);
+ String name = identifier.names.get(1);
+ typeBuilder.add(name, type);
+ }
+ }
+
+ // parse ORDER BY column
+ SqlNodeList orderBy = matchRecognize.getOrderList();
+ if (orderBy != null) {
+ for (SqlNode node : orderBy) {
+ node.validate(this, scope);
+ SqlIdentifier identifier = null;
+ if (node instanceof SqlBasicCall) {
+ identifier = (SqlIdentifier) ((SqlBasicCall) node).getOperands()[0];
+ } else {
+ identifier = (SqlIdentifier) node;
+ }
+
+ if (allRows) {
+ RelDataType type = deriveType(scope, identifier);
+ String name = identifier.names.get(1);
+ if (!typeBuilder.nameExists(name)) {
+ typeBuilder.add(name, type);
+ }
+ }
+ }
+ }
+
+ if (allRows) {
+ final SqlValidatorNamespace sqlNs =
+ getNamespace(matchRecognize.getTableRef());
+ final RelDataType inputDataType = sqlNs.getRowType();
+ for (RelDataTypeField fs : inputDataType.getFieldList()) {
+ if (!typeBuilder.nameExists(fs.getName())) {
+ typeBuilder.add(fs);
+ }
+ }
+ }
+
+ // retrieve pattern variables used in pattern and subset
+ SqlNode pattern = matchRecognize.getPattern();
+ PatternVarVisitor visitor = new PatternVarVisitor(scope);
+ pattern.accept(visitor);
+
+ SqlLiteral interval = matchRecognize.getInterval();
+ if (interval != null) {
+ interval.validate(this, scope);
+ if (((SqlIntervalLiteral) interval).signum() < 0) {
+ throw newValidationError(interval,
+ RESOURCE.intervalMustBeNonNegative(interval.toValue()));
+ }
+ if (orderBy == null || orderBy.size() == 0) {
+ throw newValidationError(interval,
+ RESOURCE.cannotUseWithinWithoutOrderBy());
+ }
+
+ SqlNode firstOrderByColumn = orderBy.getList().get(0);
+ SqlIdentifier identifier;
+ if (firstOrderByColumn instanceof SqlBasicCall) {
+ identifier = (SqlIdentifier) ((SqlBasicCall) firstOrderByColumn).getOperands()[0];
+ } else {
+ identifier = (SqlIdentifier) firstOrderByColumn;
+ }
+ RelDataType firstOrderByColumnType = deriveType(scope, identifier);
+ if (firstOrderByColumnType.getSqlTypeName() != SqlTypeName.TIMESTAMP) {
+ throw newValidationError(interval,
+ RESOURCE.firstColumnOfOrderByMustBeTimestamp());
+ }
+
+ SqlNode expand = expand(interval, scope);
+ RelDataType type = deriveType(scope, expand);
+ setValidatedNodeType(interval, type);
+ }
+
+ validateDefinitions(matchRecognize, scope);
+
+ SqlNodeList subsets = matchRecognize.getSubsetList();
+ if (subsets != null && subsets.size() > 0) {
+ for (SqlNode node : subsets) {
+ List<SqlNode> operands = ((SqlCall) node).getOperandList();
+ String leftString = ((SqlIdentifier) operands.get(0)).getSimple();
+ if (scope.getPatternVars().contains(leftString)) {
+ throw newValidationError(operands.get(0),
+ RESOURCE.patternVarAlreadyDefined(leftString));
+ }
+ scope.addPatternVar(leftString);
+ for (SqlNode right : (SqlNodeList) operands.get(1)) {
+ SqlIdentifier id = (SqlIdentifier) right;
+ if (!scope.getPatternVars().contains(id.getSimple())) {
+ throw newValidationError(id,
+ RESOURCE.unknownPattern(id.getSimple()));
+ }
+ scope.addPatternVar(id.getSimple());
+ }
+ }
+ }
+
+ // validate AFTER ... SKIP TO
+ final SqlNode skipTo = matchRecognize.getAfter();
+ if (skipTo instanceof SqlCall) {
+ final SqlCall skipToCall = (SqlCall) skipTo;
+ final SqlIdentifier id = skipToCall.operand(0);
+ if (!scope.getPatternVars().contains(id.getSimple())) {
+ throw newValidationError(id,
+ RESOURCE.unknownPattern(id.getSimple()));
+ }
+ }
+
+ List<Map.Entry<String, RelDataType>> measureColumns =
+ validateMeasure(matchRecognize, scope, allRows);
+ for (Map.Entry<String, RelDataType> c : measureColumns) {
+ if (!typeBuilder.nameExists(c.getKey())) {
+ typeBuilder.add(c.getKey(), c.getValue());
+ }
+ }
+
+ final RelDataType rowType = typeBuilder.build();
+ if (matchRecognize.getMeasureList().size() == 0) {
+ ns.setType(getNamespace(matchRecognize.getTableRef()).getRowType());
+ } else {
+ ns.setType(rowType);
+ }
+ }
+
+ private List<Map.Entry<String, RelDataType>> validateMeasure(SqlMatchRecognize mr,
+ MatchRecognizeScope scope, boolean allRows) {
+ final List<String> aliases = new ArrayList<>();
+ final List<SqlNode> sqlNodes = new ArrayList<>();
+ final SqlNodeList measures = mr.getMeasureList();
+ final List<Map.Entry<String, RelDataType>> fields = new ArrayList<>();
+
+ for (SqlNode measure : measures) {
+ assert measure instanceof SqlCall;
+ final String alias = deriveAlias(measure, aliases.size());
+ aliases.add(alias);
+
+ SqlNode expand = expand(measure, scope);
+ expand = navigationInMeasure(expand, allRows);
+ setOriginal(expand, measure);
+
+ inferUnknownTypes(unknownType, scope, expand);
+ final RelDataType type = deriveType(scope, expand);
+ setValidatedNodeType(measure, type);
+
+ fields.add(Pair.of(alias, type));
+ sqlNodes.add(
+ SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO, expand,
+ new SqlIdentifier(alias, SqlParserPos.ZERO)));
+ }
+
+ SqlNodeList list = new SqlNodeList(sqlNodes, measures.getParserPosition());
+ inferUnknownTypes(unknownType, scope, list);
+
+ for (SqlNode node : list) {
+ validateExpr(node, scope);
+ }
+
+ mr.setOperand(SqlMatchRecognize.OPERAND_MEASURES, list);
+
+ return fields;
+ }
+
+ private SqlNode navigationInMeasure(SqlNode node, boolean allRows) {
+ final Set<String> prefix = node.accept(new PatternValidator(true));
+ Util.discard(prefix);
+ final List<SqlNode> ops = ((SqlCall) node).getOperandList();
+
+ final SqlOperator defaultOp =
+ allRows ? SqlStdOperatorTable.RUNNING : SqlStdOperatorTable.FINAL;
+ final SqlNode op0 = ops.get(0);
+ if (!isRunningOrFinal(op0.getKind())
+ || !allRows && op0.getKind() == SqlKind.RUNNING) {
+ SqlNode newNode = defaultOp.createCall(SqlParserPos.ZERO, op0);
+ node = SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO, newNode, ops.get(1));
+ }
+
+ node = new NavigationExpander().go(node);
+ return node;
+ }
+
+ private void validateDefinitions(SqlMatchRecognize mr,
+ MatchRecognizeScope scope) {
+ final Set<String> aliases = catalogReader.nameMatcher().createSet();
+ for (SqlNode item : mr.getPatternDefList().getList()) {
+ final String alias = alias(item);
+ if (!aliases.add(alias)) {
+ throw newValidationError(item,
+ Static.RESOURCE.patternVarAlreadyDefined(alias));
+ }
+ scope.addPatternVar(alias);
+ }
+
+ final List<SqlNode> sqlNodes = new ArrayList<>();
+ for (SqlNode item : mr.getPatternDefList().getList()) {
+ final String alias = alias(item);
+ SqlNode expand = expand(item, scope);
+ expand = navigationInDefine(expand, alias);
+ setOriginal(expand, item);
+
+ inferUnknownTypes(booleanType, scope, expand);
+ expand.validate(this, scope);
+
+ // Some extra work need required here.
+ // In PREV, NEXT, FINAL and LAST, only one pattern variable is allowed.
+ sqlNodes.add(
+ SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO, expand,
+ new SqlIdentifier(alias, SqlParserPos.ZERO)));
+
+ final RelDataType type = deriveType(scope, expand);
+ if (!SqlTypeUtil.inBooleanFamily(type)) {
+ throw newValidationError(expand, RESOURCE.condMustBeBoolean("DEFINE"));
+ }
+ setValidatedNodeType(item, type);
+ }
+
+ SqlNodeList list =
+ new SqlNodeList(sqlNodes, mr.getPatternDefList().getParserPosition());
+ inferUnknownTypes(unknownType, scope, list);
+ for (SqlNode node : list) {
+ validateExpr(node, scope);
+ }
+ mr.setOperand(SqlMatchRecognize.OPERAND_PATTERN_DEFINES, list);
+ }
+
+ /** Returns the alias of a "expr AS alias" expression. */
+ private static String alias(SqlNode item) {
+ assert item instanceof SqlCall;
+ assert item.getKind() == SqlKind.AS;
+ final SqlIdentifier identifier = ((SqlCall) item).operand(1);
+ return identifier.getSimple();
+ }
+
+ /** Checks that all pattern variables within a function are the same,
+ * and canonizes expressions such as {@code PREV(B.price)} to
+ * {@code LAST(B.price, 0)}. */
+ private SqlNode navigationInDefine(SqlNode node, String alpha) {
+ Set<String> prefix = node.accept(new PatternValidator(false));
+ Util.discard(prefix);
+ node = new NavigationExpander().go(node);
+ node = new NavigationReplacer(alpha).go(node);
+ return node;
+ }
+
+ public void validateAggregateParams(SqlCall aggCall, SqlNode filter,
+ SqlValidatorScope scope) {
+ // For "agg(expr)", expr cannot itself contain aggregate function
+ // invocations. For example, "SUM(2 * MAX(x))" is illegal; when
+ // we see it, we'll report the error for the SUM (not the MAX).
+ // For more than one level of nesting, the error which results
+ // depends on the traversal order for validation.
+ //
+ // For a windowed aggregate "agg(expr)", expr can contain an aggregate
+ // function. For example,
+ // SELECT AVG(2 * MAX(x)) OVER (PARTITION BY y)
+ // FROM t
+ // GROUP BY y
+ // is legal. Only one level of nesting is allowed since non-windowed
+ // aggregates cannot nest aggregates.
+
+ // Store nesting level of each aggregate. If an aggregate is found at an invalid
+ // nesting level, throw an assert.
+ final AggFinder a;
+ if (inWindow) {
+ a = overFinder;
+ } else {
+ a = aggOrOverFinder;
+ }
+
+ for (SqlNode param : aggCall.getOperandList()) {
+ if (a.findAgg(param) != null) {
+ throw newValidationError(aggCall, RESOURCE.nestedAggIllegal());
+ }
+ }
+ if (filter != null) {
+ if (a.findAgg(filter) != null) {
+ throw newValidationError(filter, RESOURCE.aggregateInFilterIllegal());
+ }
+ }
+ }
+
+ public void validateCall(
+ SqlCall call,
+ SqlValidatorScope scope) {
+ final SqlOperator operator = call.getOperator();
+ if ((call.operandCount() == 0)
+ && (operator.getSyntax() == SqlSyntax.FUNCTION_ID)
+ && !call.isExpanded()
+ && !conformance.allowNiladicParentheses()) {
+ // For example, "LOCALTIME()" is illegal. (It should be
+ // "LOCALTIME", which would have been handled as a
+ // SqlIdentifier.)
+ throw handleUnresolvedFunction(call, (SqlFunction) operator,
+ ImmutableList.of(), null);
+ }
+
+ SqlValidatorScope operandScope = scope.getOperandScope(call);
+
+ if (operator instanceof SqlFunction
+ && ((SqlFunction) operator).getFunctionType()
+ == SqlFunctionCategory.MATCH_RECOGNIZE
+ && !(operandScope instanceof MatchRecognizeScope)) {
+ throw newValidationError(call,
+ Static.RESOURCE.functionMatchRecognizeOnly(call.toString()));
+ }
+ // Delegate validation to the operator.
+ operator.validateCall(call, this, scope, operandScope);
+ }
+
+ /**
+ * Validates that a particular feature is enabled. By default, all features
+ * are enabled; subclasses may override this method to be more
+ * discriminating.
+ *
+ * @param feature feature being used, represented as a resource instance
+ * @param context parser position context for error reporting, or null if
+ */
+ protected void validateFeature(
+ Feature feature,
+ SqlParserPos context) {
+ // By default, do nothing except to verify that the resource
+ // represents a real feature definition.
+ assert feature.getProperties().get("FeatureDefinition") != null;
+ }
+
+ public SqlNode expand(SqlNode expr, SqlValidatorScope scope) {
+ final Expander expander = new Expander(this, scope);
+ SqlNode newExpr = expr.accept(expander);
+ if (expr != newExpr) {
+ setOriginal(newExpr, expr);
+ }
+ return newExpr;
+ }
+
+ public SqlNode expandGroupByOrHavingExpr(SqlNode expr,
+ SqlValidatorScope scope, SqlSelect select, boolean havingExpression) {
+ final Expander expander = new ExtendedExpander(this, scope, select, expr,
+ havingExpression);
+ SqlNode newExpr = expr.accept(expander);
+ if (expr != newExpr) {
+ setOriginal(newExpr, expr);
+ }
+ return newExpr;
+ }
+
+ public boolean isSystemField(RelDataTypeField field) {
+ return false;
+ }
+
+ public List<List<String>> getFieldOrigins(SqlNode sqlQuery) {
+ if (sqlQuery instanceof SqlExplain) {
+ return Collections.emptyList();
+ }
+ final RelDataType rowType = getValidatedNodeType(sqlQuery);
+ final int fieldCount = rowType.getFieldCount();
+ if (!sqlQuery.isA(SqlKind.QUERY)) {
+ return Collections.nCopies(fieldCount, null);
+ }
+ final List<List<String>> list = new ArrayList<>();
+ for (int i = 0; i < fieldCount; i++) {
+ list.add(getFieldOrigin(sqlQuery, i));
+ }
+ return ImmutableNullableList.copyOf(list);
+ }
+
+ private List<String> getFieldOrigin(SqlNode sqlQuery, int i) {
+ if (sqlQuery instanceof SqlSelect) {
+ SqlSelect sqlSelect = (SqlSelect) sqlQuery;
+ final SelectScope scope = getRawSelectScope(sqlSelect);
+ final List<SqlNode> selectList = scope.getExpandedSelectList();
+ final SqlNode selectItem = stripAs(selectList.get(i));
+ if (selectItem instanceof SqlIdentifier) {
+ final SqlQualified qualified =
+ scope.fullyQualify((SqlIdentifier) selectItem);
+ SqlValidatorNamespace namespace = qualified.namespace;
+ final SqlValidatorTable table = namespace.getTable();
+ if (table == null) {
+ return null;
+ }
+ final List<String> origin =
+ new ArrayList<>(table.getQualifiedName());
+ for (String name : qualified.suffix()) {
+ namespace = namespace.lookupChild(name);
+ if (namespace == null) {
+ return null;
+ }
+ origin.add(name);
+ }
+ return origin;
+ }
+ return null;
+ } else if (sqlQuery instanceof SqlOrderBy) {
+ return getFieldOrigin(((SqlOrderBy) sqlQuery).query, i);
+ } else {
+ return null;
+ }
+ }
+
+ public RelDataType getParameterRowType(SqlNode sqlQuery) {
+ // NOTE: We assume that bind variables occur in depth-first tree
+ // traversal in the same order that they occurred in the SQL text.
+ final List<RelDataType> types = new ArrayList<>();
+ // NOTE: but parameters on fetch/offset would be counted twice
+ // as they are counted in the SqlOrderBy call and the inner SqlSelect call
+ final Set<SqlNode> alreadyVisited = new HashSet<>();
+ sqlQuery.accept(
+ new SqlShuttle() {
+
+ @Override public SqlNode visit(SqlDynamicParam param) {
+ if (alreadyVisited.add(param)) {
+ RelDataType type = getValidatedNodeType(param);
+ types.add(type);
+ }
+ return param;
+ }
+ });
+ return typeFactory.createStructType(
+ types,
+ new AbstractList<String>() {
+ @Override public String get(int index) {
+ return "?" + index;
+ }
+
+ @Override public int size() {
+ return types.size();
+ }
+ });
+ }
+
+ public void validateColumnListParams(
+ SqlFunction function,
+ List<RelDataType> argTypes,
+ List<SqlNode> operands) {
+ throw new UnsupportedOperationException();
+ }
+
+ private static boolean isPhysicalNavigation(SqlKind kind) {
+ return kind == SqlKind.PREV || kind == SqlKind.NEXT;
+ }
+
+ private static boolean isLogicalNavigation(SqlKind kind) {
+ return kind == SqlKind.FIRST || kind == SqlKind.LAST;
+ }
+
+ private static boolean isAggregation(SqlKind kind) {
+ return kind == SqlKind.SUM || kind == SqlKind.SUM0
+ || kind == SqlKind.AVG || kind == SqlKind.COUNT
+ || kind == SqlKind.MAX || kind == SqlKind.MIN;
+ }
+
+ private static boolean isRunningOrFinal(SqlKind kind) {
+ return kind == SqlKind.RUNNING || kind == SqlKind.FINAL;
+ }
+
+ private static boolean isSingleVarRequired(SqlKind kind) {
+ return isPhysicalNavigation(kind)
+ || isLogicalNavigation(kind)
+ || isAggregation(kind);
+ }
+
+ //~ Inner Classes ----------------------------------------------------------
+
+ /**
+ * Common base class for DML statement namespaces.
+ */
+ public static class DmlNamespace extends IdentifierNamespace {
+ protected DmlNamespace(SqlValidatorImpl validator, SqlNode id,
+ SqlNode enclosingNode, SqlValidatorScope parentScope) {
+ super(validator, id, enclosingNode, parentScope);
+ }
+ }
+
+ /**
+ * Namespace for an INSERT statement.
+ */
+ private static class InsertNamespace extends DmlNamespace {
+ private final SqlInsert node;
+
+ InsertNamespace(SqlValidatorImpl validator, SqlInsert node,
+ SqlNode enclosingNode, SqlValidatorScope parentScope) {
+ super(validator, node.getTargetTable(), enclosingNode, parentScope);
+ this.node = Objects.requireNonNull(node);
+ }
+
+ public SqlInsert getNode() {
+ return node;
+ }
+ }
+
+ /**
+ * Namespace for an UPDATE statement.
+ */
+ private static class UpdateNamespace extends DmlNamespace {
+ private final SqlUpdate node;
+
+ UpdateNamespace(SqlValidatorImpl validator, SqlUpdate node,
+ SqlNode enclosingNode, SqlValidatorScope parentScope) {
+ super(validator, node.getTargetTable(), enclosingNode, parentScope);
+ this.node = Objects.requireNonNull(node);
+ }
+
+ public SqlUpdate getNode() {
+ return node;
+ }
+ }
+
+ /**
+ * Namespace for a DELETE statement.
+ */
+ private static class DeleteNamespace extends DmlNamespace {
+ private final SqlDelete node;
+
+ DeleteNamespace(SqlValidatorImpl validator, SqlDelete node,
+ SqlNode enclosingNode, SqlValidatorScope parentScope) {
+ super(validator, node.getTargetTable(), enclosingNode, parentScope);
+ this.node = Objects.requireNonNull(node);
+ }
+
+ public SqlDelete getNode() {
+ return node;
+ }
+ }
+
+ /**
+ * Namespace for a MERGE statement.
+ */
+ private static class MergeNamespace extends DmlNamespace {
+ private final SqlMerge node;
+
+ MergeNamespace(SqlValidatorImpl validator, SqlMerge node,
+ SqlNode enclosingNode, SqlValidatorScope parentScope) {
+ super(validator, node.getTargetTable(), enclosingNode, parentScope);
+ this.node = Objects.requireNonNull(node);
+ }
+
+ public SqlMerge getNode() {
+ return node;
+ }
+ }
+
+ /**
+ * retrieve pattern variables defined
+ */
+ private class PatternVarVisitor implements SqlVisitor<Void> {
+ private MatchRecognizeScope scope;
+ PatternVarVisitor(MatchRecognizeScope scope) {
+ this.scope = scope;
+ }
+
+ @Override public Void visit(SqlLiteral literal) {
+ return null;
+ }
+
+ @Override public Void visit(SqlCall call) {
+ for (int i = 0; i < call.getOperandList().size(); i++) {
+ call.getOperandList().get(i).accept(this);
+ }
+ return null;
+ }
+
+ @Override public Void visit(SqlNodeList nodeList) {
+ throw Util.needToImplement(nodeList);
+ }
+
+ @Override public Void visit(SqlIdentifier id) {
+ Preconditions.checkArgument(id.isSimple());
+ scope.addPatternVar(id.getSimple());
+ return null;
+ }
+
+ @Override public Void visit(SqlDataTypeSpec type) {
+ throw Util.needToImplement(type);
+ }
+
+ @Override public Void visit(SqlDynamicParam param) {
+ throw Util.needToImplement(param);
+ }
+
+ @Override public Void visit(SqlIntervalQualifier intervalQualifier) {
+ throw Util.needToImplement(intervalQualifier);
+ }
+ }
+
+ /**
+ * Visitor which derives the type of a given {@link SqlNode}.
+ *
+ * <p>Each method must return the derived type. This visitor is basically a
+ * single-use dispatcher; the visit is never recursive.
+ */
+ private class DeriveTypeVisitor implements SqlVisitor<RelDataType> {
+ private final SqlValidatorScope scope;
+
+ DeriveTypeVisitor(SqlValidatorScope scope) {
+ this.scope = scope;
+ }
+
+ public RelDataType visit(SqlLiteral literal) {
+ return literal.createSqlType(typeFactory);
+ }
+
+ public RelDataType visit(SqlCall call) {
+ final SqlOperator operator = call.getOperator();
+ return operator.deriveType(SqlValidatorImpl.this, scope, call);
+ }
+
+ public RelDataType visit(SqlNodeList nodeList) {
+ // Operand is of a type that we can't derive a type for. If the
+ // operand is of a peculiar type, such as a SqlNodeList, then you
+ // should override the operator's validateCall() method so that it
+ // doesn't try to validate that operand as an expression.
+ throw Util.needToImplement(nodeList);
+ }
+
+ public RelDataType visit(SqlIdentifier id) {
+ // First check for builtin functions which don't have parentheses,
+ // like "LOCALTIME".
+ SqlCall call = SqlUtil.makeCall(opTab, id);
+ if (call != null) {
+ return call.getOperator().validateOperands(
+ SqlValidatorImpl.this,
+ scope,
+ call);
+ }
+
+ RelDataType type = null;
+ if (!(scope instanceof EmptyScope)) {
+ id = scope.fullyQualify(id).identifier;
+ }
+
+ // Resolve the longest prefix of id that we can
+ int i;
+ for (i = id.names.size() - 1; i > 0; i--) {
+ // REVIEW jvs 9-June-2005: The name resolution rules used
+ // here are supposed to match SQL:2003 Part 2 Section 6.6
+ // (identifier chain), but we don't currently have enough
+ // information to get everything right. In particular,
+ // routine parameters are currently looked up via resolve;
+ // we could do a better job if they were looked up via
+ // resolveColumn.
+
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ final SqlValidatorScope.ResolvedImpl resolved =
+ new SqlValidatorScope.ResolvedImpl();
+ scope.resolve(id.names.subList(0, i), nameMatcher, false, resolved);
+ if (resolved.count() == 1) {
+ // There's a namespace with the name we seek.
+ final SqlValidatorScope.Resolve resolve = resolved.only();
+ type = resolve.rowType();
+ for (SqlValidatorScope.Step p : Util.skip(resolve.path.steps())) {
+ type = type.getFieldList().get(p.i).getType();
+ }
+ break;
+ }
+ }
+
+ // Give precedence to namespace found, unless there
+ // are no more identifier components.
+ if (type == null || id.names.size() == 1) {
+ // See if there's a column with the name we seek in
+ // precisely one of the namespaces in this scope.
+ RelDataType colType = scope.resolveColumn(id.names.get(0), id);
+ if (colType != null) {
+ type = colType;
+ }
+ ++i;
+ }
+
+ if (type == null) {
+ final SqlIdentifier last = id.getComponent(i - 1, i);
+ throw newValidationError(last,
+ RESOURCE.unknownIdentifier(last.toString()));
+ }
+
+ // Resolve rest of identifier
+ for (; i < id.names.size(); i++) {
+ String name = id.names.get(i);
+ final RelDataTypeField field;
+ if (name.equals("")) {
+ // The wildcard "*" is represented as an empty name. It never
+ // resolves to a field.
+ name = "*";
+ field = null;
+ } else {
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ field = nameMatcher.field(type, name);
+ }
+ if (field == null) {
+ throw newValidationError(id.getComponent(i),
+ RESOURCE.unknownField(name));
+ }
+ type = field.getType();
+ }
+ type =
+ SqlTypeUtil.addCharsetAndCollation(
+ type,
+ getTypeFactory());
+ return type;
+ }
+
+ public RelDataType visit(SqlDataTypeSpec dataType) {
+ // Q. How can a data type have a type?
+ // A. When it appears in an expression. (Say as the 2nd arg to the
+ // CAST operator.)
+ validateDataType(dataType);
+ return dataType.deriveType(SqlValidatorImpl.this);
+ }
+
+ public RelDataType visit(SqlDynamicParam param) {
+ return unknownType;
+ }
+
+ public RelDataType visit(SqlIntervalQualifier intervalQualifier) {
+ return typeFactory.createSqlIntervalType(intervalQualifier);
+ }
+ }
+
+ /**
+ * Converts an expression into canonical form by fully-qualifying any
+ * identifiers.
+ */
+ private static class Expander extends SqlScopedShuttle {
+ protected final SqlValidatorImpl validator;
+
+ Expander(SqlValidatorImpl validator, SqlValidatorScope scope) {
+ super(scope);
+ this.validator = validator;
+ }
+
+ @Override public SqlNode visit(SqlIdentifier id) {
+ // First check for builtin functions which don't have
+ // parentheses, like "LOCALTIME".
+ SqlCall call =
+ SqlUtil.makeCall(
+ validator.getOperatorTable(),
+ id);
+ if (call != null) {
+ return call.accept(this);
+ }
+ final SqlIdentifier fqId = getScope().fullyQualify(id).identifier;
+ SqlNode expandedExpr = fqId;
+ // Convert a column ref into ITEM(*, 'col_name').
+ // select col_name from (select * from dynTable)
+ // SqlIdentifier "col_name" would be resolved to a dynamic star field in dynTable's rowType.
+ // Expand such SqlIdentifier to ITEM operator.
+ if (DynamicRecordType.isDynamicStarColName(Util.last(fqId.names))
+ && !DynamicRecordType.isDynamicStarColName(Util.last(id.names))) {
+ SqlNode[] inputs = new SqlNode[2];
+ inputs[0] = fqId;
+ inputs[1] = SqlLiteral.createCharString(
+ Util.last(id.names),
+ id.getParserPosition());
+ SqlBasicCall item_call = new SqlBasicCall(
+ SqlStdOperatorTable.ITEM,
+ inputs,
+ id.getParserPosition());
+ expandedExpr = item_call;
+ }
+ validator.setOriginal(expandedExpr, id);
+ return expandedExpr;
+ }
+
+ @Override protected SqlNode visitScoped(SqlCall call) {
+ switch (call.getKind()) {
+ case SCALAR_QUERY:
+ case CURRENT_VALUE:
+ case NEXT_VALUE:
+ case WITH:
+ return call;
+ }
+ // Only visits arguments which are expressions. We don't want to
+ // qualify non-expressions such as 'x' in 'empno * 5 AS x'.
+ ArgHandler<SqlNode> argHandler =
+ new CallCopyingArgHandler(call, false);
+ call.getOperator().acceptCall(this, call, true, argHandler);
+ final SqlNode result = argHandler.result();
+ validator.setOriginal(result, call);
+ return result;
+ }
+ }
+
+ /**
+ * Shuttle which walks over an expression in the ORDER BY clause, replacing
+ * usages of aliases with the underlying expression.
+ */
+ class OrderExpressionExpander extends SqlScopedShuttle {
+ private final List<String> aliasList;
+ private final SqlSelect select;
+ private final SqlNode root;
+
+ OrderExpressionExpander(SqlSelect select, SqlNode root) {
+ super(getOrderScope(select));
+ this.select = select;
+ this.root = root;
+ this.aliasList = getNamespace(select).getRowType().getFieldNames();
+ }
+
+ public SqlNode go() {
+ return root.accept(this);
+ }
+
+ public SqlNode visit(SqlLiteral literal) {
+ // Ordinal markers, e.g. 'select a, b from t order by 2'.
+ // Only recognize them if they are the whole expression,
+ // and if the dialect permits.
+ if (literal == root && getConformance().isSortByOrdinal()) {
+ switch (literal.getTypeName()) {
+ case DECIMAL:
+ case DOUBLE:
+ final int intValue = literal.intValue(false);
+ if (intValue >= 0) {
+ if (intValue < 1 || intValue > aliasList.size()) {
+ throw newValidationError(
+ literal, RESOURCE.orderByOrdinalOutOfRange());
+ }
+
+ // SQL ordinals are 1-based, but Sort's are 0-based
+ int ordinal = intValue - 1;
+ return nthSelectItem(ordinal, literal.getParserPosition());
+ }
+ break;
+ }
+ }
+
+ return super.visit(literal);
+ }
+
+ /**
+ * Returns the <code>ordinal</code>th item in the select list.
+ */
+ private SqlNode nthSelectItem(int ordinal, final SqlParserPos pos) {
+ // TODO: Don't expand the list every time. Maybe keep an expanded
+ // version of each expression -- select lists and identifiers -- in
+ // the validator.
+
+ SqlNodeList expandedSelectList =
+ expandStar(
+ select.getSelectList(),
+ select,
+ false);
+ SqlNode expr = expandedSelectList.get(ordinal);
+ expr = stripAs(expr);
+ if (expr instanceof SqlIdentifier) {
+ expr = getScope().fullyQualify((SqlIdentifier) expr).identifier;
+ }
+
+ // Create a copy of the expression with the position of the order
+ // item.
+ return expr.clone(pos);
+ }
+
+ public SqlNode visit(SqlIdentifier id) {
+ // Aliases, e.g. 'select a as x, b from t order by x'.
+ if (id.isSimple()
+ && getConformance().isSortByAlias()) {
+ String alias = id.getSimple();
+ final SqlValidatorNamespace selectNs = getNamespace(select);
+ final RelDataType rowType =
+ selectNs.getRowTypeSansSystemColumns();
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ RelDataTypeField field = nameMatcher.field(rowType, alias);
+ if (field != null) {
+ return nthSelectItem(
+ field.getIndex(),
+ id.getParserPosition());
+ }
+ }
+
+ // No match. Return identifier unchanged.
+ return getScope().fullyQualify(id).identifier;
+ }
+
+ protected SqlNode visitScoped(SqlCall call) {
+ // Don't attempt to expand sub-queries. We haven't implemented
+ // these yet.
+ if (call instanceof SqlSelect) {
+ return call;
+ }
+ return super.visitScoped(call);
+ }
+ }
+
+ /**
+ * Shuttle which walks over an expression in the GROUP BY/HAVING clause, replacing
+ * usages of aliases or ordinals with the underlying expression.
+ */
+ static class ExtendedExpander extends Expander {
+ final SqlSelect select;
+ final SqlNode root;
+ final boolean havingExpr;
+
+ ExtendedExpander(SqlValidatorImpl validator, SqlValidatorScope scope,
+ SqlSelect select, SqlNode root, boolean havingExpr) {
+ super(validator, scope);
+ this.select = select;
+ this.root = root;
+ this.havingExpr = havingExpr;
+ }
+
+ @Override public SqlNode visit(SqlIdentifier id) {
+ if (id.isSimple()
+ && (havingExpr
+ ? validator.getConformance().isHavingAlias()
+ : validator.getConformance().isGroupByAlias())) {
+ String name = id.getSimple();
+ SqlNode expr = null;
+ final SqlNameMatcher nameMatcher =
+ validator.catalogReader.nameMatcher();
+ int n = 0;
+ for (SqlNode s : select.getSelectList()) {
+ final String alias = SqlValidatorUtil.getAlias(s, -1);
+ if (alias != null && nameMatcher.matches(alias, name)) {
+ expr = s;
+ n++;
+ }
+ }
+ if (n == 0) {
+ return super.visit(id);
+ } else if (n > 1) {
+ // More than one column has this alias.
+ throw validator.newValidationError(id,
+ RESOURCE.columnAmbiguous(name));
+ }
+ if (havingExpr && validator.isAggregate(root)) {
+ return super.visit(id);
+ }
+ expr = stripAs(expr);
+ if (expr instanceof SqlIdentifier) {
+ expr = getScope().fullyQualify((SqlIdentifier) expr).identifier;
+ }
+ return expr;
+ }
+ return super.visit(id);
+ }
+
+ public SqlNode visit(SqlLiteral literal) {
+ if (havingExpr || !validator.getConformance().isGroupByOrdinal()) {
+ return super.visit(literal);
+ }
+ boolean isOrdinalLiteral = literal == root;
+ switch (root.getKind()) {
+ case GROUPING_SETS:
+ case ROLLUP:
+ case CUBE:
+ if (root instanceof SqlBasicCall) {
+ List<SqlNode> operandList = ((SqlBasicCall) root).getOperandList();
+ for (SqlNode node : operandList) {
+ if (node.equals(literal)) {
+ isOrdinalLiteral = true;
+ break;
+ }
+ }
+ }
+ break;
+ }
+ if (isOrdinalLiteral) {
+ switch (literal.getTypeName()) {
+ case DECIMAL:
+ case DOUBLE:
+ final int intValue = literal.intValue(false);
+ if (intValue >= 0) {
+ if (intValue < 1 || intValue > select.getSelectList().size()) {
+ throw validator.newValidationError(literal,
+ RESOURCE.orderByOrdinalOutOfRange());
+ }
+
+ // SQL ordinals are 1-based, but Sort's are 0-based
+ int ordinal = intValue - 1;
+ return SqlUtil.stripAs(select.getSelectList().get(ordinal));
+ }
+ break;
+ }
+ }
+
+ return super.visit(literal);
+ }
+ }
+
+ /** Information about an identifier in a particular scope. */
+ protected static class IdInfo {
+ public final SqlValidatorScope scope;
+ public final SqlIdentifier id;
+
+ public IdInfo(SqlValidatorScope scope, SqlIdentifier id) {
+ this.scope = scope;
+ this.id = id;
+ }
+ }
+
+ /**
+ * Utility object used to maintain information about the parameters in a
+ * function call.
+ */
+ protected static class FunctionParamInfo {
+ /**
+ * Maps a cursor (based on its position relative to other cursor
+ * parameters within a function call) to the SELECT associated with the
+ * cursor.
+ */
+ public final Map<Integer, SqlSelect> cursorPosToSelectMap;
+
+ /**
+ * Maps a column list parameter to the parent cursor parameter it
+ * references. The parameters are id'd by their names.
+ */
+ public final Map<String, String> columnListParamToParentCursorMap;
+
+ public FunctionParamInfo() {
+ cursorPosToSelectMap = new HashMap<>();
+ columnListParamToParentCursorMap = new HashMap<>();
+ }
+ }
+
+ /**
+ * Modify the nodes in navigation function
+ * such as FIRST, LAST, PREV AND NEXT.
+ */
+ private static class NavigationModifier extends SqlShuttle {
+ public SqlNode go(SqlNode node) {
+ return node.accept(this);
+ }
+ }
+
+ /**
+ * Shuttle that expands navigation expressions in a MATCH_RECOGNIZE clause.
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>{@code PREV(A.price + A.amount)} →
+ * {@code PREV(A.price) + PREV(A.amount)}
+ *
+ * <li>{@code FIRST(A.price * 2)} → {@code FIRST(A.PRICE) * 2}
+ * </ul>
+ */
+ private static class NavigationExpander extends NavigationModifier {
+ final SqlOperator op;
+ final SqlNode offset;
+
+ NavigationExpander() {
+ this(null, null);
+ }
+
+ NavigationExpander(SqlOperator operator, SqlNode offset) {
+ this.offset = offset;
+ this.op = operator;
+ }
+
+ @Override public SqlNode visit(SqlCall call) {
+ SqlKind kind = call.getKind();
+ List<SqlNode> operands = call.getOperandList();
+ List<SqlNode> newOperands = new ArrayList<>();
+
+ // This code is a workaround for CALCITE-2707
+ if (call.getFunctionQuantifier() != null
+ && call.getFunctionQuantifier().getValue() == SqlSelectKeyword.DISTINCT) {
+ final SqlParserPos pos = call.getParserPosition();
+ throw SqlUtil.newContextException(pos, Static.RESOURCE.functionQuantifierNotAllowed(call.toString()));
+ }
+ // This code is a workaround for CALCITE-2707
+
+ if (isLogicalNavigation(kind) || isPhysicalNavigation(kind)) {
+ SqlNode inner = operands.get(0);
+ SqlNode offset = operands.get(1);
+
+ // merge two straight prev/next, update offset
+ if (isPhysicalNavigation(kind)) {
+ SqlKind innerKind = inner.getKind();
+ if (isPhysicalNavigation(innerKind)) {
+ List<SqlNode> innerOperands = ((SqlCall) inner).getOperandList();
+ SqlNode innerOffset = innerOperands.get(1);
+ SqlOperator newOperator = innerKind == kind
+ ? SqlStdOperatorTable.PLUS : SqlStdOperatorTable.MINUS;
+ offset = newOperator.createCall(SqlParserPos.ZERO,
+ offset, innerOffset);
+ inner = call.getOperator().createCall(SqlParserPos.ZERO,
+ innerOperands.get(0), offset);
+ }
+ }
+ SqlNode newInnerNode =
+ inner.accept(new NavigationExpander(call.getOperator(), offset));
+ if (op != null) {
+ newInnerNode = op.createCall(SqlParserPos.ZERO, newInnerNode,
+ this.offset);
+ }
+ return newInnerNode;
+ }
+
+ if (operands.size() > 0) {
+ for (SqlNode node : operands) {
+ if (node != null) {
+ SqlNode newNode = node.accept(new NavigationExpander());
+ if (op != null) {
+ newNode = op.createCall(SqlParserPos.ZERO, newNode, offset);
+ }
+ newOperands.add(newNode);
+ } else {
+ newOperands.add(null);
+ }
+ }
+ return call.getOperator().createCall(SqlParserPos.ZERO, newOperands);
+ } else {
+ if (op == null) {
+ return call;
+ } else {
+ return op.createCall(SqlParserPos.ZERO, call, offset);
+ }
+ }
+ }
+
+ @Override public SqlNode visit(SqlIdentifier id) {
+ if (op == null) {
+ return id;
+ } else {
+ return op.createCall(SqlParserPos.ZERO, id, offset);
+ }
+ }
+ }
+
+ /**
+ * Shuttle that replaces {@code A as A.price > PREV(B.price)} with
+ * {@code PREV(A.price, 0) > LAST(B.price, 0)}.
+ *
+ * <p>Replacing {@code A.price} with {@code PREV(A.price, 0)} makes the
+ * implementation of
+ * {@link RexVisitor#visitPatternFieldRef(RexPatternFieldRef)} more unified.
+ * Otherwise, it's difficult to implement this method. If it returns the
+ * specified field, then the navigation such as {@code PREV(A.price, 1)}
+ * becomes impossible; if not, then comparisons such as
+ * {@code A.price > PREV(A.price, 1)} become meaningless.
+ */
+ private static class NavigationReplacer extends NavigationModifier {
+ private final String alpha;
+
+ NavigationReplacer(String alpha) {
+ this.alpha = alpha;
+ }
+
+ @Override public SqlNode visit(SqlCall call) {
+ SqlKind kind = call.getKind();
+ if (isLogicalNavigation(kind)
+ || isAggregation(kind)
+ || isRunningOrFinal(kind)) {
+ return call;
+ }
+
+ switch (kind) {
+ case PREV:
+ final List<SqlNode> operands = call.getOperandList();
+ if (operands.get(0) instanceof SqlIdentifier) {
+ String name = ((SqlIdentifier) operands.get(0)).names.get(0);
+ return name.equals(alpha) ? call
+ : SqlStdOperatorTable.LAST.createCall(SqlParserPos.ZERO, operands);
+ }
+ }
+ return super.visit(call);
+ }
+
+ @Override public SqlNode visit(SqlIdentifier id) {
+ if (id.isSimple()) {
+ return id;
+ }
+ SqlOperator operator = id.names.get(0).equals(alpha)
+ ? SqlStdOperatorTable.PREV : SqlStdOperatorTable.LAST;
+
+ return operator.createCall(SqlParserPos.ZERO, id,
+ SqlLiteral.createExactNumeric("0", SqlParserPos.ZERO));
+ }
+ }
+
+ /**
+ * Within one navigation function, the pattern var should be same
+ */
+ private class PatternValidator extends SqlBasicVisitor<Set<String>> {
+ private final boolean isMeasure;
+ int firstLastCount;
+ int prevNextCount;
+ int aggregateCount;
+
+ PatternValidator(boolean isMeasure) {
+ this(isMeasure, 0, 0, 0);
+ }
+
+ PatternValidator(boolean isMeasure, int firstLastCount, int prevNextCount,
+ int aggregateCount) {
+ this.isMeasure = isMeasure;
+ this.firstLastCount = firstLastCount;
+ this.prevNextCount = prevNextCount;
+ this.aggregateCount = aggregateCount;
+ }
+
+ @Override public Set<String> visit(SqlCall call) {
+ boolean isSingle = false;
+ Set<String> vars = new HashSet<>();
+ SqlKind kind = call.getKind();
+ List<SqlNode> operands = call.getOperandList();
+
+ if (isSingleVarRequired(kind)) {
+ isSingle = true;
+ if (isPhysicalNavigation(kind)) {
+ if (isMeasure) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternPrevFunctionInMeasure(call.toString()));
+ }
+ if (firstLastCount != 0) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternPrevFunctionOrder(call.toString()));
+ }
+ prevNextCount++;
+ } else if (isLogicalNavigation(kind)) {
+ if (firstLastCount != 0) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternPrevFunctionOrder(call.toString()));
+ }
+ firstLastCount++;
+ } else if (isAggregation(kind)) {
+ // cannot apply aggregation in PREV/NEXT, FIRST/LAST
+ if (firstLastCount != 0 || prevNextCount != 0) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternAggregationInNavigation(call.toString()));
+ }
+ if (kind == SqlKind.COUNT && call.getOperandList().size() > 1) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternCountFunctionArg());
+ }
+ aggregateCount++;
+ }
+ }
+
+ if (isRunningOrFinal(kind) && !isMeasure) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternRunningFunctionInDefine(call.toString()));
+ }
+
+ for (SqlNode node : operands) {
+ if (node != null) {
+ vars.addAll(
+ node.accept(
+ new PatternValidator(isMeasure, firstLastCount, prevNextCount,
+ aggregateCount)));
+ }
+ }
+
+ if (isSingle) {
+ switch (kind) {
+ case COUNT:
+ if (vars.size() > 1) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternCountFunctionArg());
+ }
+ break;
+ default:
+ if (operands.size() == 0
+ || !(operands.get(0) instanceof SqlCall)
+ || ((SqlCall) operands.get(0)).getOperator() != SqlStdOperatorTable.CLASSIFIER) {
+ if (vars.isEmpty()) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternFunctionNullCheck(call.toString()));
+ }
+ if (vars.size() != 1) {
+ throw newValidationError(call,
+ Static.RESOURCE.patternFunctionVariableCheck(call.toString()));
+ }
+ }
+ break;
+ }
+ }
+ return vars;
+ }
+
+ @Override public Set<String> visit(SqlIdentifier identifier) {
+ boolean check = prevNextCount > 0 || firstLastCount > 0 || aggregateCount > 0;
+ Set<String> vars = new HashSet<>();
+ if (identifier.names.size() > 1 && check) {
+ vars.add(identifier.names.get(0));
+ }
+ return vars;
+ }
+
+ @Override public Set<String> visit(SqlLiteral literal) {
+ return ImmutableSet.of();
+ }
+
+ @Override public Set<String> visit(SqlIntervalQualifier qualifier) {
+ return ImmutableSet.of();
+ }
+
+ @Override public Set<String> visit(SqlDataTypeSpec type) {
+ return ImmutableSet.of();
+ }
+
+ @Override public Set<String> visit(SqlDynamicParam param) {
+ return ImmutableSet.of();
+ }
+ }
+
+ /** Permutation of fields in NATURAL JOIN or USING. */
+ private class Permute {
+ final List<ImmutableIntList> sources;
+ final RelDataType rowType;
+ final boolean trivial;
+
+ Permute(SqlNode from, int offset) {
+ switch (from.getKind()) {
+ case JOIN:
+ final SqlJoin join = (SqlJoin) from;
+ final Permute left = new Permute(join.getLeft(), offset);
+ final int fieldCount =
+ getValidatedNodeType(join.getLeft()).getFieldList().size();
+ final Permute right =
+ new Permute(join.getRight(), offset + fieldCount);
+ final List<String> names = usingNames(join);
+ final List<ImmutableIntList> sources = new ArrayList<>();
+ final Set<ImmutableIntList> sourceSet = new HashSet<>();
+ final RelDataTypeFactory.Builder b = typeFactory.builder();
+ if (names != null) {
+ for (String name : names) {
+ final RelDataTypeField f = left.field(name);
+ final ImmutableIntList source = left.sources.get(f.getIndex());
+ sourceSet.add(source);
+ final RelDataTypeField f2 = right.field(name);
+ final ImmutableIntList source2 = right.sources.get(f2.getIndex());
+ sourceSet.add(source2);
+ sources.add(source.appendAll(source2));
+ final boolean nullable =
+ (f.getType().isNullable()
+ || join.getJoinType().generatesNullsOnLeft())
+ && (f2.getType().isNullable()
+ || join.getJoinType().generatesNullsOnRight());
+ b.add(f).nullable(nullable);
+ }
+ }
+ for (RelDataTypeField f : left.rowType.getFieldList()) {
+ final ImmutableIntList source = left.sources.get(f.getIndex());
+ if (sourceSet.add(source)) {
+ sources.add(source);
+ b.add(f);
+ }
+ }
+ for (RelDataTypeField f : right.rowType.getFieldList()) {
+ final ImmutableIntList source = right.sources.get(f.getIndex());
+ if (sourceSet.add(source)) {
+ sources.add(source);
+ b.add(f);
+ }
+ }
+ rowType = b.build();
+ this.sources = ImmutableList.copyOf(sources);
+ this.trivial = left.trivial
+ && right.trivial
+ && (names == null || names.isEmpty());
+ break;
+
+ default:
+ rowType = getValidatedNodeType(from);
+ this.sources = Functions.generate(rowType.getFieldCount(),
+ i -> ImmutableIntList.of(offset + i));
+ this.trivial = true;
+ }
+ }
+
+ private RelDataTypeField field(String name) {
+ return catalogReader.nameMatcher().field(rowType, name);
+ }
+
+ /** Returns the set of field names in the join condition specified by USING
+ * or implicitly by NATURAL, de-duplicated and in order. */
+ private List<String> usingNames(SqlJoin join) {
+ switch (join.getConditionType()) {
+ case USING:
+ final ImmutableList.Builder<String> list = ImmutableList.builder();
+ final Set<String> names = catalogReader.nameMatcher().createSet();
+ for (SqlNode node : (SqlNodeList) join.getCondition()) {
+ final String name = ((SqlIdentifier) node).getSimple();
+ if (names.add(name)) {
+ list.add(name);
+ }
+ }
+ return list.build();
+ case NONE:
+ if (join.isNatural()) {
+ final RelDataType t0 = getValidatedNodeType(join.getLeft());
+ final RelDataType t1 = getValidatedNodeType(join.getRight());
+ return SqlValidatorUtil.deriveNaturalJoinColumnList(
+ catalogReader.nameMatcher(), t0, t1);
+ }
+ }
+ return null;
+ }
+
+ /** Moves fields according to the permutation. */
+ public void permute(List<SqlNode> selectItems,
+ List<Map.Entry<String, RelDataType>> fields) {
+ if (trivial) {
+ return;
+ }
+
+ final List<SqlNode> oldSelectItems = ImmutableList.copyOf(selectItems);
+ selectItems.clear();
+ final List<Map.Entry<String, RelDataType>> oldFields =
+ ImmutableList.copyOf(fields);
+ fields.clear();
+ for (ImmutableIntList source : sources) {
+ final int p0 = source.get(0);
+ Map.Entry<String, RelDataType> field = oldFields.get(p0);
+ final String name = field.getKey();
+ RelDataType type = field.getValue();
+ SqlNode selectItem = oldSelectItems.get(p0);
+ for (int p1 : Util.skip(source)) {
+ final Map.Entry<String, RelDataType> field1 = oldFields.get(p1);
+ final SqlNode selectItem1 = oldSelectItems.get(p1);
+ final RelDataType type1 = field1.getValue();
+ // output is nullable only if both inputs are
+ final boolean nullable = type.isNullable() && type1.isNullable();
+ final RelDataType type2 =
+ SqlTypeUtil.leastRestrictiveForComparison(typeFactory, type,
+ type1);
+ selectItem =
+ SqlStdOperatorTable.AS.createCall(SqlParserPos.ZERO,
+ SqlStdOperatorTable.COALESCE.createCall(SqlParserPos.ZERO,
+ maybeCast(selectItem, type, type2),
+ maybeCast(selectItem1, type1, type2)),
+ new SqlIdentifier(name, SqlParserPos.ZERO));
+ type = typeFactory.createTypeWithNullability(type2, nullable);
+ }
+ fields.add(Pair.of(name, type));
+ selectItems.add(selectItem);
+ }
+ }
+ }
+
+ //~ Enums ------------------------------------------------------------------
+
+ /**
+ * Validation status.
+ */
+ public enum Status {
+ /**
+ * Validation has not started for this scope.
+ */
+ UNVALIDATED,
+
+ /**
+ * Validation is in progress for this scope.
+ */
+ IN_PROGRESS,
+
+ /**
+ * Validation has completed (perhaps unsuccessfully).
+ */
+ VALID
+ }
+
+}
+
+// End SqlValidatorImpl.java
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index 8c6a1e0a04b..4fa501cde4d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -669,12 +669,15 @@ abstract class StreamTableEnvironment(
case (RowtimeAttribute(UnresolvedFieldReference(name)), idx) =>
extractRowtime(idx, name, None)
- case (RowtimeAttribute(Alias(UnresolvedFieldReference(origName), name, _)), idx) =>
+ case (Alias(RowtimeAttribute(UnresolvedFieldReference(origName)), name, _), idx) =>
extractRowtime(idx, name, Some(origName))
case (ProctimeAttribute(UnresolvedFieldReference(name)), idx) =>
extractProctime(idx, name)
+ case (Alias(ProctimeAttribute(UnresolvedFieldReference(_)), name, _), idx) =>
+ extractProctime(idx, name)
+
case (UnresolvedFieldReference(name), _) => fieldNames = name :: fieldNames
case (Alias(UnresolvedFieldReference(_), name, _), _) => fieldNames = name :: fieldNames
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index e28a471681d..ba789638ca3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -1089,7 +1089,7 @@ abstract class TableEnvironment(val config: TableConfig) {
} else {
referenceByName(origName, t).map((_, name))
}
- case (_: TimeAttribute, _) =>
+ case (_: TimeAttribute, _) | (Alias(_: TimeAttribute, _, _), _) =>
None
case _ => throw new TableException(
"Field reference expression or alias on field expression expected.")
@@ -1101,7 +1101,7 @@ abstract class TableEnvironment(val config: TableConfig) {
referenceByName(name, p).map((_, name))
case Alias(UnresolvedFieldReference(origName), name: String, _) =>
referenceByName(origName, p).map((_, name))
- case _: TimeAttribute =>
+ case _: TimeAttribute | Alias(_: TimeAttribute, _, _) =>
None
case _ => throw new TableException(
"Field reference expression or alias on field expression expected.")
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala
index 566e3d7cbc5..57cc815fee0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala
@@ -142,6 +142,21 @@ class AggregationCodeGenerator(
fields.mkString(", ")
}
+ val parametersCodeForDistinctMerge = aggFields.map { inFields =>
+ val fields = inFields.filter(_ > -1).zipWithIndex.map { case (f, i) =>
+ // index to constant
+ if (f >= physicalInputTypes.length) {
+ constantFields(f - physicalInputTypes.length)
+ }
+ // index to input field
+ else {
+ s"(${CodeGenUtils.boxedTypeTermForTypeInfo(physicalInputTypes(f))}) k.getField($i)"
+ }
+ }
+
+ fields.mkString(", ")
+ }
+
// get method signatures
val classes = UserDefinedFunctionUtils.typeInfoToClass(physicalInputTypes)
val constantClasses = UserDefinedFunctionUtils.typeInfoToClass(constantTypes)
@@ -643,7 +658,7 @@ class AggregationCodeGenerator(
| (${classOf[Row].getCanonicalName}) entry.getKey();
| Long v = (Long) entry.getValue();
| if (aDistinctAcc$i.add(k, v)) {
- | ${aggs(i)}.accumulate(aAcc$i, k);
+ | ${aggs(i)}.accumulate(aAcc$i, ${parametersCodeForDistinctMerge(i)});
| }
| }
| a.setField($i, aDistinctAcc$i);
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index fdb0d509554..6c05ff0f801 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -1048,6 +1048,25 @@ abstract class CodeGenerator(
// generator helping methods
// ----------------------------------------------------------------------------------------------
+ protected def makeReusableInSplits(exprs: Iterable[GeneratedExpression]): Unit = {
+ // add results of expressions to member area such that all split functions can access it
+ exprs.foreach { expr =>
+
+ // declaration
+ val resultTypeTerm = primitiveTypeTermForTypeInfo(expr.resultType)
+ if (nullCheck && !expr.nullTerm.equals(NEVER_NULL) && !expr.nullTerm.equals(ALWAYS_NULL)) {
+ reusableMemberStatements.add(s"private boolean ${expr.nullTerm};")
+ }
+ reusableMemberStatements.add(s"private $resultTypeTerm ${expr.resultTerm};")
+
+ // assignment
+ if (nullCheck && !expr.nullTerm.equals(NEVER_NULL) && !expr.nullTerm.equals(ALWAYS_NULL)) {
+ reusablePerRecordStatements.add(s"this.${expr.nullTerm} = ${expr.nullTerm};")
+ }
+ reusablePerRecordStatements.add(s"this.${expr.resultTerm} = ${expr.resultTerm};")
+ }
+ }
+
private def generateCodeSplits(splits: Seq[String]): String = {
val totalLen = splits.map(_.length + 1).sum // 1 for a line break
@@ -1057,21 +1076,7 @@ abstract class CodeGenerator(
hasCodeSplits = true
// add input unboxing to member area such that all split functions can access it
- reusableInputUnboxingExprs.foreach { case (_, expr) =>
-
- // declaration
- val resultTypeTerm = primitiveTypeTermForTypeInfo(expr.resultType)
- if (nullCheck && !expr.nullTerm.equals(NEVER_NULL) && !expr.nullTerm.equals(ALWAYS_NULL)) {
- reusableMemberStatements.add(s"private boolean ${expr.nullTerm};")
- }
- reusableMemberStatements.add(s"private $resultTypeTerm ${expr.resultTerm};")
-
- // assignment
- if (nullCheck && !expr.nullTerm.equals(NEVER_NULL) && !expr.nullTerm.equals(ALWAYS_NULL)) {
- reusablePerRecordStatements.add(s"this.${expr.nullTerm} = ${expr.nullTerm};")
- }
- reusablePerRecordStatements.add(s"this.${expr.resultTerm} = ${expr.resultTerm};")
- }
+ makeReusableInSplits(reusableInputUnboxingExprs.values)
// add split methods to the member area and return the code necessary to call those methods
val methodCalls = splits.map { split =>
@@ -1196,7 +1201,7 @@ abstract class CodeGenerator(
GeneratedExpression(resultTerm, nullTerm, inputCheckCode, fieldType)
}
- private def generateFieldAccess(
+ protected def generateFieldAccess(
inputType: TypeInformation[_],
inputTerm: String,
index: Int)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala
index 3305a510753..ffa7fc24315 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala
@@ -21,28 +21,177 @@ package org.apache.flink.table.codegen
import java.lang.{Long => JLong}
import java.util
+import org.apache.calcite.rel.RelFieldCollation
+import org.apache.calcite.rel.`type`.RelDataType
import org.apache.calcite.rex._
+import org.apache.calcite.sql.SqlAggFunction
import org.apache.calcite.sql.fun.SqlStdOperatorTable._
import org.apache.flink.api.common.functions._
import org.apache.flink.api.common.typeinfo.{SqlTimeTypeInfo, TypeInformation}
import org.apache.flink.cep.pattern.conditions.{IterativeCondition, RichIterativeCondition}
-import org.apache.flink.cep.{PatternFlatSelectFunction, PatternSelectFunction, RichPatternFlatSelectFunction, RichPatternSelectFunction}
+import org.apache.flink.cep.{RichPatternFlatSelectFunction, RichPatternSelectFunction}
import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.api.dataview.DataViewSpec
import org.apache.flink.table.api.{TableConfig, TableException}
-import org.apache.flink.table.codegen.CodeGenUtils.{boxedTypeTermForTypeInfo, newName}
+import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.codegen.CodeGenUtils.{boxedTypeTermForTypeInfo, newName, primitiveDefaultValue, primitiveTypeTermForTypeInfo}
import org.apache.flink.table.codegen.GeneratedExpression.{NEVER_NULL, NO_CODE}
import org.apache.flink.table.codegen.Indenter.toISC
+import org.apache.flink.table.functions.{AggregateFunction => TableAggregateFunction}
import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.runtime.`match`.{IterativeConditionRunner, PatternSelectFunctionRunner}
+import org.apache.flink.table.runtime.aggregate.AggregateUtil
+import org.apache.flink.table.util.MatchUtil.{ALL_PATTERN_VARIABLE, AggregationPatternVariableFinder}
import org.apache.flink.table.utils.EncodingUtils
+import org.apache.flink.types.Row
import org.apache.flink.util.Collector
import org.apache.flink.util.MathUtils.checkedDownCast
import scala.collection.JavaConverters._
import scala.collection.mutable
+object MatchCodeGenerator {
+
+ def generateIterativeCondition(
+ config: TableConfig,
+ patternDefinition: RexNode,
+ inputTypeInfo: TypeInformation[_],
+ patternName: String,
+ names: Seq[String])
+ : IterativeConditionRunner = {
+ val generator = new MatchCodeGenerator(config, inputTypeInfo, names, Some(patternName))
+ val condition = generator.generateCondition(patternDefinition)
+ val body =
+ s"""
+ |${condition.code}
+ |return ${condition.resultTerm};
+ |""".stripMargin
+
+ val genCondition = generator
+ .generateMatchFunction("MatchRecognizeCondition",
+ classOf[RichIterativeCondition[Row]],
+ body,
+ condition.resultType)
+ new IterativeConditionRunner(genCondition.name, genCondition.code)
+ }
+
+ def generateOneRowPerMatchExpression(
+ config: TableConfig,
+ returnType: RowSchema,
+ partitionKeys: util.List[RexNode],
+ orderKeys: util.List[RelFieldCollation],
+ measures: util.Map[String, RexNode],
+ inputTypeInfo: TypeInformation[_],
+ patternNames: Seq[String])
+ : PatternSelectFunctionRunner = {
+ val generator = new MatchCodeGenerator(config, inputTypeInfo, patternNames)
+
+ val resultExpression = generator.generateOneRowPerMatchExpression(
+ partitionKeys,
+ measures,
+ returnType)
+ val body =
+ s"""
+ |${resultExpression.code}
+ |return ${resultExpression.resultTerm};
+ |""".stripMargin
+
+ val genFunction = generator.generateMatchFunction(
+ "MatchRecognizePatternSelectFunction",
+ classOf[RichPatternSelectFunction[Row, Row]],
+ body,
+ resultExpression.resultType)
+ new PatternSelectFunctionRunner(genFunction.name, genFunction.code)
+ }
+}
+
/**
* A code generator for generating CEP related functions.
*
+ * Aggregates are generated as follows:
+ * 1. all aggregate [[RexCall]]s are grouped by corresponding pattern variable
+ * 2. even if the same aggregation is used multiple times in an expression
+ * (e.g. SUM(A.price) > SUM(A.price) + 1) it will be calculated once. To do so [[AggBuilder]]
+ * keeps set of already seen different aggregation calls, and reuses the code to access
+ * appropriate field of aggregation result
+ * 3. after translating every expression (either in [[generateCondition]] or in
+ * [[generateOneRowPerMatchExpression]]) there will be generated code for
+ * - [[GeneratedFunction]], which will be an inner class
+ * - said [[GeneratedFunction]] will be instantiated in the ctor and opened/closed
+ * in corresponding methods of top level generated classes
+ * - function that transforms input rows (row by row) into aggregate input rows
+ * - function that calculates aggregates for variable, that uses the previous method
+ * The generated code will look similar to this:
+ *
+ *
+ * {{{
+ *
+ * public class MatchRecognizePatternSelectFunction$175 extends RichPatternSelectFunction {
+ *
+ * // Class used to calculate aggregates for a single pattern variable
+ * public final class AggFunction_variable$115$151 extends GeneratedAggregations {
+ * ...
+ * }
+ *
+ * private final AggFunction_variable$115$151 aggregator_variable$115;
+ *
+ * public MatchRecognizePatternSelectFunction$175() {
+ * aggregator_variable$115 = new AggFunction_variable$115$151();
+ * }
+ *
+ * public void open() {
+ * aggregator_variable$115.open();
+ * ...
+ * }
+ *
+ * // Function to transform incoming row into aggregate specific row. It can e.g calculate
+ * // inner expression of said aggregate
+ * private Row transformRowForAgg_variable$115(Row inAgg) {
+ * ...
+ * }
+ *
+ * // Function to calculate all aggregates for a single pattern variable
+ * private Row calculateAgg_variable$115(List<Row> input) {
+ * Acc accumulator = aggregator_variable$115.createAccumulator();
+ * for (Row row : input) {
+ * aggregator_variable$115.accumulate(accumulator, transformRowForAgg_variable$115(row));
+ * }
+ *
+ * return aggregator_variable$115.getResult(accumulator);
+ * }
+ *
+ * @Override
+ * public Object select(Map<String, List<Row>> in1) throws Exception {
+ *
+ * // Extract list of rows assigned to a single pattern variable
+ * java.util.List patternEvents$130 = (java.util.List) in1.get("A");
+ * ...
+ *
+ * // Calculate aggregates
+ * Row aggRow_variable$110$111 = calculateAgg_variable$110(patternEvents$114);
+ *
+ * // Every aggregation (e.g SUM(A.price) and AVG(A.price)) will be extracted to a variable
+ * double result$135 = aggRow_variable$126$127.getField(0);
+ * long result$137 = aggRow_variable$126$127.getField(1);
+ *
+ * // Result of aggregation will be used in expression evaluation
+ * out.setField(0, result$135)
+ *
+ * long result$140 = result$137 * 2;
+ * out.setField(1, result$140);
+ *
+ * double result$144 = $result135 + result$137;
+ * out.setField(2, result$144);
+ * }
+ *
+ * public void close() {
+ * aggregator_variable$115.close();
+ * ...
+ * }
+ *
+ * }
+ * }}}
+ *
* @param config configuration that determines runtime behavior
* @param patternNames sorted sequence of pattern variables
* @param input type information about the first input of the Function
@@ -65,6 +214,12 @@ class MatchCodeGenerator(
private val reusablePatternLists: mutable.HashMap[String, GeneratedPatternList] = mutable
.HashMap[String, GeneratedPatternList]()
+ /**
+ * Used to deduplicate aggregations calculation. The deduplication is performed by
+ * [[RexNode.toString]]. Those expressions needs to be accessible from splits, if such exists.
+ */
+ private val reusableAggregationExpr = new mutable.HashMap[String, GeneratedExpression]()
+
/**
* Context information used by Pattern reference variable to index rows mapped to it.
* Indexes element at offset either from beginning or the end based on the value of first.
@@ -72,11 +227,27 @@ class MatchCodeGenerator(
private var offset: Int = 0
private var first : Boolean = false
+ /**
+ * Flags that tells if we generate expressions inside an aggregate. It tells how to access input
+ * row.
+ */
+ private var isWithinAggExprState: Boolean = false
+
+ /**
+ * Name of term in function used to transform input row into aggregate input row.
+ */
+ private val inputAggRowTerm = "inAgg"
+
/** Term for row for key extraction */
- private val keyRowTerm = newName("keyRow")
+ private val keyRowTerm = "keyRow"
/** Term for list of all pattern names */
- private val patternNamesTerm = newName("patternNames")
+ private val patternNamesTerm = "patternNames"
+
+ /**
+ * Used to collect all aggregates per pattern variable.
+ */
+ private val aggregatesPerVariable = new mutable.HashMap[String, AggBuilder]
/**
* Sets the new reference variable indexing context. This should be used when resolving logical
@@ -194,7 +365,6 @@ class MatchCodeGenerator(
private def generateKeyRow() : GeneratedExpression = {
val exp = reusableInputUnboxingExprs
.get((keyRowTerm, 0)) match {
- // input access and unboxing has already been generated
case Some(expr) =>
expr
@@ -252,10 +422,26 @@ class MatchCodeGenerator(
generateExpression(measures.get(fieldName))
}
- generateResultExpression(
+ val exp = generateResultExpression(
resultExprs,
returnType.typeInfo,
returnType.fieldNames)
+ aggregatesPerVariable.values.foreach(_.generateAggFunction())
+ if (hasCodeSplits) {
+ makeReusableInSplits(reusableAggregationExpr.values)
+ }
+
+ exp
+ }
+
+ def generateCondition(call: RexNode): GeneratedExpression = {
+ val exp = call.accept(this)
+ aggregatesPerVariable.values.foreach(_.generateAggFunction())
+ if (hasCodeSplits) {
+ makeReusableInSplits(reusableAggregationExpr.values)
+ }
+
+ exp
}
override def visitCall(call: RexCall): GeneratedExpression = {
@@ -283,6 +469,21 @@ class MatchCodeGenerator(
case FINAL =>
call.getOperands.get(0).accept(this)
+ case _ : SqlAggFunction =>
+
+ val variable = call.accept(new AggregationPatternVariableFinder)
+ .getOrElse(throw new TableException("No pattern variable specified in aggregate"))
+
+ val matchAgg = aggregatesPerVariable.get(variable) match {
+ case Some(agg) => agg
+ case None =>
+ val agg = new AggBuilder(variable)
+ aggregatesPerVariable(variable) = agg
+ agg
+ }
+
+ matchAgg.generateDeduplicatedAggAccess(call)
+
case _ => super.visitCall(call)
}
}
@@ -299,10 +500,15 @@ class MatchCodeGenerator(
}
override def visitPatternFieldRef(fieldRef: RexPatternFieldRef): GeneratedExpression = {
- if (fieldRef.getAlpha.equals("*") && currentPattern.isDefined && offset == 0 && !first) {
- generateInputAccess(input, input1Term, fieldRef.getIndex)
+ if (isWithinAggExprState) {
+ generateFieldAccess(input, inputAggRowTerm, fieldRef.getIndex)
} else {
- generatePatternFieldRef(fieldRef)
+ if (fieldRef.getAlpha.equals(ALL_PATTERN_VARIABLE) &&
+ currentPattern.isDefined && offset == 0 && !first) {
+ generateInputAccess(input, input1Term, fieldRef.getIndex)
+ } else {
+ generatePatternFieldRef(fieldRef)
+ }
}
}
@@ -314,14 +520,14 @@ class MatchCodeGenerator(
val eventTypeTerm = boxedTypeTermForTypeInfo(input)
val eventNameTerm = newName("event")
- val addCurrent = if (currentPattern == patternName || patternName == "*") {
+ val addCurrent = if (currentPattern == patternName || patternName == ALL_PATTERN_VARIABLE) {
j"""
|$listName.add($input1Term);
""".stripMargin
} else {
""
}
- val listCode = if (patternName == "*") {
+ val listCode = if (patternName == ALL_PATTERN_VARIABLE) {
addReusablePatternNames()
val patternTerm = newName("pattern")
j"""
@@ -356,7 +562,7 @@ class MatchCodeGenerator(
private def generateMeasurePatternVariableExp(patternName: String): GeneratedPatternList = {
val listName = newName("patternEvents")
- val code = if (patternName == "*") {
+ val code = if (patternName == ALL_PATTERN_VARIABLE) {
addReusablePatternNames()
val patternTerm = newName("pattern")
@@ -390,21 +596,7 @@ class MatchCodeGenerator(
val eventTypeTerm = boxedTypeTermForTypeInfo(input)
val isRowNull = newName("isRowNull")
- val findEventsByPatternName = reusablePatternLists.get(patternFieldAlpha) match {
- // input access and unboxing has already been generated
- case Some(expr) =>
- expr
-
- case None =>
- val exp = currentPattern match {
- case Some(p) => generateDefinePatternVariableExp(patternFieldAlpha, p)
- case None => generateMeasurePatternVariableExp(patternFieldAlpha)
- }
- reusablePatternLists(patternFieldAlpha) = exp
- exp
- }
-
- val listName = findEventsByPatternName.resultTerm
+ val listName = findEventsByPatternName(patternFieldAlpha).resultTerm
val resultIndex = if (first) {
j"""$offset"""
} else {
@@ -424,11 +616,27 @@ class MatchCodeGenerator(
GeneratedExpression(rowNameTerm, isRowNull, funcCode, input)
}
+ private def findEventsByPatternName(
+ patternFieldAlpha: String)
+ : GeneratedPatternList = {
+ reusablePatternLists.get(patternFieldAlpha) match {
+ case Some(expr) =>
+ expr
+
+ case None =>
+ val exp = currentPattern match {
+ case Some(p) => generateDefinePatternVariableExp(patternFieldAlpha, p)
+ case None => generateMeasurePatternVariableExp(patternFieldAlpha)
+ }
+ reusablePatternLists(patternFieldAlpha) = exp
+ exp
+ }
+ }
+
private def generatePatternFieldRef(fieldRef: RexPatternFieldRef): GeneratedExpression = {
val escapedAlpha = EncodingUtils.escapeJava(fieldRef.getAlpha)
val patternVariableRef = reusableInputUnboxingExprs
.get((s"$escapedAlpha#$first", offset)) match {
- // input access and unboxing has already been generated
case Some(expr) =>
expr
@@ -440,4 +648,223 @@ class MatchCodeGenerator(
generateFieldAccess(patternVariableRef.copy(code = NO_CODE), fieldRef.getIndex)
}
+
+ class AggBuilder(variable: String) {
+
+ private val aggregates = new mutable.ListBuffer[RexCall]()
+
+ private val variableUID = newName("variable")
+
+ private val rowTypeTerm = "org.apache.flink.types.Row"
+
+ private val calculateAggFuncName = s"calculateAgg_$variableUID"
+
+ def generateDeduplicatedAggAccess(aggCall: RexCall): GeneratedExpression = {
+ reusableAggregationExpr.get(aggCall.toString) match {
+ case Some(expr) =>
+ expr
+
+ case None =>
+ val exp: GeneratedExpression = generateAggAccess(aggCall)
+ aggregates += aggCall
+ reusableAggregationExpr(aggCall.toString) = exp
+ reusablePerRecordStatements += exp.code
+ exp.copy(code = NO_CODE)
+ }
+ }
+
+ private def generateAggAccess(aggCall: RexCall): GeneratedExpression = {
+ val singleAggResultTerm = newName("result")
+ val singleAggNullTerm = newName("nullTerm")
+ val singleAggResultType = FlinkTypeFactory.toTypeInfo(aggCall.`type`)
+ val primitiveSingleAggResultTypeTerm = primitiveTypeTermForTypeInfo(singleAggResultType)
+ val boxedSingleAggResultTypeTerm = boxedTypeTermForTypeInfo(singleAggResultType)
+
+ val allAggRowTerm = s"aggRow_$variableUID"
+
+ val rowsForVariableCode = findEventsByPatternName(variable)
+ val codeForAgg =
+ j"""
+ |$rowTypeTerm $allAggRowTerm = $calculateAggFuncName(${rowsForVariableCode.resultTerm});
+ |""".stripMargin
+
+ reusablePerRecordStatements += codeForAgg
+
+ val defaultValue = primitiveDefaultValue(singleAggResultType)
+ val codeForSingleAgg = if (nullCheck) {
+ j"""
+ |boolean $singleAggNullTerm;
+ |$primitiveSingleAggResultTypeTerm $singleAggResultTerm;
+ |if ($allAggRowTerm.getField(${aggregates.size}) != null) {
+ | $singleAggResultTerm = ($boxedSingleAggResultTypeTerm) $allAggRowTerm
+ | .getField(${aggregates.size});
+ | $singleAggNullTerm = false;
+ |} else {
+ | $singleAggNullTerm = true;
+ | $singleAggResultTerm = $defaultValue;
+ |}
+ |""".stripMargin
+ } else {
+ j"""
+ |$primitiveSingleAggResultTypeTerm $singleAggResultTerm =
+ | ($boxedSingleAggResultTypeTerm) $allAggRowTerm.getField(${aggregates.size});
+ |""".stripMargin
+ }
+
+ reusablePerRecordStatements += codeForSingleAgg
+
+ GeneratedExpression(singleAggResultTerm, singleAggNullTerm, NO_CODE, singleAggResultType)
+ }
+
+ def generateAggFunction(): Unit = {
+ val matchAgg = extractAggregatesAndExpressions
+
+ val aggGenerator = new AggregationCodeGenerator(config, false, input, None)
+
+ val aggFunc = aggGenerator.generateAggregations(
+ s"AggFunction_$variableUID",
+ matchAgg.inputExprs.map(r => FlinkTypeFactory.toTypeInfo(r.getType)),
+ matchAgg.aggregations.map(_.aggFunction).toArray,
+ matchAgg.aggregations.map(_.inputIndices).toArray,
+ matchAgg.aggregations.indices.toArray,
+ Array.fill(matchAgg.aggregations.size)(false),
+ isStateBackedDataViews = false,
+ partialResults = false,
+ Array.emptyIntArray,
+ None,
+ matchAgg.aggregations.size,
+ needRetract = false,
+ needMerge = false,
+ needReset = false,
+ None
+ )
+
+ reusableMemberStatements.add(aggFunc.code)
+
+ val transformFuncName = s"transformRowForAgg_$variableUID"
+ val inputTransform: String = generateAggInputExprEvaluation(
+ matchAgg.inputExprs,
+ transformFuncName)
+
+ generateAggCalculation(aggFunc, transformFuncName, inputTransform)
+ }
+
+ private def extractAggregatesAndExpressions: MatchAgg = {
+ val inputRows = new mutable.LinkedHashMap[String, (RexNode, Int)]
+
+ val logicalAggregates = aggregates.map(aggCall => {
+ val callsWithIndices = aggCall.operands.asScala.map(innerCall => {
+ inputRows.get(innerCall.toString) match {
+ case Some(x) =>
+ x
+
+ case None =>
+ val callWithIndex = (innerCall, inputRows.size)
+ inputRows(innerCall.toString) = callWithIndex
+ callWithIndex
+ }
+ })
+
+ val agg = aggCall.getOperator.asInstanceOf[SqlAggFunction]
+ LogicalSingleAggCall(agg,
+ callsWithIndices.map(_._1.getType),
+ callsWithIndices.map(_._2).toArray)
+ })
+
+ val aggs = logicalAggregates.zipWithIndex.map {
+ case (agg, index) =>
+ val result = AggregateUtil.extractAggregateCallMetadata(
+ agg.function,
+ isDistinct = false, // TODO properly set once supported in Calcite
+ agg.inputTypes,
+ needRetraction = false,
+ config,
+ isStateBackedDataViews = false,
+ index)
+
+ SingleAggCall(result.aggregateFunction, agg.exprIndices.toArray, result.accumulatorSpecs)
+ }
+
+ MatchAgg(aggs, inputRows.values.map(_._1).toSeq)
+ }
+
+ private def generateAggCalculation(
+ aggFunc: GeneratedAggregationsFunction,
+ transformFuncName: String,
+ inputTransformFunc: String)
+ : Unit = {
+ val aggregatorTerm = s"aggregator_$variableUID"
+ val code =
+ j"""
+ |private final ${aggFunc.name} $aggregatorTerm;
+ |
+ |$inputTransformFunc
+ |
+ |private $rowTypeTerm $calculateAggFuncName(java.util.List input)
+ | throws Exception {
+ | $rowTypeTerm accumulator = $aggregatorTerm.createAccumulators();
+ | for ($rowTypeTerm row : input) {
+ | $aggregatorTerm.accumulate(accumulator, $transformFuncName(row));
+ | }
+ | $rowTypeTerm result = $aggregatorTerm.createOutputRow();
+ | $aggregatorTerm.setAggregationResults(accumulator, result);
+ | return result;
+ |}
+ """.stripMargin
+
+ reusableInitStatements.add(s"$aggregatorTerm = new ${aggFunc.name}();")
+ reusableOpenStatements.add(s"$aggregatorTerm.open(getRuntimeContext());")
+ reusableCloseStatements.add(s"$aggregatorTerm.close();")
+ reusableMemberStatements.add(code)
+ }
+
+ private def generateAggInputExprEvaluation(
+ inputExprs: Seq[RexNode],
+ funcName: String)
+ : String = {
+ isWithinAggExprState = true
+ val resultTerm = newName("result")
+ val exprs = inputExprs.zipWithIndex.map {
+ case (inputExpr, outputIndex) => {
+ val expr = generateExpression(inputExpr)
+ s"""
+ |${expr.code}
+ |if (${expr.nullTerm}) {
+ | $resultTerm.setField($outputIndex, null);
+ |} else {
+ | $resultTerm.setField($outputIndex, ${expr.resultTerm});
+ |}
+ """.stripMargin
+ }
+ }.mkString("\n")
+ isWithinAggExprState = false
+
+ j"""
+ |private $rowTypeTerm $funcName($rowTypeTerm $inputAggRowTerm) {
+ | $rowTypeTerm $resultTerm = new $rowTypeTerm(${inputExprs.size});
+ | $exprs
+ | return $resultTerm;
+ |}
+ """.stripMargin
+ }
+
+ private case class LogicalSingleAggCall(
+ function: SqlAggFunction,
+ inputTypes: Seq[RelDataType],
+ exprIndices: Seq[Int]
+ )
+
+ private case class SingleAggCall(
+ aggFunction: TableAggregateFunction[_, _],
+ inputIndices: Array[Int],
+ dataViews: Seq[DataViewSpec[_]]
+ )
+
+ private case class MatchAgg(
+ aggregations: Seq[SingleAggCall],
+ inputExprs: Seq[RexNode]
+ )
+
+ }
+
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
index 7fd9309b5db..d5d64b48d69 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala
@@ -355,7 +355,9 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
// expression with distinct suffix modifier
suffixDistinct |
// function call must always be at the end
- suffixFunctionCall | suffixFunctionCallOneArg
+ suffixFunctionCall | suffixFunctionCallOneArg |
+ // rowtime or proctime
+ timeIndicator
// prefix operators
@@ -525,15 +527,13 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
lazy val timeIndicator: PackratParser[Expression] = proctime | rowtime
- lazy val proctime: PackratParser[Expression] =
- (aliasMapping | "(" ~> aliasMapping <~ ")" | fieldReference) ~ "." ~ PROCTIME ^^ {
- case f ~ _ ~ _ => ProctimeAttribute(f)
- }
+ lazy val proctime: PackratParser[Expression] = fieldReference ~ "." ~ PROCTIME ^^ {
+ case f ~ _ ~ _ => ProctimeAttribute(f)
+ }
- lazy val rowtime: PackratParser[Expression] =
- (aliasMapping | "(" ~> aliasMapping <~ ")" | fieldReference) ~ "." ~ ROWTIME ^^ {
- case f ~ _ ~ _ => RowtimeAttribute(f)
- }
+ lazy val rowtime: PackratParser[Expression] = fieldReference ~ "." ~ ROWTIME ^^ {
+ case f ~ _ ~ _ => RowtimeAttribute(f)
+ }
// alias
@@ -547,7 +547,7 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
case e ~ _ ~ name => Alias(e, name.name)
}
- lazy val expression: PackratParser[Expression] = timeIndicator | overConstant | alias |
+ lazy val expression: PackratParser[Expression] = overConstant | alias |
failure("Invalid expression.")
lazy val expressionList: Parser[List[Expression]] = rep1sep(expression, ",")
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CollectAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CollectAggFunction.scala
index b10be61a166..5186d66d2be 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CollectAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/CollectAggFunction.scala
@@ -112,10 +112,12 @@ class CollectAggFunction[E](valueTypeInfo: TypeInformation[_])
def retract(acc: CollectAccumulator[E], value: E): Unit = {
if (value != null) {
val count = acc.map.get(value)
- if (count == 1) {
- acc.map.remove(value)
- } else {
- acc.map.put(value, count - 1)
+ if (count != null) {
+ if (count == 1) {
+ acc.map.remove(value)
+ } else {
+ acc.map.put(value, count - 1)
+ }
}
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index 78a7273f6af..e8bfda089db 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -199,7 +199,7 @@ class DataStreamGroupWindowAggregate(
createKeyedWindowedStream(queryConfig, window, keyedStream)
.asInstanceOf[WindowedStream[CRow, Row, DataStreamWindow]]
- val (aggFunction, accumulatorRowType, aggResultRowType) =
+ val (aggFunction, accumulatorRowType) =
AggregateUtil.createDataStreamAggregateFunction(
generator,
namedAggregates,
@@ -211,7 +211,7 @@ class DataStreamGroupWindowAggregate(
tableEnv.getConfig)
windowedStream
- .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, outRowType)
+ .aggregate(aggFunction, windowFunction, accumulatorRowType, outRowType)
.name(keyedAggOpName)
}
// global / non-keyed aggregation
@@ -225,7 +225,7 @@ class DataStreamGroupWindowAggregate(
createNonKeyedWindowedStream(queryConfig, window, timestampedInput)
.asInstanceOf[AllWindowedStream[CRow, DataStreamWindow]]
- val (aggFunction, accumulatorRowType, aggResultRowType) =
+ val (aggFunction, accumulatorRowType) =
AggregateUtil.createDataStreamAggregateFunction(
generator,
namedAggregates,
@@ -237,7 +237,7 @@ class DataStreamGroupWindowAggregate(
tableEnv.getConfig)
windowedStream
- .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, outRowType)
+ .aggregate(aggFunction, windowFunction, accumulatorRowType, outRowType)
.name(nonKeyedAggOpName)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala
index 493c0923ed2..62ca531a3b5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala
@@ -45,6 +45,7 @@ import org.apache.flink.streaming.api.windowing.time.Time
import scala.collection.JavaConverters._
import org.apache.flink.table.api._
import org.apache.flink.table.calcite.FlinkTypeFactory
+import org.apache.flink.table.codegen.MatchCodeGenerator
import org.apache.flink.table.plan.logical.MatchRecognize
import org.apache.flink.table.plan.nodes.CommonMatchRecognize
import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
@@ -184,7 +185,7 @@ class DataStreamMatch(
throw new TableException("All rows per match mode is not supported yet.")
} else {
val patternSelectFunction =
- MatchUtil.generateOneRowPerMatchExpression(
+ MatchCodeGenerator.generateOneRowPerMatchExpression(
config,
schema,
partitionKeys,
@@ -269,7 +270,7 @@ private class PatternVisitor(
val patternDefinition = logicalMatch.patternDefinitions.get(patternName)
if (patternDefinition != null) {
- val condition = MatchUtil.generateIterativeCondition(
+ val condition = MatchCodeGenerator.generateIterativeCondition(
config,
patternDefinition,
inputTypeInfo,
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala
index 5b0aa65362f..bc0f56e38ba 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala
@@ -18,15 +18,21 @@
package org.apache.flink.table.plan.rules.datastream
-import org.apache.calcite.plan.{RelOptRule, RelTraitSet}
+import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet}
import org.apache.calcite.rel.RelNode
import org.apache.calcite.rel.convert.ConverterRule
+import org.apache.calcite.rex.{RexCall, RexNode}
+import org.apache.calcite.sql.SqlAggFunction
import org.apache.flink.table.api.TableException
import org.apache.flink.table.plan.logical.MatchRecognize
import org.apache.flink.table.plan.nodes.FlinkConventions
import org.apache.flink.table.plan.nodes.datastream.DataStreamMatch
import org.apache.flink.table.plan.nodes.logical.FlinkLogicalMatch
import org.apache.flink.table.plan.schema.RowSchema
+import org.apache.flink.table.plan.util.RexDefaultVisitor
+import org.apache.flink.table.util.MatchUtil
+
+import scala.collection.JavaConverters._
class DataStreamMatchRule
extends ConverterRule(
@@ -35,6 +41,14 @@ class DataStreamMatchRule
FlinkConventions.DATASTREAM,
"DataStreamMatchRule") {
+ override def matches(call: RelOptRuleCall): Boolean = {
+ val logicalMatch: FlinkLogicalMatch = call.rel(0).asInstanceOf[FlinkLogicalMatch]
+
+ validateAggregations(logicalMatch.getMeasures.values().asScala)
+ validateAggregations(logicalMatch.getPatternDefinitions.values().asScala)
+ true
+ }
+
override def convert(rel: RelNode): RelNode = {
val logicalMatch: FlinkLogicalMatch = rel.asInstanceOf[FlinkLogicalMatch]
val traitSet: RelTraitSet = rel.getTraitSet.replace(FlinkConventions.DATASTREAM)
@@ -71,6 +85,30 @@ class DataStreamMatchRule
new RowSchema(logicalMatch.getRowType),
new RowSchema(logicalMatch.getInput.getRowType))
}
+
+ private def validateAggregations(expr: Iterable[RexNode]): Unit = {
+ val validator = new AggregationsValidator
+ expr.foreach(_.accept(validator))
+ }
+
+ class AggregationsValidator extends RexDefaultVisitor[Object] {
+
+ override def visitCall(call: RexCall): AnyRef = {
+ call.getOperator match {
+ case _: SqlAggFunction =>
+ call.accept(new MatchUtil.AggregationPatternVariableFinder)
+ case _ =>
+ call.getOperands.asScala.foreach(_.accept(this))
+ }
+
+ null
+ }
+
+ override def visitNode(rexNode: RexNode): AnyRef = {
+ null
+ }
+ }
+
}
object DataStreamMatchRule {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 1e2df6e0f93..f1386dfea46 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -85,27 +85,26 @@ object AggregateUtil {
isRowsClause: Boolean)
: ProcessFunction[CRow, CRow] = {
- val (aggFields, aggregates, isDistinctAggs, accTypes, accSpecs) =
- transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
aggregateInputType,
needRetraction = false,
tableConfig,
isStateBackedDataViews = true)
- val aggregationStateType: RowTypeInfo = new RowTypeInfo(accTypes: _*)
val forwardMapping = (0 until inputType.getFieldCount).toArray
- val aggMapping = aggregates.indices.map(x => x + inputType.getFieldCount).toArray
- val outputArity = inputType.getFieldCount + aggregates.length
+ val aggMapping = aggregateMetadata.getAdjustedMapping(inputType.getFieldCount)
+
+ val outputArity = inputType.getFieldCount + aggregateMetadata.getAggregateCallsCount
val genFunction = generator.generateAggregations(
"UnboundedProcessingOverAggregateHelper",
inputFieldTypeInfo,
- aggregates,
- aggFields,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = true,
partialResults = false,
forwardMapping,
@@ -114,9 +113,11 @@ object AggregateUtil {
needRetract = false,
needMerge = false,
needReset = false,
- accConfig = Some(accSpecs)
+ accConfig = Some(aggregateMetadata.getAggregatesAccumulatorSpecs)
)
+ val aggregationStateType: RowTypeInfo = new RowTypeInfo(aggregateMetadata
+ .getAggregatesAccumulatorTypes: _*)
if (rowTimeIdx.isDefined) {
if (isRowsClause) {
// ROWS unbounded over process function
@@ -168,27 +169,23 @@ object AggregateUtil {
generateRetraction: Boolean,
consumeRetraction: Boolean): ProcessFunction[CRow, CRow] = {
- val (aggFields, aggregates, isDistinctAggs, accTypes, accSpecs) =
- transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
inputRowType,
consumeRetraction,
tableConfig,
isStateBackedDataViews = true)
- val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
-
- val outputArity = groupings.length + aggregates.length
-
- val aggregationStateType: RowTypeInfo = new RowTypeInfo(accTypes: _*)
+ val aggMapping = aggregateMetadata.getAdjustedMapping(groupings.length)
+ val outputArity = groupings.length + aggregateMetadata.getAggregateCallsCount
val genFunction = generator.generateAggregations(
"NonWindowedAggregationHelper",
inputFieldTypes,
- aggregates,
- aggFields,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = true,
partialResults = false,
groupings,
@@ -197,9 +194,11 @@ object AggregateUtil {
consumeRetraction,
needMerge = false,
needReset = false,
- accConfig = Some(accSpecs)
+ accConfig = Some(aggregateMetadata.getAggregatesAccumulatorSpecs)
)
+ val aggregationStateType: RowTypeInfo = new RowTypeInfo(aggregateMetadata
+ .getAggregatesAccumulatorTypes: _*)
new GroupAggProcessFunction(
genFunction,
aggregationStateType,
@@ -238,28 +237,27 @@ object AggregateUtil {
: ProcessFunction[CRow, CRow] = {
val needRetract = true
- val (aggFields, aggregates, isDistinctAggs, accTypes, accSpecs) =
- transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
aggregateInputType,
needRetract,
tableConfig,
isStateBackedDataViews = true)
- val aggregationStateType: RowTypeInfo = new RowTypeInfo(accTypes: _*)
val inputRowType = CRowTypeInfo(inputTypeInfo)
val forwardMapping = (0 until inputType.getFieldCount).toArray
- val aggMapping = aggregates.indices.map(x => x + inputType.getFieldCount).toArray
- val outputArity = inputType.getFieldCount + aggregates.length
+ val aggMapping = aggregateMetadata.getAdjustedMapping(inputType.getFieldCount)
+
+ val outputArity = inputType.getFieldCount + aggregateMetadata.getAggregateCallsCount
val genFunction = generator.generateAggregations(
"BoundedOverAggregateHelper",
inputFieldTypeInfo,
- aggregates,
- aggFields,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = true,
partialResults = false,
forwardMapping,
@@ -268,9 +266,11 @@ object AggregateUtil {
needRetract,
needMerge = false,
needReset = false,
- accConfig = Some(accSpecs)
+ accConfig = Some(aggregateMetadata.getAggregatesAccumulatorSpecs)
)
+ val aggregationStateType: RowTypeInfo = new RowTypeInfo(aggregateMetadata
+ .getAggregatesAccumulatorTypes: _*)
if (rowTimeIdx.isDefined) {
if (isRowsClause) {
new RowTimeBoundedRowsOver(
@@ -343,7 +343,7 @@ object AggregateUtil {
: MapFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, isDistinctAggs, accTypes, _) = transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
inputType,
needRetract,
@@ -352,8 +352,8 @@ object AggregateUtil {
val mapReturnType: RowTypeInfo =
createRowTypeForKeysAndAggregates(
groupings,
- aggregates,
- accTypes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregatesAccumulatorTypes,
inputType,
Some(Array(BasicTypeInfo.LONG_TYPE_INFO)))
@@ -385,16 +385,16 @@ object AggregateUtil {
throw new UnsupportedOperationException(s"$window is currently not supported on batch")
}
- val aggMapping = aggregates.indices.toArray.map(_ + groupings.length)
- val outputArity = aggregates.length + groupings.length + 1
+ val aggMapping = aggregateMetadata.getAdjustedMapping(groupings.length)
+ val outputArity = aggregateMetadata.getAggregateCallsCount + groupings.length + 1
val genFunction = generator.generateAggregations(
"DataSetAggregatePrepareMapHelper",
inputFieldTypeInfo,
- aggregates,
- aggFieldIndexes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = true,
groupings,
@@ -452,7 +452,7 @@ object AggregateUtil {
: RichGroupReduceFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, isDistinctAggs, accTypes, _) = transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract,
@@ -460,8 +460,8 @@ object AggregateUtil {
val returnType: RowTypeInfo = createRowTypeForKeysAndAggregates(
groupings,
- aggregates,
- accTypes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregatesAccumulatorTypes,
physicalInputRowType,
Some(Array(BasicTypeInfo.LONG_TYPE_INFO)))
@@ -470,17 +470,18 @@ object AggregateUtil {
window match {
case SlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
// sliding time-window for partial aggregations
+ val aggMappings = aggregateMetadata.getAdjustedMapping(groupings.length)
val genFunction = generator.generateAggregations(
"DataSetAggregatePrepareMapHelper",
physicalInputTypes,
- aggregates,
- aggFieldIndexes,
- aggregates.indices.map(_ + groupings.length).toArray,
- isDistinctAggs,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
+ aggMappings,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = true,
groupings.indices.toArray,
- Some(aggregates.indices.map(_ + groupings.length).toArray),
+ Some(aggMappings),
keysAndAggregatesArity + 1,
needRetract,
needMerge = true,
@@ -569,25 +570,25 @@ object AggregateUtil {
: RichGroupReduceFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, isDistinctAggs, _, _) = transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract,
tableConfig)
- val aggMapping = aggregates.indices.toArray.map(_ + groupings.length)
+ val aggMapping = aggregateMetadata.getAdjustedMapping(groupings.length)
val genPreAggFunction = generator.generateAggregations(
"GroupingWindowAggregateHelper",
physicalInputTypes,
- aggregates,
- aggFieldIndexes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = true,
groupings.indices.toArray,
- Some(aggregates.indices.map(_ + groupings.length).toArray),
+ Some(aggMapping),
outputType.getFieldCount,
needRetract,
needMerge = true,
@@ -598,14 +599,14 @@ object AggregateUtil {
val genFinalAggFunction = generator.generateAggregations(
"GroupingWindowAggregateHelper",
physicalInputTypes,
- aggregates,
- aggFieldIndexes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = false,
groupings.indices.toArray,
- Some(aggregates.indices.map(_ + groupings.length).toArray),
+ Some(aggMapping),
outputType.getFieldCount,
needRetract,
needMerge = true,
@@ -619,7 +620,7 @@ object AggregateUtil {
case TumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
// tumbling time window
val (startPos, endPos, timePos) = computeWindowPropertyPos(properties)
- if (doAllSupportPartialMerge(aggregates)) {
+ if (doAllSupportPartialMerge(aggregateMetadata.getAggregateFunctions)) {
// for incremental aggregations
new DataSetTumbleTimeWindowAggReduceCombineFunction(
genPreAggFunction,
@@ -659,7 +660,7 @@ object AggregateUtil {
case SlidingGroupWindow(_, _, size, _) if isTimeInterval(size.resultType) =>
val (startPos, endPos, timePos) = computeWindowPropertyPos(properties)
- if (doAllSupportPartialMerge(aggregates)) {
+ if (doAllSupportPartialMerge(aggregateMetadata.getAggregateFunctions)) {
// for partial aggregations
new DataSetSlideWindowAggReduceCombineFunction(
genPreAggFunction,
@@ -726,13 +727,13 @@ object AggregateUtil {
tableConfig: TableConfig): MapPartitionFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, isDistinctAggs, accTypes, _) = transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract,
tableConfig)
- val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
+ val aggMapping = aggregateMetadata.getAdjustedMapping(groupings.length)
val keysAndAggregatesArity = groupings.length + namedAggregates.length
@@ -741,23 +742,23 @@ object AggregateUtil {
val combineReturnType: RowTypeInfo =
createRowTypeForKeysAndAggregates(
groupings,
- aggregates,
- accTypes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregatesAccumulatorTypes,
physicalInputRowType,
Option(Array(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO)))
val genFunction = generator.generateAggregations(
"GroupingWindowAggregateHelper",
physicalInputTypes,
- aggregates,
- aggFieldIndexes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = true,
groupings.indices.toArray,
- Some(aggregates.indices.map(_ + groupings.length).toArray),
- groupings.length + aggregates.length + 2,
+ Some(aggMapping),
+ groupings.length + aggregateMetadata.getAggregateCallsCount + 2,
needRetract,
needMerge = true,
needReset = true,
@@ -803,14 +804,13 @@ object AggregateUtil {
: GroupCombineFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, isDistinctAggs, accTypes, _) = transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract,
tableConfig)
- val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
-
+ val aggMapping = aggregateMetadata.getAdjustedMapping(groupings.length)
val keysAndAggregatesArity = groupings.length + namedAggregates.length
window match {
@@ -819,23 +819,23 @@ object AggregateUtil {
val combineReturnType: RowTypeInfo =
createRowTypeForKeysAndAggregates(
groupings,
- aggregates,
- accTypes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregatesAccumulatorTypes,
physicalInputRowType,
Option(Array(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO)))
val genFunction = generator.generateAggregations(
"GroupingWindowAggregateHelper",
physicalInputTypes,
- aggregates,
- aggFieldIndexes,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = true,
groupings.indices.toArray,
- Some(aggregates.indices.map(_ + groupings.length).toArray),
- groupings.length + aggregates.length + 2,
+ Some(aggMapping),
+ keysAndAggregatesArity + 2,
needRetract,
needMerge = true,
needReset = true,
@@ -873,7 +873,7 @@ object AggregateUtil {
Either[DataSetAggFunction, DataSetFinalAggFunction]) = {
val needRetract = false
- val (aggInFields, aggregates, isDistinctAggs, accTypes, _) = transformToAggregateFunctions(
+ val aggregateMetadata = extractAggregateMetadata(
namedAggregates.map(_.getKey),
inputType,
needRetract,
@@ -888,26 +888,28 @@ object AggregateUtil {
val aggOutFields = aggOutMapping.map(_._1)
- if (doAllSupportPartialMerge(aggregates)) {
+ if (doAllSupportPartialMerge(aggregateMetadata.getAggregateFunctions)) {
+
+ val aggMapping = aggregateMetadata.getAdjustedMapping(groupings.length)
// compute preaggregation type
val preAggFieldTypes = gkeyOutMapping.map(_._2)
.map(inputType.getFieldList.get(_).getType)
- .map(FlinkTypeFactory.toTypeInfo) ++ accTypes
+ .map(FlinkTypeFactory.toTypeInfo) ++ aggregateMetadata.getAggregatesAccumulatorTypes
val preAggRowType = new RowTypeInfo(preAggFieldTypes: _*)
val genPreAggFunction = generator.generateAggregations(
"DataSetAggregatePrepareMapHelper",
inputFieldTypeInfo,
- aggregates,
- aggInFields,
- aggregates.indices.map(_ + groupings.length).toArray,
- isDistinctAggs,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
+ aggMapping,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = true,
groupings,
None,
- groupings.length + aggregates.length,
+ groupings.length + aggregateMetadata.getAggregateCallsCount,
needRetract,
needMerge = false,
needReset = true,
@@ -927,14 +929,14 @@ object AggregateUtil {
val genFinalAggFunction = generator.generateAggregations(
"DataSetAggregateFinalHelper",
inputFieldTypeInfo,
- aggregates,
- aggInFields,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggOutFields,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = false,
gkeyMapping,
- Some(aggregates.indices.map(_ + groupings.length).toArray),
+ Some(aggMapping),
outputType.getFieldCount,
needRetract,
needMerge = true,
@@ -952,10 +954,10 @@ object AggregateUtil {
val genFunction = generator.generateAggregations(
"DataSetAggregateHelper",
inputFieldTypeInfo,
- aggregates,
- aggInFields,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggOutFields,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = false,
groupings,
@@ -1037,26 +1039,26 @@ object AggregateUtil {
groupingKeys: Array[Int],
needMerge: Boolean,
tableConfig: TableConfig)
- : (DataStreamAggFunction[CRow, Row, Row], RowTypeInfo, RowTypeInfo) = {
+ : (DataStreamAggFunction[CRow, Row, Row], RowTypeInfo) = {
val needRetract = false
- val (aggFields, aggregates, isDistinctAggs, accTypes, _) =
- transformToAggregateFunctions(
+ val aggregateMetadata =
+ extractAggregateMetadata(
namedAggregates.map(_.getKey),
inputType,
needRetract,
tableConfig)
- val aggMapping = aggregates.indices.toArray
- val outputArity = aggregates.length
+ val aggMapping = aggregateMetadata.getAdjustedMapping(0)
+ val outputArity = aggregateMetadata.getAggregateCallsCount
val genFunction = generator.generateAggregations(
"GroupingWindowAggregateHelper",
inputFieldTypeInfo,
- aggregates,
- aggFields,
+ aggregateMetadata.getAggregateFunctions,
+ aggregateMetadata.getAggregateIndices,
aggMapping,
- isDistinctAggs,
+ aggregateMetadata.getAggregatesDistinctFlags,
isStateBackedDataViews = false,
partialResults = false,
groupingKeys,
@@ -1068,13 +1070,10 @@ object AggregateUtil {
None
)
- val aggResultTypes = namedAggregates.map(a => FlinkTypeFactory.toTypeInfo(a.left.getType))
-
- val accumulatorRowType = new RowTypeInfo(accTypes: _*)
- val aggResultRowType = new RowTypeInfo(aggResultTypes: _*)
+ val accumulatorRowType = new RowTypeInfo(aggregateMetadata.getAggregatesAccumulatorTypes: _*)
val aggFunction = new AggregateAggFunction(genFunction)
- (aggFunction, accumulatorRowType, aggResultRowType)
+ (aggFunction, accumulatorRowType)
}
/**
@@ -1086,11 +1085,11 @@ object AggregateUtil {
groupKeysCount: Int,
tableConfig: TableConfig): Boolean = {
- val aggregateList = transformToAggregateFunctions(
+ val aggregateList = extractAggregateMetadata(
aggregateCalls,
inputType,
needRetraction = false,
- tableConfig)._2
+ tableConfig).getAggregateFunctions
doAllSupportPartialMerge(aggregateList)
}
@@ -1169,347 +1168,458 @@ object AggregateUtil {
(propPos._1, propPos._2, propPos._3)
}
- private def transformToAggregateFunctions(
+ /**
+ * Meta info of a multiple [[AggregateCall]] required to generate a single
+ * [[GeneratedAggregations]] function.
+ */
+ private[flink] class AggregateMetadata(
+ private val aggregates: Seq[(AggregateCallMetadata, Array[Int])]) {
+
+ def getAggregateFunctions: Array[TableAggregateFunction[_, _]] = {
+ aggregates.map(_._1.aggregateFunction).toArray
+ }
+
+ def getAggregatesAccumulatorTypes: Array[TypeInformation[_]] = {
+ aggregates.map(_._1.accumulatorType).toArray
+ }
+
+ def getAggregatesAccumulatorSpecs: Array[Seq[DataViewSpec[_]]] = {
+ aggregates.map(_._1.accumulatorSpecs).toArray
+ }
+
+ def getAggregatesDistinctFlags: Array[Boolean] = {
+ aggregates.map(_._1.isDistinct).toArray
+ }
+
+ def getAggregateCallsCount: Int = {
+ aggregates.length
+ }
+
+ def getAggregateIndices: Array[Array[Int]] = {
+ aggregates.map(_._2).toArray
+ }
+
+ def getAdjustedMapping(offset: Int): Array[Int] = {
+ (0 until getAggregateCallsCount).map(_ + offset).toArray
+ }
+ }
+
+ /**
+ * Meta info of a single [[SqlAggFunction]] required to generate [[GeneratedAggregations]]
+ * function.
+ */
+ private[flink] case class AggregateCallMetadata(
+ aggregateFunction: TableAggregateFunction[_, _],
+ accumulatorType: TypeInformation[_],
+ accumulatorSpecs: Seq[DataViewSpec[_]],
+ isDistinct: Boolean
+ )
+
+ /**
+ * Prepares metadata [[AggregateCallMetadata]] required to generate code for
+ * [[GeneratedAggregations]] for a single [[SqlAggFunction]].
+ *
+ * @param aggregateFunction calcite's aggregate function
+ * @param isDistinct true if should be distinct aggregation
+ * @param aggregateInputTypes input types of given aggregate
+ * @param needRetraction if the [[TableAggregateFunction]] should produce retractions
+ * @param tableConfig tableConfig, required for decimal precision
+ * @param isStateBackedDataViews if data should be backed by state backend
+ * @param uniqueIdWithinAggregate index within an AggregateCallMetadata, used to create unique
+ * state names for each aggregate function
+ * @return the result contains required metadata:
+ * - flink's aggregate function
+ * - required accumulator information (type and specifications)
+ * - if the aggregate is distinct
+ */
+ private[flink] def extractAggregateCallMetadata(
+ aggregateFunction: SqlAggFunction,
+ isDistinct: Boolean,
+ aggregateInputTypes: Seq[RelDataType],
+ needRetraction: Boolean,
+ tableConfig: TableConfig,
+ isStateBackedDataViews: Boolean,
+ uniqueIdWithinAggregate: Int)
+ : AggregateCallMetadata = {
+ // store the aggregate fields of each aggregate function, by the same order of aggregates.
+ // create aggregate function instances by function type and aggregate field data type.
+
+ val aggregate: TableAggregateFunction[_, _] = createFlinkAggFunction(
+ aggregateFunction,
+ needRetraction,
+ aggregateInputTypes,
+ tableConfig)
+
+ val (accumulatorType, accSpecs) = aggregateFunction match {
+ case collect: SqlAggFunction if collect.getKind == SqlKind.COLLECT =>
+ removeStateViewFieldsFromAccTypeInfo(
+ uniqueIdWithinAggregate,
+ aggregate,
+ aggregate.getAccumulatorType,
+ isStateBackedDataViews)
+
+ case udagg: AggSqlFunction =>
+ removeStateViewFieldsFromAccTypeInfo(
+ uniqueIdWithinAggregate,
+ aggregate,
+ udagg.accType,
+ isStateBackedDataViews)
+
+ case _ =>
+ (getAccumulatorTypeOfAggregateFunction(aggregate), None)
+ }
+
+ // create distinct accumulator filter argument
+ val distinctAccumulatorType = if (isDistinct) {
+ createDistinctAccumulatorType(aggregateInputTypes, isStateBackedDataViews, accumulatorType)
+ } else {
+ accumulatorType
+ }
+
+ AggregateCallMetadata(aggregate, distinctAccumulatorType, accSpecs.getOrElse(Seq()), isDistinct)
+ }
+
+ private def createDistinctAccumulatorType(
+ aggregateInputTypes: Seq[RelDataType],
+ isStateBackedDataViews: Boolean,
+ accumulatorType: TypeInformation[_])
+ : PojoTypeInfo[DistinctAccumulator[_]] = {
+ // Using Pojo fields for the real underlying accumulator
+ val pojoFields = new util.ArrayList[PojoField]()
+ pojoFields.add(new PojoField(
+ classOf[DistinctAccumulator[_]].getDeclaredField("realAcc"),
+ accumulatorType)
+ )
+ // If StateBackend is not enabled, the distinct mapping also needs
+ // to be added to the Pojo fields.
+ if (!isStateBackedDataViews) {
+
+ val argTypes: Array[TypeInformation[_]] = aggregateInputTypes
+ .map(FlinkTypeFactory.toTypeInfo).toArray
+
+ val mapViewTypeInfo = new MapViewTypeInfo(
+ new RowTypeInfo(argTypes: _*),
+ BasicTypeInfo.LONG_TYPE_INFO)
+ pojoFields.add(new PojoField(
+ classOf[DistinctAccumulator[_]].getDeclaredField("distinctValueMap"),
+ mapViewTypeInfo)
+ )
+ }
+ new PojoTypeInfo(classOf[DistinctAccumulator[_]], pojoFields)
+ }
+
+ /**
+ * Prepares metadata [[AggregateMetadata]] required to generate code for
+ * [[GeneratedAggregations]] for all [[AggregateCall]].
+ *
+ * @param aggregateCalls calcite's aggregate function
+ * @param aggregateInputType input type of given aggregates
+ * @param needRetraction if the [[TableAggregateFunction]] should produce retractions
+ * @param tableConfig tableConfig, required for decimal precision
+ * @param isStateBackedDataViews if data should be backed by state backend
+ * @return the result contains required metadata:
+ * - flink's aggregate function
+ * - required accumulator information (type and specifications)
+ * - indices important for each aggregate
+ * - if the aggregate is distinct
+ */
+ private def extractAggregateMetadata(
aggregateCalls: Seq[AggregateCall],
aggregateInputType: RelDataType,
needRetraction: Boolean,
tableConfig: TableConfig,
isStateBackedDataViews: Boolean = false)
- : (Array[Array[Int]],
- Array[TableAggregateFunction[_, _]],
- Array[Boolean],
- Array[TypeInformation[_]],
- Array[Seq[DataViewSpec[_]]]) = {
+ : AggregateMetadata = {
+
+ val aggregatesWithIndices = aggregateCalls.zipWithIndex.map {
+ case (aggregateCall, index) =>
+ val argList: util.List[Integer] = aggregateCall.getArgList
+
+ val aggFieldIndices = if (argList.isEmpty) {
+ if (aggregateCall.getAggregation.isInstanceOf[SqlCountAggFunction]) {
+ Array[Int](-1)
+ } else {
+ throw new TableException("Aggregate fields should not be empty.")
+ }
+ } else {
+ argList.asScala.map(i => i.intValue).toArray
+ }
+
+ val inputTypes = argList.map(aggregateInputType.getFieldList.get(_).getType)
+ val aggregateCallMetadata = extractAggregateCallMetadata(aggregateCall.getAggregation,
+ aggregateCall.isDistinct,
+ inputTypes,
+ needRetraction,
+ tableConfig,
+ isStateBackedDataViews,
+ index)
+
+ (aggregateCallMetadata, aggFieldIndices)
+ }
// store the aggregate fields of each aggregate function, by the same order of aggregates.
- val aggFieldIndexes = new Array[Array[Int]](aggregateCalls.size)
- val aggregates = new Array[TableAggregateFunction[_ <: Any, _ <: Any]](aggregateCalls.size)
- val accTypes = new Array[TypeInformation[_]](aggregateCalls.size)
+ new AggregateMetadata(aggregatesWithIndices)
+ }
- // create aggregate function instances by function type and aggregate field data type.
- aggregateCalls.zipWithIndex.foreach { case (aggregateCall, index) =>
- val argList: util.List[Integer] = aggregateCall.getArgList
+ /**
+ * Converts calcite's [[SqlAggFunction]] to a Flink's UDF [[TableAggregateFunction]].
+ * create aggregate function instances by function type and aggregate field data type.
+ */
+ private def createFlinkAggFunction(
+ aggFunc: SqlAggFunction,
+ needRetraction: Boolean,
+ inputDataType: Seq[RelDataType],
+ tableConfig: TableConfig)
+ : TableAggregateFunction[_ <: Any, _ <: Any] = {
+
+ lazy val outputType = inputDataType.get(0)
+ lazy val outputTypeName = if (inputDataType.isEmpty) {
+ throw new TableException("Aggregate fields should not be empty.")
+ } else {
+ outputType.getSqlTypeName
+ }
- if (aggregateCall.getAggregation.isInstanceOf[SqlCountAggFunction]) {
- aggregates(index) = new CountAggFunction
- if (argList.isEmpty) {
- aggFieldIndexes(index) = Array[Int](-1)
+ aggFunc match {
+
+ case collect: SqlAggFunction if collect.getKind == SqlKind.COLLECT =>
+ new CollectAggFunction(FlinkTypeFactory.toTypeInfo(outputType))
+
+ case udagg: AggSqlFunction =>
+ udagg.getFunction
+
+ case _: SqlCountAggFunction =>
+ new CountAggFunction
+
+ case _: SqlSumAggFunction =>
+ if (needRetraction) {
+ outputTypeName match {
+ case TINYINT =>
+ new ByteSumWithRetractAggFunction
+ case SMALLINT =>
+ new ShortSumWithRetractAggFunction
+ case INTEGER =>
+ new IntSumWithRetractAggFunction
+ case BIGINT =>
+ new LongSumWithRetractAggFunction
+ case FLOAT =>
+ new FloatSumWithRetractAggFunction
+ case DOUBLE =>
+ new DoubleSumWithRetractAggFunction
+ case DECIMAL =>
+ new DecimalSumWithRetractAggFunction
+ case sqlType: SqlTypeName =>
+ throw new TableException(s"Sum aggregate does no support type: '$sqlType'")
+ }
} else {
- aggFieldIndexes(index) = argList.asScala.map(i => i.intValue).toArray
+ outputTypeName match {
+ case TINYINT =>
+ new ByteSumAggFunction
+ case SMALLINT =>
+ new ShortSumAggFunction
+ case INTEGER =>
+ new IntSumAggFunction
+ case BIGINT =>
+ new LongSumAggFunction
+ case FLOAT =>
+ new FloatSumAggFunction
+ case DOUBLE =>
+ new DoubleSumAggFunction
+ case DECIMAL =>
+ new DecimalSumAggFunction
+ case sqlType: SqlTypeName =>
+ throw new TableException(s"Sum aggregate does no support type: '$sqlType'")
+ }
}
- } else {
- if (argList.isEmpty) {
- throw new TableException("Aggregate fields should not be empty.")
+
+ case _: SqlSumEmptyIsZeroAggFunction =>
+ if (needRetraction) {
+ outputTypeName match {
+ case TINYINT =>
+ new ByteSum0WithRetractAggFunction
+ case SMALLINT =>
+ new ShortSum0WithRetractAggFunction
+ case INTEGER =>
+ new IntSum0WithRetractAggFunction
+ case BIGINT =>
+ new LongSum0WithRetractAggFunction
+ case FLOAT =>
+ new FloatSum0WithRetractAggFunction
+ case DOUBLE =>
+ new DoubleSum0WithRetractAggFunction
+ case DECIMAL =>
+ new DecimalSum0WithRetractAggFunction
+ case sqlType: SqlTypeName =>
+ throw new TableException(s"Sum0 aggregate does no support type: '$sqlType'")
+ }
} else {
- aggFieldIndexes(index) = argList.asScala.map(i => i.intValue).toArray
+ outputTypeName match {
+ case TINYINT =>
+ new ByteSum0AggFunction
+ case SMALLINT =>
+ new ShortSum0AggFunction
+ case INTEGER =>
+ new IntSum0AggFunction
+ case BIGINT =>
+ new LongSum0AggFunction
+ case FLOAT =>
+ new FloatSum0AggFunction
+ case DOUBLE =>
+ new DoubleSum0AggFunction
+ case DECIMAL =>
+ new DecimalSum0AggFunction
+ case sqlType: SqlTypeName =>
+ throw new TableException(s"Sum0 aggregate does no support type: '$sqlType'")
+ }
}
- val relDataType = aggregateInputType.getFieldList.get(aggFieldIndexes(index)(0)).getType
- val sqlTypeName = relDataType.getSqlTypeName
- aggregateCall.getAggregation match {
-
- case _: SqlSumAggFunction =>
- if (needRetraction) {
- aggregates(index) = sqlTypeName match {
- case TINYINT =>
- new ByteSumWithRetractAggFunction
- case SMALLINT =>
- new ShortSumWithRetractAggFunction
- case INTEGER =>
- new IntSumWithRetractAggFunction
- case BIGINT =>
- new LongSumWithRetractAggFunction
- case FLOAT =>
- new FloatSumWithRetractAggFunction
- case DOUBLE =>
- new DoubleSumWithRetractAggFunction
- case DECIMAL =>
- new DecimalSumWithRetractAggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(s"Sum aggregate does no support type: '$sqlType'")
- }
- } else {
- aggregates(index) = sqlTypeName match {
- case TINYINT =>
- new ByteSumAggFunction
- case SMALLINT =>
- new ShortSumAggFunction
- case INTEGER =>
- new IntSumAggFunction
- case BIGINT =>
- new LongSumAggFunction
- case FLOAT =>
- new FloatSumAggFunction
- case DOUBLE =>
- new DoubleSumAggFunction
- case DECIMAL =>
- new DecimalSumAggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(s"Sum aggregate does no support type: '$sqlType'")
- }
- }
-
- case _: SqlSumEmptyIsZeroAggFunction =>
- if (needRetraction) {
- aggregates(index) = sqlTypeName match {
- case TINYINT =>
- new ByteSum0WithRetractAggFunction
- case SMALLINT =>
- new ShortSum0WithRetractAggFunction
- case INTEGER =>
- new IntSum0WithRetractAggFunction
- case BIGINT =>
- new LongSum0WithRetractAggFunction
- case FLOAT =>
- new FloatSum0WithRetractAggFunction
- case DOUBLE =>
- new DoubleSum0WithRetractAggFunction
- case DECIMAL =>
- new DecimalSum0WithRetractAggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(s"Sum0 aggregate does no support type: '$sqlType'")
- }
- } else {
- aggregates(index) = sqlTypeName match {
- case TINYINT =>
- new ByteSum0AggFunction
- case SMALLINT =>
- new ShortSum0AggFunction
- case INTEGER =>
- new IntSum0AggFunction
- case BIGINT =>
- new LongSum0AggFunction
- case FLOAT =>
- new FloatSum0AggFunction
- case DOUBLE =>
- new DoubleSum0AggFunction
- case DECIMAL =>
- new DecimalSum0AggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(s"Sum0 aggregate does no support type: '$sqlType'")
- }
- }
+ case a: SqlAvgAggFunction if a.kind == SqlKind.AVG =>
+ outputTypeName match {
+ case TINYINT =>
+ new ByteAvgAggFunction
+ case SMALLINT =>
+ new ShortAvgAggFunction
+ case INTEGER =>
+ new IntAvgAggFunction
+ case BIGINT =>
+ new LongAvgAggFunction
+ case FLOAT =>
+ new FloatAvgAggFunction
+ case DOUBLE =>
+ new DoubleAvgAggFunction
+ case DECIMAL =>
+ new DecimalAvgAggFunction(tableConfig.getDecimalContext)
+ case sqlType: SqlTypeName =>
+ throw new TableException(s"Avg aggregate does no support type: '$sqlType'")
+ }
- case a: SqlAvgAggFunction if a.kind == SqlKind.AVG =>
- aggregates(index) = sqlTypeName match {
+ case sqlMinMaxFunction: SqlMinMaxAggFunction =>
+ if (sqlMinMaxFunction.getKind == SqlKind.MIN) {
+ if (needRetraction) {
+ outputTypeName match {
case TINYINT =>
- new ByteAvgAggFunction
+ new ByteMinWithRetractAggFunction
case SMALLINT =>
- new ShortAvgAggFunction
+ new ShortMinWithRetractAggFunction
case INTEGER =>
- new IntAvgAggFunction
+ new IntMinWithRetractAggFunction
case BIGINT =>
- new LongAvgAggFunction
+ new LongMinWithRetractAggFunction
case FLOAT =>
- new FloatAvgAggFunction
+ new FloatMinWithRetractAggFunction
case DOUBLE =>
- new DoubleAvgAggFunction
+ new DoubleMinWithRetractAggFunction
case DECIMAL =>
- new DecimalAvgAggFunction(tableConfig.getDecimalContext)
+ new DecimalMinWithRetractAggFunction
+ case BOOLEAN =>
+ new BooleanMinWithRetractAggFunction
+ case VARCHAR | CHAR =>
+ new StringMinWithRetractAggFunction
+ case TIMESTAMP =>
+ new TimestampMinWithRetractAggFunction
+ case DATE =>
+ new DateMinWithRetractAggFunction
+ case TIME =>
+ new TimeMinWithRetractAggFunction
case sqlType: SqlTypeName =>
- throw new TableException(s"Avg aggregate does no support type: '$sqlType'")
+ throw new TableException(
+ s"Min with retract aggregate does no support type: '$sqlType'")
}
-
- case sqlMinMaxFunction: SqlMinMaxAggFunction =>
- aggregates(index) = if (sqlMinMaxFunction.getKind == SqlKind.MIN) {
- if (needRetraction) {
- sqlTypeName match {
- case TINYINT =>
- new ByteMinWithRetractAggFunction
- case SMALLINT =>
- new ShortMinWithRetractAggFunction
- case INTEGER =>
- new IntMinWithRetractAggFunction
- case BIGINT =>
- new LongMinWithRetractAggFunction
- case FLOAT =>
- new FloatMinWithRetractAggFunction
- case DOUBLE =>
- new DoubleMinWithRetractAggFunction
- case DECIMAL =>
- new DecimalMinWithRetractAggFunction
- case BOOLEAN =>
- new BooleanMinWithRetractAggFunction
- case VARCHAR | CHAR =>
- new StringMinWithRetractAggFunction
- case TIMESTAMP =>
- new TimestampMinWithRetractAggFunction
- case DATE =>
- new DateMinWithRetractAggFunction
- case TIME =>
- new TimeMinWithRetractAggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(
- s"Min with retract aggregate does no support type: '$sqlType'")
- }
- } else {
- sqlTypeName match {
- case TINYINT =>
- new ByteMinAggFunction
- case SMALLINT =>
- new ShortMinAggFunction
- case INTEGER =>
- new IntMinAggFunction
- case BIGINT =>
- new LongMinAggFunction
- case FLOAT =>
- new FloatMinAggFunction
- case DOUBLE =>
- new DoubleMinAggFunction
- case DECIMAL =>
- new DecimalMinAggFunction
- case BOOLEAN =>
- new BooleanMinAggFunction
- case VARCHAR | CHAR =>
- new StringMinAggFunction
- case TIMESTAMP =>
- new TimestampMinAggFunction
- case DATE =>
- new DateMinAggFunction
- case TIME =>
- new TimeMinAggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(s"Min aggregate does no support type: '$sqlType'")
- }
- }
- } else {
- if (needRetraction) {
- sqlTypeName match {
- case TINYINT =>
- new ByteMaxWithRetractAggFunction
- case SMALLINT =>
- new ShortMaxWithRetractAggFunction
- case INTEGER =>
- new IntMaxWithRetractAggFunction
- case BIGINT =>
- new LongMaxWithRetractAggFunction
- case FLOAT =>
- new FloatMaxWithRetractAggFunction
- case DOUBLE =>
- new DoubleMaxWithRetractAggFunction
- case DECIMAL =>
- new DecimalMaxWithRetractAggFunction
- case BOOLEAN =>
- new BooleanMaxWithRetractAggFunction
- case VARCHAR | CHAR =>
- new StringMaxWithRetractAggFunction
- case TIMESTAMP =>
- new TimestampMaxWithRetractAggFunction
- case DATE =>
- new DateMaxWithRetractAggFunction
- case TIME =>
- new TimeMaxWithRetractAggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(
- s"Max with retract aggregate does no support type: '$sqlType'")
- }
- } else {
- sqlTypeName match {
- case TINYINT =>
- new ByteMaxAggFunction
- case SMALLINT =>
- new ShortMaxAggFunction
- case INTEGER =>
- new IntMaxAggFunction
- case BIGINT =>
- new LongMaxAggFunction
- case FLOAT =>
- new FloatMaxAggFunction
- case DOUBLE =>
- new DoubleMaxAggFunction
- case DECIMAL =>
- new DecimalMaxAggFunction
- case BOOLEAN =>
- new BooleanMaxAggFunction
- case VARCHAR | CHAR =>
- new StringMaxAggFunction
- case TIMESTAMP =>
- new TimestampMaxAggFunction
- case DATE =>
- new DateMaxAggFunction
- case TIME =>
- new TimeMaxAggFunction
- case sqlType: SqlTypeName =>
- throw new TableException(s"Max aggregate does no support type: '$sqlType'")
- }
- }
+ } else {
+ outputTypeName match {
+ case TINYINT =>
+ new ByteMinAggFunction
+ case SMALLINT =>
+ new ShortMinAggFunction
+ case INTEGER =>
+ new IntMinAggFunction
+ case BIGINT =>
+ new LongMinAggFunction
+ case FLOAT =>
+ new FloatMinAggFunction
+ case DOUBLE =>
+ new DoubleMinAggFunction
+ case DECIMAL =>
+ new DecimalMinAggFunction
+ case BOOLEAN =>
+ new BooleanMinAggFunction
+ case VARCHAR | CHAR =>
+ new StringMinAggFunction
+ case TIMESTAMP =>
+ new TimestampMinAggFunction
+ case DATE =>
+ new DateMinAggFunction
+ case TIME =>
+ new TimeMinAggFunction
+ case sqlType: SqlTypeName =>
+ throw new TableException(s"Min aggregate does no support type: '$sqlType'")
}
-
- case collect: SqlAggFunction if collect.getKind == SqlKind.COLLECT =>
- aggregates(index) = new CollectAggFunction(FlinkTypeFactory.toTypeInfo(relDataType))
- accTypes(index) = aggregates(index).getAccumulatorType
-
- case udagg: AggSqlFunction =>
- aggregates(index) = udagg.getFunction
- accTypes(index) = udagg.accType
-
- case unSupported: SqlAggFunction =>
- throw new TableException(s"Unsupported Function: '${unSupported.getName}'")
- }
- }
- }
-
- val accSpecs = new Array[Seq[DataViewSpec[_]]](aggregateCalls.size)
-
- // create accumulator type information for every aggregate function
- aggregates.zipWithIndex.foreach { case (agg, index) =>
- if (accTypes(index) != null) {
- val (accType, specs) = removeStateViewFieldsFromAccTypeInfo(index,
- agg,
- accTypes(index),
- isStateBackedDataViews)
- if (specs.isDefined) {
- accSpecs(index) = specs.get
- accTypes(index) = accType
- } else {
- accSpecs(index) = Seq()
- }
- } else {
- accSpecs(index) = Seq()
- accTypes(index) = getAccumulatorTypeOfAggregateFunction(agg)
- }
- }
-
- // create distinct accumulator filter argument
- val isDistinctAggs = new Array[Boolean](aggregateCalls.size)
-
- aggregateCalls.zipWithIndex.foreach {
- case (aggCall, index) =>
- if (aggCall.isDistinct) {
- // Generate distinct aggregates and the corresponding DistinctAccumulator
- // wrappers for storing distinct mapping
- val argList: util.List[Integer] = aggCall.getArgList
-
- // Using Pojo fields for the real underlying accumulator
- val pojoFields = new util.ArrayList[PojoField]()
- pojoFields.add(new PojoField(
- classOf[DistinctAccumulator[_]].getDeclaredField("realAcc"),
- accTypes(index))
- )
- // If StateBackend is not enabled, the distinct mapping also needs
- // to be added to the Pojo fields.
- if (!isStateBackedDataViews) {
-
- val argTypes: Array[TypeInformation[_]] = argList
- .map(aggregateInputType.getFieldList.get(_).getType)
- .map(FlinkTypeFactory.toTypeInfo).toArray
-
- val mapViewTypeInfo = new MapViewTypeInfo(
- new RowTypeInfo(argTypes:_*),
- BasicTypeInfo.LONG_TYPE_INFO)
- pojoFields.add(new PojoField(
- classOf[DistinctAccumulator[_]].getDeclaredField("distinctValueMap"),
- mapViewTypeInfo)
- )
}
- accTypes(index) = new PojoTypeInfo(classOf[DistinctAccumulator[_]], pojoFields)
- isDistinctAggs(index) = true
} else {
- isDistinctAggs(index) = false
+ if (needRetraction) {
+ outputTypeName match {
+ case TINYINT =>
+ new ByteMaxWithRetractAggFunction
+ case SMALLINT =>
+ new ShortMaxWithRetractAggFunction
+ case INTEGER =>
+ new IntMaxWithRetractAggFunction
+ case BIGINT =>
+ new LongMaxWithRetractAggFunction
+ case FLOAT =>
+ new FloatMaxWithRetractAggFunction
+ case DOUBLE =>
+ new DoubleMaxWithRetractAggFunction
+ case DECIMAL =>
+ new DecimalMaxWithRetractAggFunction
+ case BOOLEAN =>
+ new BooleanMaxWithRetractAggFunction
+ case VARCHAR | CHAR =>
+ new StringMaxWithRetractAggFunction
+ case TIMESTAMP =>
+ new TimestampMaxWithRetractAggFunction
+ case DATE =>
+ new DateMaxWithRetractAggFunction
+ case TIME =>
+ new TimeMaxWithRetractAggFunction
+ case sqlType: SqlTypeName =>
+ throw new TableException(
+ s"Max with retract aggregate does no support type: '$sqlType'")
+ }
+ } else {
+ outputTypeName match {
+ case TINYINT =>
+ new ByteMaxAggFunction
+ case SMALLINT =>
+ new ShortMaxAggFunction
+ case INTEGER =>
+ new IntMaxAggFunction
+ case BIGINT =>
+ new LongMaxAggFunction
+ case FLOAT =>
+ new FloatMaxAggFunction
+ case DOUBLE =>
+ new DoubleMaxAggFunction
+ case DECIMAL =>
+ new DecimalMaxAggFunction
+ case BOOLEAN =>
+ new BooleanMaxAggFunction
+ case VARCHAR | CHAR =>
+ new StringMaxAggFunction
+ case TIMESTAMP =>
+ new TimestampMaxAggFunction
+ case DATE =>
+ new DateMaxAggFunction
+ case TIME =>
+ new TimeMaxAggFunction
+ case sqlType: SqlTypeName =>
+ throw new TableException(s"Max aggregate does no support type: '$sqlType'")
+ }
+ }
}
- }
- (aggFieldIndexes, aggregates, isDistinctAggs, accTypes, accSpecs)
+ case unSupported: SqlAggFunction =>
+ throw new TableException(s"Unsupported Function: '${unSupported.getName}'")
+ }
}
private def createRowTypeForKeysAndAggregates(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala
new file mode 100644
index 00000000000..d9c8e2ccaee
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.aggregate
+
+import org.apache.flink.api.common.state.ValueState
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.streaming.api.functions.co.CoProcessFunction
+import java.lang.{Long => JLong}
+
+import org.apache.flink.streaming.api.TimerService
+
+/**
+ * Base class for clean up state, both for [[ProcessFunction]] and [[CoProcessFunction]].
+ */
+trait CleanupState {
+
+ def registerProcessingCleanupTimer(
+ cleanupTimeState: ValueState[JLong],
+ currentTime: Long,
+ minRetentionTime: Long,
+ maxRetentionTime: Long,
+ timerService: TimerService): Unit = {
+
+ // last registered timer
+ val curCleanupTime = cleanupTimeState.value()
+
+ // check if a cleanup timer is registered and
+ // that the current cleanup timer won't delete state we need to keep
+ if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
+ // we need to register a new (later) timer
+ val cleanupTime = currentTime + maxRetentionTime
+ // register timer and remember clean-up time
+ timerService.registerProcessingTimeTimer(cleanupTime)
+ // delete expired timer
+ if (curCleanupTime != null) {
+ timerService.deleteProcessingTimeTimer(curCleanupTime)
+ }
+ cleanupTimeState.update(cleanupTime)
+ }
+ }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala
new file mode 100644
index 00000000000..0c7663621ba
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.aggregate
+
+import java.lang.{Long => JLong}
+
+import org.apache.flink.api.common.state.{State, ValueState, ValueStateDescriptor}
+import org.apache.flink.streaming.api.TimeDomain
+import org.apache.flink.streaming.api.functions.co.CoProcessFunction
+import org.apache.flink.table.api.{StreamQueryConfig, Types}
+
+abstract class CoProcessFunctionWithCleanupState[IN1, IN2, OUT](queryConfig: StreamQueryConfig)
+ extends CoProcessFunction[IN1, IN2, OUT]
+ with CleanupState {
+
+ protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
+ protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
+ protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
+
+ // holds the latest registered cleanup timer
+ private var cleanupTimeState: ValueState[JLong] = _
+
+ protected def initCleanupTimeState(stateName: String) {
+ if (stateCleaningEnabled) {
+ val cleanupTimeDescriptor: ValueStateDescriptor[JLong] =
+ new ValueStateDescriptor[JLong](stateName, Types.LONG)
+ cleanupTimeState = getRuntimeContext.getState(cleanupTimeDescriptor)
+ }
+ }
+
+ protected def processCleanupTimer(
+ ctx: CoProcessFunction[IN1, IN2, OUT]#Context,
+ currentTime: Long): Unit = {
+ if (stateCleaningEnabled) {
+ registerProcessingCleanupTimer(
+ cleanupTimeState,
+ currentTime,
+ minRetentionTime,
+ maxRetentionTime,
+ ctx.timerService()
+ )
+ }
+ }
+
+ protected def isProcessingTimeTimer(ctx: OnTimerContext): Boolean = {
+ ctx.timeDomain() == TimeDomain.PROCESSING_TIME
+ }
+
+ protected def cleanupState(states: State*): Unit = {
+ // clear all state
+ states.foreach(_.clear())
+ this.cleanupTimeState.clear()
+ }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index c59efe250c3..2d72e6de961 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -86,7 +86,7 @@ class GroupAggProcessFunction(
val currentTime = ctx.timerService().currentProcessingTime()
// register state-cleanup timer
- registerProcessingCleanupTimer(ctx, currentTime)
+ processCleanupTimer(ctx, currentTime)
val input = inputC.row
@@ -172,7 +172,7 @@ class GroupAggProcessFunction(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (needToCleanupState(timestamp)) {
+ if (stateCleaningEnabled) {
cleanupState(state, cntState)
function.cleanup()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala
index 31566159f3b..d9bc8d12dcb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/IncrementalAggregateTimeWindowFunction.scala
@@ -19,7 +19,6 @@ package org.apache.flink.table.runtime.aggregate
import java.lang.Iterable
-import org.apache.flink.api.java.tuple.Tuple
import org.apache.flink.types.Row
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.windowing.windows.TimeWindow
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
index 4d6840a3f43..edf5c2cd101 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
@@ -25,13 +25,15 @@ import org.apache.flink.streaming.api.functions.{KeyedProcessFunction, ProcessFu
import org.apache.flink.table.api.{StreamQueryConfig, Types}
abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: StreamQueryConfig)
- extends KeyedProcessFunction[K, I, O] {
+ extends KeyedProcessFunction[K, I, O]
+ with CleanupState {
+
protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
// holds the latest registered cleanup timer
- private var cleanupTimeState: ValueState[JLong] = _
+ protected var cleanupTimeState: ValueState[JLong] = _
protected def initCleanupTimeState(stateName: String) {
if (stateCleaningEnabled) {
@@ -41,23 +43,17 @@ abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: Stream
}
}
- protected def registerProcessingCleanupTimer(
+ protected def processCleanupTimer(
ctx: KeyedProcessFunction[K, I, O]#Context,
currentTime: Long): Unit = {
if (stateCleaningEnabled) {
-
- // last registered timer
- val curCleanupTime = cleanupTimeState.value()
-
- // check if a cleanup timer is registered and
- // that the current cleanup timer won't delete state we need to keep
- if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
- // we need to register a new (later) timer
- val cleanupTime = currentTime + maxRetentionTime
- // register timer and remember clean-up time
- ctx.timerService().registerProcessingTimeTimer(cleanupTime)
- cleanupTimeState.update(cleanupTime)
- }
+ registerProcessingCleanupTimer(
+ cleanupTimeState,
+ currentTime,
+ minRetentionTime,
+ maxRetentionTime,
+ ctx.timerService()
+ )
}
}
@@ -65,16 +61,6 @@ abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: Stream
ctx.timeDomain() == TimeDomain.PROCESSING_TIME
}
- protected def needToCleanupState(timestamp: Long): Boolean = {
- if (stateCleaningEnabled) {
- val cleanupTime = cleanupTimeState.value()
- // check that the triggered timer is the last registered processing time timer.
- null != cleanupTime && timestamp == cleanupTime
- } else {
- false
- }
- }
-
protected def cleanupState(states: State*): Unit = {
// clear all state
states.foreach(_.clear())
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index 591b942571f..6126dc73b30 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -95,7 +95,7 @@ class ProcTimeBoundedRangeOver(
val currentTime = ctx.timerService.currentProcessingTime
// register state-cleanup timer
- registerProcessingCleanupTimer(ctx, currentTime)
+ processCleanupTimer(ctx, currentTime)
// buffer the event incoming event
@@ -117,11 +117,14 @@ class ProcTimeBoundedRangeOver(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (needToCleanupState(timestamp)) {
- // clean up and return
- cleanupState(rowMapState, accumulatorState)
- function.cleanup()
- return
+ if (stateCleaningEnabled) {
+ val cleanupTime = cleanupTimeState.value()
+ if (null != cleanupTime && timestamp == cleanupTime) {
+ // clean up and return
+ cleanupState(rowMapState, accumulatorState)
+ function.cleanup()
+ return
+ }
}
// remove timestamp set outside of ProcessFunction.
@@ -131,11 +134,10 @@ class ProcTimeBoundedRangeOver(
// that have registered this time trigger 1 ms ago
val currentTime = timestamp - 1
- var i = 0
// get the list of elements of current proctime
val currentElements = rowMapState.get(currentTime)
- // Expired clean-up timers pass the needToCleanupState() check.
+ // Expired clean-up timers pass the needToCleanupState check.
// Perform a null check to verify that we have data to process.
if (null == currentElements) {
return
@@ -156,7 +158,6 @@ class ProcTimeBoundedRangeOver(
// and eliminate them. Multiple elements could have been received at the same timestamp
// the removal of old elements happens only once per proctime as onTimer is called only once
val iter = rowMapState.iterator
- val markToRemove = new ArrayList[Long]()
while (iter.hasNext) {
val entry = iter.next()
val elementKey = entry.getKey
@@ -169,17 +170,9 @@ class ProcTimeBoundedRangeOver(
function.retract(accumulators, retractRow)
iRemove += 1
}
- // mark element for later removal not to modify the iterator over MapState
- markToRemove.add(elementKey)
+ iter.remove()
}
}
- // need to remove in 2 steps not to have concurrent access errors via iterator to the MapState
- i = 0
- while (i < markToRemove.size()) {
- rowMapState.remove(markToRemove.get(i))
- i += 1
- }
-
// add current elements to aggregator. Multiple elements might
// have arrived in the same proctime
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
index ccddaa5b10f..fa58ac50529 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
@@ -110,7 +110,7 @@ class ProcTimeBoundedRowsOver(
val currentTime = ctx.timerService.currentProcessingTime
// register state-cleanup timer
- registerProcessingCleanupTimer(ctx, currentTime)
+ processCleanupTimer(ctx, currentTime)
// initialize state for the processed element
var accumulators = accumulatorState.value
@@ -187,7 +187,7 @@ class ProcTimeBoundedRowsOver(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (needToCleanupState(timestamp)) {
+ if (stateCleaningEnabled) {
cleanupState(rowMapState, accumulatorState, counterState, smallestTsState)
function.cleanup()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
index 6e4c5105786..ce1a95971a9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
@@ -71,7 +71,7 @@ class ProcTimeUnboundedOver(
out: Collector[CRow]): Unit = {
// register state-cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
val input = inputC.row
@@ -95,7 +95,7 @@ class ProcTimeUnboundedOver(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (needToCleanupState(timestamp)) {
+ if (stateCleaningEnabled) {
cleanupState(state)
function.cleanup()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
index 292fd3bdf1e..7263de72c4b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
@@ -26,40 +26,35 @@ import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.table.api.{StreamQueryConfig, Types}
abstract class ProcessFunctionWithCleanupState[IN,OUT](queryConfig: StreamQueryConfig)
- extends ProcessFunction[IN, OUT]{
+ extends ProcessFunction[IN, OUT]
+ with CleanupState {
protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
// holds the latest registered cleanup timer
- private var cleanupTimeState: ValueState[JLong] = _
+ protected var cleanupTimeState: ValueState[JLong] = _
protected def initCleanupTimeState(stateName: String) {
if (stateCleaningEnabled) {
- val inputCntDescriptor: ValueStateDescriptor[JLong] =
+ val cleanupTimeDescriptor: ValueStateDescriptor[JLong] =
new ValueStateDescriptor[JLong](stateName, Types.LONG)
- cleanupTimeState = getRuntimeContext.getState(inputCntDescriptor)
+ cleanupTimeState = getRuntimeContext.getState(cleanupTimeDescriptor)
}
}
- protected def registerProcessingCleanupTimer(
+ protected def processCleanupTimer(
ctx: ProcessFunction[IN, OUT]#Context,
currentTime: Long): Unit = {
if (stateCleaningEnabled) {
-
- // last registered timer
- val curCleanupTime = cleanupTimeState.value()
-
- // check if a cleanup timer is registered and
- // that the current cleanup timer won't delete state we need to keep
- if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
- // we need to register a new (later) timer
- val cleanupTime = currentTime + maxRetentionTime
- // register timer and remember clean-up time
- ctx.timerService().registerProcessingTimeTimer(cleanupTime)
- cleanupTimeState.update(cleanupTime)
- }
+ registerProcessingCleanupTimer(
+ cleanupTimeState,
+ currentTime,
+ minRetentionTime,
+ maxRetentionTime,
+ ctx.timerService()
+ )
}
}
@@ -67,16 +62,6 @@ abstract class ProcessFunctionWithCleanupState[IN,OUT](queryConfig: StreamQueryC
ctx.timeDomain() == TimeDomain.PROCESSING_TIME
}
- protected def needToCleanupState(timestamp: Long): Boolean = {
- if (stateCleaningEnabled) {
- val cleanupTime = cleanupTimeState.value()
- // check that the triggered timer is the last registered processing time timer.
- null != cleanupTime && timestamp == cleanupTime
- } else {
- false
- }
- }
-
protected def cleanupState(states: State*): Unit = {
// clear all state
states.foreach(_.clear())
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index b13acdf43cc..7c509d699e8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -114,7 +114,7 @@ class RowTimeBoundedRangeOver(
val input = inputC.row
// register state-cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
// triggering timestamp for trigger calculation
val triggeringTs = input.getField(rowTimeIdx).asInstanceOf[Long]
@@ -143,7 +143,7 @@ class RowTimeBoundedRangeOver(
out: Collector[CRow]): Unit = {
if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
- if (needToCleanupState(timestamp)) {
+ if (stateCleaningEnabled) {
val keysIt = dataState.keys.iterator()
val lastProcessedTime = lastTriggeringTsState.value
@@ -164,7 +164,7 @@ class RowTimeBoundedRangeOver(
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
// We leave the state as it is and schedule a new cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
}
return
@@ -188,9 +188,6 @@ class RowTimeBoundedRangeOver(
aggregatesIndex = 0
}
- // keep up timestamps of retract data
- val retractTsList: JList[Long] = new JArrayList[Long]
-
// do retraction
val iter = dataState.iterator()
while (iter.hasNext) {
@@ -205,7 +202,7 @@ class RowTimeBoundedRangeOver(
function.retract(accumulators, retractRow)
dataListIndex += 1
}
- retractTsList.add(dataTs)
+ iter.remove()
}
}
@@ -230,20 +227,13 @@ class RowTimeBoundedRangeOver(
dataListIndex += 1
}
- // remove the data that has been retracted
- dataListIndex = 0
- while (dataListIndex < retractTsList.size) {
- dataState.remove(retractTsList.get(dataListIndex))
- dataListIndex += 1
- }
-
// update state
accumulatorState.update(accumulators)
}
lastTriggeringTsState.update(timestamp)
// update cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
override def close(): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index e120d6b0afd..d01a499e88e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -123,7 +123,7 @@ class RowTimeBoundedRowsOver(
val input = inputC.row
// register state-cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
// triggering timestamp for trigger calculation
val triggeringTs = input.getField(rowTimeIdx).asInstanceOf[Long]
@@ -152,7 +152,7 @@ class RowTimeBoundedRowsOver(
out: Collector[CRow]): Unit = {
if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
- if (needToCleanupState(timestamp)) {
+ if (stateCleaningEnabled) {
val keysIt = dataState.keys.iterator()
val lastProcessedTime = lastTriggeringTsState.value
@@ -173,7 +173,7 @@ class RowTimeBoundedRowsOver(
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
// We leave the state as it is and schedule a new cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
}
return
@@ -263,7 +263,7 @@ class RowTimeBoundedRowsOver(
lastTriggeringTsState.update(timestamp)
// update cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
override def close(): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
index 181c7680a35..690d0d05ee3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -108,7 +108,7 @@ abstract class RowTimeUnboundedOver(
val input = inputC.row
// register state-cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
val timestamp = input.getField(rowTimeIdx).asInstanceOf[Long]
val curWatermark = ctx.timerService().currentWatermark()
@@ -143,7 +143,7 @@ abstract class RowTimeUnboundedOver(
out: Collector[CRow]): Unit = {
if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
- if (needToCleanupState(timestamp)) {
+ if (stateCleaningEnabled) {
// we check whether there are still records which have not been processed yet
val noRecordsToProcess = !rowMapState.keys.iterator().hasNext
@@ -155,7 +155,7 @@ abstract class RowTimeUnboundedOver(
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
// We leave the state as it is and schedule a new cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
}
return
@@ -207,7 +207,7 @@ abstract class RowTimeUnboundedOver(
}
// update cleanup timer
- registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
/**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
index 57c60f179c6..5b1069ff28b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
@@ -66,13 +66,12 @@ class NonWindowFullJoin(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
- timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val inputRow = value.row
- updateCurrentSide(value, ctx, timerState, currentSideState)
+ updateCurrentSide(value, ctx, currentSideState)
cRowWrapper.reset()
cRowWrapper.setCollector(out)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
index 9c27eb461a6..0166eef7961 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
@@ -68,14 +68,13 @@ class NonWindowFullJoinWithNonEquiPredicates(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
- timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val currentJoinCntState = getJoinCntState(joinCntState, recordFromLeft)
val inputRow = value.row
- val cntAndExpiredTime = updateCurrentSide(value, ctx, timerState, currentSideState)
+ val cntAndExpiredTime = updateCurrentSide(value, ctx, currentSideState)
if (!value.change && cntAndExpiredTime.f0 <= 0) {
currentJoinCntState.remove(inputRow)
}
@@ -99,18 +98,18 @@ class NonWindowFullJoinWithNonEquiPredicates(
}
/**
- * Removes records which are expired from left state. Register a new timer if the state still
- * holds records after the clean-up. Also, clear leftJoinCnt map state when clear left
- * rowMapState.
+ * Called when a processing timer trigger.
+ * Expire left/right expired records and expired joinCnt state.
*/
- override def expireOutTimeRow(
- curTime: Long,
- rowMapState: MapState[Row, JTuple2[Long, Long]],
- timerState: ValueState[Long],
- isLeft: Boolean,
- ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
+ override def onTimer(
+ timestamp: Long,
+ ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
+ out: Collector[CRow]): Unit = {
- expireOutTimeRow(curTime, rowMapState, timerState, isLeft, joinCntState, ctx)
+ // expired timer has already been removed, delete state directly.
+ if (stateCleaningEnabled) {
+ cleanupState(leftState, rightState, joinCntState(0), joinCntState(1))
+ }
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
index 2e5832c2694..91a75077cd8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
@@ -63,13 +63,12 @@ class NonWindowInnerJoin(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
- timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
isLeft: Boolean): Unit = {
val inputRow = value.row
- updateCurrentSide(value, ctx, timerState, currentSideState)
+ updateCurrentSide(value, ctx, currentSideState)
cRowWrapper.setCollector(out)
cRowWrapper.setChange(value.change)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
index c59f4c2f44b..e15cbfa550c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.runtime.join
import org.apache.flink.api.common.functions.FlatJoinFunction
import org.apache.flink.api.common.functions.util.FunctionUtils
-import org.apache.flink.api.common.state.{MapState, MapStateDescriptor, ValueState, ValueStateDescriptor}
+import org.apache.flink.api.common.state.{MapState, MapStateDescriptor}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
import org.apache.flink.api.java.typeutils.TupleTypeInfo
@@ -27,6 +27,7 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.co.CoProcessFunction
import org.apache.flink.table.api.{StreamQueryConfig, Types}
import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.runtime.aggregate.CoProcessFunctionWithCleanupState
import org.apache.flink.table.runtime.types.CRow
import org.apache.flink.table.typeutils.TypeCheckUtils._
import org.apache.flink.table.util.Logging
@@ -48,7 +49,7 @@ abstract class NonWindowJoin(
genJoinFuncName: String,
genJoinFuncCode: String,
queryConfig: StreamQueryConfig)
- extends CoProcessFunction[CRow, CRow, CRow]
+ extends CoProcessFunctionWithCleanupState[CRow, CRow, CRow](queryConfig)
with Compiler[FlatJoinFunction[Row, Row, Row]]
with Logging {
@@ -62,15 +63,6 @@ abstract class NonWindowJoin(
protected var rightState: MapState[Row, JTuple2[Long, Long]] = _
protected var cRowWrapper: CRowWrappingMultiOutputCollector = _
- protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
- protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
- protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
-
- // state to record last timer of left stream, 0 means no timer
- protected var leftTimer: ValueState[Long] = _
- // state to record last timer of right stream, 0 means no timer
- protected var rightTimer: ValueState[Long] = _
-
// other condition function
protected var joinFunction: FlatJoinFunction[Row, Row, Row] = _
@@ -78,7 +70,8 @@ abstract class NonWindowJoin(
protected var curProcessTime: Long = _
override def open(parameters: Configuration): Unit = {
- LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n Code:\n$genJoinFuncCode")
+ LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n " +
+ s"Code:\n$genJoinFuncCode")
val clazz = compile(
getRuntimeContext.getUserCodeClassLoader,
genJoinFuncName,
@@ -100,10 +93,7 @@ abstract class NonWindowJoin(
rightState = getRuntimeContext.getMapState(rightStateDescriptor)
// initialize timer state
- val valueStateDescriptor1 = new ValueStateDescriptor[Long]("timervaluestate1", classOf[Long])
- leftTimer = getRuntimeContext.getState(valueStateDescriptor1)
- val valueStateDescriptor2 = new ValueStateDescriptor[Long]("timervaluestate2", classOf[Long])
- rightTimer = getRuntimeContext.getState(valueStateDescriptor2)
+ initCleanupTimeState("NonWindowJoinCleanupTime")
cRowWrapper = new CRowWrappingMultiOutputCollector()
LOG.debug("Instantiating NonWindowJoin.")
@@ -122,7 +112,7 @@ abstract class NonWindowJoin(
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow]): Unit = {
- processElement(valueC, ctx, out, leftTimer, leftState, rightState, isLeft = true)
+ processElement(valueC, ctx, out, leftState, rightState, isLeft = true)
}
/**
@@ -138,7 +128,7 @@ abstract class NonWindowJoin(
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow]): Unit = {
- processElement(valueC, ctx, out, rightTimer, rightState, leftState, isLeft = false)
+ processElement(valueC, ctx, out, rightState, leftState, isLeft = false)
}
/**
@@ -154,28 +144,13 @@ abstract class NonWindowJoin(
ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (stateCleaningEnabled && leftTimer.value == timestamp) {
- expireOutTimeRow(
- timestamp,
- leftState,
- leftTimer,
- isLeft = true,
- ctx
- )
- }
-
- if (stateCleaningEnabled && rightTimer.value == timestamp) {
- expireOutTimeRow(
- timestamp,
- rightState,
- rightTimer,
- isLeft = false,
- ctx
- )
+ // expired timer has already been removed, delete state directly.
+ if (stateCleaningEnabled) {
+ cleanupState(leftState, rightState)
}
}
- def getNewExpiredTime(curProcessTime: Long, oldExpiredTime: Long): Long = {
+ protected def getNewExpiredTime(curProcessTime: Long, oldExpiredTime: Long): Long = {
if (stateCleaningEnabled && curProcessTime + minRetentionTime > oldExpiredTime) {
curProcessTime + maxRetentionTime
} else {
@@ -183,53 +158,15 @@ abstract class NonWindowJoin(
}
}
- /**
- * Removes records which are expired from the state. Register a new timer if the state still
- * holds records after the clean-up.
- */
- def expireOutTimeRow(
- curTime: Long,
- rowMapState: MapState[Row, JTuple2[Long, Long]],
- timerState: ValueState[Long],
- isLeft: Boolean,
- ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
-
- val rowMapIter = rowMapState.iterator()
- var validTimestamp: Boolean = false
-
- while (rowMapIter.hasNext) {
- val mapEntry = rowMapIter.next()
- val recordExpiredTime = mapEntry.getValue.f1
- if (recordExpiredTime <= curTime) {
- rowMapIter.remove()
- } else {
- // we found a timestamp that is still valid
- validTimestamp = true
- }
- }
-
- // If the state has non-expired timestamps, register a new timer.
- // Otherwise clean the complete state for this input.
- if (validTimestamp) {
- val cleanupTime = curTime + maxRetentionTime
- ctx.timerService.registerProcessingTimeTimer(cleanupTime)
- timerState.update(cleanupTime)
- } else {
- timerState.clear()
- rowMapState.clear()
- }
- }
-
/**
* Puts or Retract an element from the input stream into state and search the other state to
* output records meet the condition. Records will be expired in state if state retention time
* has been specified.
*/
- def processElement(
+ protected def processElement(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
- timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
isLeft: Boolean): Unit
@@ -240,14 +177,12 @@ abstract class NonWindowJoin(
*
* @param value The input CRow
* @param ctx The ctx to register timer or get current time
- * @param timerState The state to record last timer
* @param currentSideState The state to hold current side stream element
* @return The row number and expired time for current input row
*/
- def updateCurrentSide(
+ protected def updateCurrentSide(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
- timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]]): JTuple2[Long, Long] = {
val inputRow = value.row
@@ -261,10 +196,7 @@ abstract class NonWindowJoin(
cntAndExpiredTime.f1 = getNewExpiredTime(curProcessTime, cntAndExpiredTime.f1)
// update timer if necessary
- if (stateCleaningEnabled && timerState.value() == 0) {
- timerState.update(cntAndExpiredTime.f1)
- ctx.timerService().registerProcessingTimeTimer(cntAndExpiredTime.f1)
- }
+ processCleanupTimer(ctx, curProcessTime)
// update current side stream state
if (!value.change) {
@@ -282,7 +214,7 @@ abstract class NonWindowJoin(
cntAndExpiredTime
}
- def callJoinFunction(
+ protected def callJoinFunction(
inputRow: Row,
inputRowFromLeft: Boolean,
otherSideRow: Row,
@@ -294,8 +226,4 @@ abstract class NonWindowJoin(
joinFunction.join(otherSideRow, inputRow, cRowWrapper)
}
}
-
- override def close(): Unit = {
- FunctionUtils.closeFunction(joinFunction)
- }
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
index b4f14e494f7..5995fb81a2c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
@@ -69,13 +69,12 @@ class NonWindowLeftRightJoin(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
- timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val inputRow = value.row
- updateCurrentSide(value, ctx, timerState, currentSideState)
+ updateCurrentSide(value, ctx, currentSideState)
cRowWrapper.reset()
cRowWrapper.setCollector(out)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
index 33517cca5f9..a3e25f9489d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
@@ -71,14 +71,13 @@ class NonWindowLeftRightJoinWithNonEquiPredicates(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
- timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val currentJoinCntState = getJoinCntState(joinCntState, recordFromLeft)
val inputRow = value.row
- val cntAndExpiredTime = updateCurrentSide(value, ctx, timerState, currentSideState)
+ val cntAndExpiredTime = updateCurrentSide(value, ctx, currentSideState)
if (!value.change && cntAndExpiredTime.f0 <= 0 && recordFromLeft == isLeftJoin) {
currentJoinCntState.remove(inputRow)
}
@@ -101,17 +100,21 @@ class NonWindowLeftRightJoinWithNonEquiPredicates(
}
/**
- * Removes records which are expired from state. Register a new timer if the state still
- * holds records after the clean-up. Also, clear joinCnt map state when clear rowMapState.
+ * Called when a processing timer trigger.
+ * Expire left/right expired records and expired joinCnt state.
*/
- override def expireOutTimeRow(
- curTime: Long,
- rowMapState: MapState[Row, JTuple2[Long, Long]],
- timerState: ValueState[Long],
- isLeft: Boolean,
- ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
+ override def onTimer(
+ timestamp: Long,
+ ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
+ out: Collector[CRow]): Unit = {
- expireOutTimeRow(curTime, rowMapState, timerState, isLeft, joinCntState, ctx)
+ // expired timer has already been removed, delete state directly.
+ if (stateCleaningEnabled) {
+ cleanupState(
+ leftState,
+ rightState,
+ getJoinCntState(joinCntState, isLeftJoin))
+ }
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
index 0018a16bc23..9fadbb0eb64 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
@@ -73,7 +73,7 @@ abstract class NonWindowOuterJoin(
* @param otherSideState the other side state
* @return the number of matched rows
*/
- def preservedJoin(
+ protected def preservedJoin(
inputRow: Row,
inputRowFromLeft: Boolean,
otherSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
@@ -106,7 +106,7 @@ abstract class NonWindowOuterJoin(
* RowWrapper has been reset before we call retractJoin and we also assume that the current
* change of cRowWrapper is equal to value.change.
*/
- def retractJoin(
+ protected def retractJoin(
value: CRow,
inputRowFromLeft: Boolean,
currentSideState: MapState[Row, JTuple2[Long, Long]],
@@ -152,7 +152,8 @@ abstract class NonWindowOuterJoin(
* Return approximate number of records in corresponding state. Only check if record number is
* 0, 1 or bigger.
*/
- def approxiRecordNumInState(currentSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
+ protected def approxiRecordNumInState(
+ currentSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
var recordNum = 0L
val it = currentSideState.iterator()
while(it.hasNext && recordNum < 2) {
@@ -164,7 +165,7 @@ abstract class NonWindowOuterJoin(
/**
* Append input row with default null value if there is no match and Collect.
*/
- def collectAppendNull(
+ protected def collectAppendNull(
inputRow: Row,
inputFromLeft: Boolean,
out: Collector[Row]): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
index 8fe2f4fe410..0eb6a8114fd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
@@ -21,7 +21,6 @@ import org.apache.flink.api.common.state._
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.functions.co.CoProcessFunction
import org.apache.flink.table.api.{StreamQueryConfig, Types}
import org.apache.flink.table.runtime.types.CRow
import org.apache.flink.types.Row
@@ -83,7 +82,7 @@ import org.apache.flink.types.Row
* unmatched or vice versa. The RowWrapper has been reset before we call retractJoin and we
* also assume that the current change of cRowWrapper is equal to value.change.
*/
- def retractJoinWithNonEquiPreds(
+ protected def retractJoinWithNonEquiPreds(
value: CRow,
inputRowFromLeft: Boolean,
otherSideState: MapState[Row, JTuple2[Long, Long]],
@@ -131,48 +130,6 @@ import org.apache.flink.types.Row
}
}
- /**
- * Removes records which are expired from state. Registers a new timer if the state still
- * holds records after the clean-up. Also, clear joinCnt map state when clear rowMapState.
- */
- def expireOutTimeRow(
- curTime: Long,
- rowMapState: MapState[Row, JTuple2[Long, Long]],
- timerState: ValueState[Long],
- isLeft: Boolean,
- joinCntState: Array[MapState[Row, Long]],
- ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
-
- val currentJoinCntState = getJoinCntState(joinCntState, isLeft)
- val rowMapIter = rowMapState.iterator()
- var validTimestamp: Boolean = false
-
- while (rowMapIter.hasNext) {
- val mapEntry = rowMapIter.next()
- val recordExpiredTime = mapEntry.getValue.f1
- if (recordExpiredTime <= curTime) {
- rowMapIter.remove()
- currentJoinCntState.remove(mapEntry.getKey)
- } else {
- // we found a timestamp that is still valid
- validTimestamp = true
- }
- }
- // If the state has non-expired timestamps, register a new timer.
- // Otherwise clean the complete state for this input.
- if (validTimestamp) {
- val cleanupTime = curTime + maxRetentionTime
- ctx.timerService.registerProcessingTimeTimer(cleanupTime)
- timerState.update(cleanupTime)
- } else {
- timerState.clear()
- rowMapState.clear()
- if (isLeft == isLeftJoin) {
- currentJoinCntState.clear()
- }
- }
- }
-
/**
* Get left or right join cnt state.
*
@@ -181,7 +138,7 @@ import org.apache.flink.types.Row
* @param isLeftCntState the flag whether get the left join cnt state
* @return the corresponding join cnt state
*/
- def getJoinCntState(
+ protected def getJoinCntState(
joinCntState: Array[MapState[Row, Long]],
isLeftCntState: Boolean)
: MapState[Row, Long] = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/MatchUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/MatchUtil.scala
deleted file mode 100644
index 06bab956bc4..00000000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/match/MatchUtil.scala
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.runtime.`match`
-
-import java.util
-
-import org.apache.calcite.rel.RelFieldCollation
-import org.apache.calcite.rex.RexNode
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.cep.pattern.conditions.RichIterativeCondition
-import org.apache.flink.cep.{PatternFlatSelectFunction, RichPatternSelectFunction}
-import org.apache.flink.table.api.TableConfig
-import org.apache.flink.table.codegen.MatchCodeGenerator
-import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.table.runtime.types.CRow
-import org.apache.flink.types.Row
-
-/**
- * An util class to generate match functions.
- */
-object MatchUtil {
-
- private[flink] def generateIterativeCondition(
- config: TableConfig,
- patternDefinition: RexNode,
- inputTypeInfo: TypeInformation[_],
- patternName: String,
- names: Seq[String])
- : IterativeConditionRunner = {
- val generator = new MatchCodeGenerator(config, inputTypeInfo, names, Some(patternName))
- val condition = generator.generateExpression(patternDefinition)
- val body =
- s"""
- |${condition.code}
- |return ${condition.resultTerm};
- |""".stripMargin
-
- val genCondition = generator
- .generateMatchFunction("MatchRecognizeCondition",
- classOf[RichIterativeCondition[Row]],
- body,
- condition.resultType)
- new IterativeConditionRunner(genCondition.name, genCondition.code)
- }
-
- private[flink] def generateOneRowPerMatchExpression(
- config: TableConfig,
- returnType: RowSchema,
- partitionKeys: util.List[RexNode],
- orderKeys: util.List[RelFieldCollation],
- measures: util.Map[String, RexNode],
- inputTypeInfo: TypeInformation[_],
- patternNames: Seq[String])
- : PatternFlatSelectFunction[Row, CRow] = {
- val generator = new MatchCodeGenerator(config, inputTypeInfo, patternNames)
-
- val resultExpression = generator.generateOneRowPerMatchExpression(
- partitionKeys,
- measures,
- returnType)
- val body =
- s"""
- |${resultExpression.code}
- |return ${resultExpression.resultTerm};
- |""".stripMargin
-
- val genFunction = generator.generateMatchFunction(
- "MatchRecognizePatternSelectFunction",
- classOf[RichPatternSelectFunction[Row, Row]],
- body,
- resultExpression.resultType)
- new PatternSelectFunctionRunner(genFunction.name, genFunction.code)
- }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
index 3c18449e953..6ae5e6340a6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
@@ -71,6 +71,10 @@ class StateCleaningCountTrigger(queryConfig: StreamQueryConfig, maxCount: Long)
val cleanupTime = currentTime + maxRetentionTime
// register timer and remember clean-up time
ctx.registerProcessingTimeTimer(cleanupTime)
+ // delete expired timer
+ if (curCleanupTime != null) {
+ ctx.deleteProcessingTimeTimer(curCleanupTime)
+ }
ctx.getPartitionedState(cleanupStateDesc).update(cleanupTime)
}
@@ -93,12 +97,9 @@ class StateCleaningCountTrigger(queryConfig: StreamQueryConfig, maxCount: Long)
ctx: TriggerContext): TriggerResult = {
if (stateCleaningEnabled) {
- val cleanupTime = ctx.getPartitionedState(cleanupStateDesc).value()
- // check that the triggered timer is the last registered processing time timer.
- if (null != cleanupTime && time == cleanupTime) {
- clear(window, ctx)
- return TriggerResult.FIRE_AND_PURGE
- }
+ // do clear directly, since we have already deleted expired timer
+ clear(window, ctx)
+ return TriggerResult.FIRE_AND_PURGE
}
TriggerResult.CONTINUE
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
index 0ce3aee3739..b3fe5085151 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala
@@ -81,7 +81,7 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali
// --------------------------------------------------------------------------------------------
override def snapshotConfiguration(): TypeSerializerConfigSnapshot[CRow] = {
- new CRowSerializer.CRowSerializerConfigSnapshot(rowSerializer)
+ new CRowSerializer.CRowSerializerConfigSnapshot(Array(rowSerializer))
}
override def ensureCompatibility(
@@ -115,9 +115,13 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali
object CRowSerializer {
- class CRowSerializerConfigSnapshot(rowSerializers: TypeSerializer[Row]*)
+ class CRowSerializerConfigSnapshot(rowSerializers: Array[TypeSerializer[Row]])
extends CompositeTypeSerializerConfigSnapshot[CRow](rowSerializers: _*) {
+ def this() {
+ this(Array.empty)
+ }
+
override def getVersion: Int = CRowSerializerConfigSnapshot.VERSION
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/MatchUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/MatchUtil.scala
new file mode 100644
index 00000000000..0cce17153ef
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/MatchUtil.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.util
+
+import org.apache.calcite.rex.{RexCall, RexNode, RexPatternFieldRef}
+import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.plan.util.RexDefaultVisitor
+import scala.collection.JavaConverters._
+
+object MatchUtil {
+ val ALL_PATTERN_VARIABLE = "*"
+
+ class AggregationPatternVariableFinder extends RexDefaultVisitor[Option[String]] {
+
+ override def visitPatternFieldRef(patternFieldRef: RexPatternFieldRef): Option[String] = Some(
+ patternFieldRef.getAlpha)
+
+ override def visitCall(call: RexCall): Option[String] = {
+ if (call.operands.size() == 0) {
+ Some(ALL_PATTERN_VARIABLE)
+ } else {
+ call.operands.asScala.map(n => n.accept(this)).reduce((op1, op2) => (op1, op2) match {
+ case (None, None) => None
+ case (x, None) => x
+ case (None, x) => x
+ case (Some(var1), Some(var2)) if var1.equals(var2) =>
+ Some(var1)
+ case _ =>
+ throw new ValidationException(s"Aggregation must be applied to a single pattern " +
+ s"variable. Malformed expression: $call")
+ })
+ }
+ }
+
+ override def visitNode(rexNode: RexNode): Option[String] = None
+ }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala
index 1c097d3b19c..91077269e83 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala
@@ -531,17 +531,17 @@ class TableEnvironmentTest extends TableTestBase {
// case class
util.verifySchema(
- util.addTable[CClassWithTime]('cf1, ('cf2 as 'new).rowtime, 'cf3),
+ util.addTable[CClassWithTime]('cf1, 'cf2.rowtime as 'new, 'cf3),
Seq("cf1" -> INT, "new" -> ROWTIME, "cf3" -> STRING))
// row
util.verifySchema(
- util.addTable('rf1, ('rf2 as 'new).rowtime, 'rf3)(TEST_ROW_WITH_TIME),
+ util.addTable('rf1, 'rf2.rowtime as 'new, 'rf3)(TEST_ROW_WITH_TIME),
Seq("rf1" -> INT, "new" -> ROWTIME, "rf3" -> STRING))
// tuple
util.verifySchema(
- util.addTable[JTuple3[Int, Long, String]]('f0, ('f1 as 'new).rowtime, 'f2),
+ util.addTable[JTuple3[Int, Long, String]]('f0, 'f1.rowtime as 'new, 'f2),
Seq("f0" -> INT, "new" -> ROWTIME, "f2" -> STRING))
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala
index bfa7bfa805b..e256ee89b75 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala
@@ -38,7 +38,7 @@ class StreamTableEnvironmentValidationTest extends TableTestBase {
def testInvalidRowtimeAliasByPosition(): Unit = {
val util = streamTestUtil()
// don't allow aliasing by position
- util.addTable[(Long, Int, String, Int, Long)](('a as 'b).rowtime, 'b, 'c, 'd, 'e)
+ util.addTable[(Long, Int, String, Int, Long)]('a.rowtime as 'b, 'b, 'c, 'd, 'e)
}
@Test(expected = classOf[TableException])
@@ -178,13 +178,13 @@ class StreamTableEnvironmentValidationTest extends TableTestBase {
def testInvalidAliasWithRowtimeAttribute(): Unit = {
val util = streamTestUtil()
// aliased field does not exist
- util.addTable[(Int, Long, String)]('_1, ('newnew as 'new).rowtime, '_3)
+ util.addTable[(Int, Long, String)]('_1, 'newnew.rowtime as 'new, '_3)
}
@Test(expected = classOf[TableException])
def testInvalidAliasWithRowtimeAttribute2(): Unit = {
val util = streamTestUtil()
// aliased field has wrong type
- util.addTable[(Int, Long, String)]('_1, ('_3 as 'new).rowtime, '_2)
+ util.addTable[(Int, Long, String)]('_1, '_3.rowtime as 'new, '_2)
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala
index ec57436b420..0833c240c28 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.api.stream.table.stringexpr
import org.apache.flink.api.scala._
import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.java.{Tumble => JTumble}
import org.apache.flink.table.functions.aggfunctions.CountAggFunction
import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMergeAndReset}
import org.apache.flink.table.utils.TableTestBase
@@ -128,4 +129,50 @@ class AggregateStringExpressionTest extends TableTestBase {
verifyTableEquals(resJava, resScala)
}
+
+ @Test
+ def testProctimeRename(): Unit = {
+ val util = streamTestUtil()
+ val t = util.addTable[(Int, Long, String)]('int, 'long, 'string, 'p.proctime as 'proctime)
+
+ // Expression / Scala API
+ val resScala = t
+ .window(Tumble over 50.milli on 'proctime as 'w1)
+ .groupBy('w1, 'string)
+ .select('w1.proctime as 'proctime, 'w1.start as 'start, 'w1.end as 'end, 'string, 'int.count)
+
+ // String / Java API
+ val resJava = t
+ .window(JTumble.over("50.milli").on("proctime").as("w1"))
+ .groupBy("w1, string")
+ .select("w1.proctime as proctime, w1.start as start, w1.end as end, string, int.count")
+
+ verifyTableEquals(resJava, resScala)
+ }
+
+ @Test
+ def testRowtimeRename(): Unit = {
+ val util = streamTestUtil()
+ val t = util.addTable[TestPojo]('int, 'long.rowtime as 'rowtime, 'string)
+
+ // Expression / Scala API
+ val resScala = t
+ .window(Tumble over 50.milli on 'rowtime as 'w1)
+ .groupBy('w1, 'string)
+ .select('w1.rowtime as 'rowtime, 'string, 'int.count)
+
+ // String / Java API
+ val resJava = t
+ .window(JTumble.over("50.milli").on("rowtime").as("w1"))
+ .groupBy("w1, string")
+ .select("w1.rowtime as rowtime, string, int.count")
+
+ verifyTableEquals(resJava, resScala)
+ }
+}
+
+class TestPojo() {
+ var int: Int = _
+ var long: Long = _
+ var string: String = _
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchOperatorValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala
similarity index 84%
rename from flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchOperatorValidationTest.scala
rename to flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala
index e10a568b2eb..b77a60e1af5 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchOperatorValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala
@@ -21,13 +21,13 @@ package org.apache.flink.table.`match`
import org.apache.flink.api.scala._
import org.apache.flink.table.api.scala._
import org.apache.flink.table.api.{TableException, ValidationException}
-import org.apache.flink.table.codegen.CodeGenException
import org.apache.flink.table.runtime.stream.sql.ToMillis
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg
import org.apache.flink.table.utils.TableTestBase
import org.apache.flink.types.Row
import org.junit.Test
-class MatchOperatorValidationTest extends TableTestBase {
+class MatchRecognizeValidationTest extends TableTestBase {
private val streamUtils = streamTestUtil()
streamUtils.addTable[(String, Long, Int, Int)]("Ticker",
@@ -128,6 +128,52 @@ class MatchOperatorValidationTest extends TableTestBase {
streamUtils.tableEnv.sqlQuery(sqlQuery).toRetractStream[Row]
}
+ @Test
+ def testAggregatesOnMultiplePatternVariablesNotSupported(): Unit = {
+ thrown.expect(classOf[ValidationException])
+ thrown.expectMessage("SQL validation failed.")
+
+ val sqlQuery =
+ s"""
+ |SELECT *
+ |FROM Ticker
+ |MATCH_RECOGNIZE (
+ | ORDER BY proctime
+ | MEASURES
+ | SUM(A.price + B.tax) AS taxedPrice
+ | PATTERN (A B)
+ | DEFINE
+ | A AS A.symbol = 'a'
+ |) AS T
+ |""".stripMargin
+
+ streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row]
+ }
+
+ @Test
+ def testAggregatesOnMultiplePatternVariablesNotSupportedInUDAGs(): Unit = {
+ thrown.expect(classOf[ValidationException])
+ thrown.expectMessage("Aggregation must be applied to a single pattern variable")
+
+ streamUtils.tableEnv.registerFunction("weightedAvg", new WeightedAvg)
+
+ val sqlQuery =
+ s"""
+ |SELECT *
+ |FROM Ticker
+ |MATCH_RECOGNIZE (
+ | ORDER BY proctime
+ | MEASURES
+ | weightedAvg(A.price, B.tax) AS weightedAvg
+ | PATTERN (A B)
+ | DEFINE
+ | A AS A.symbol = 'a'
+ |) AS T
+ |""".stripMargin
+
+ streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row]
+ }
+
// ***************************************************************************************
// * Those validations are temporary. We should remove those tests once we support those *
// * features. *
@@ -203,11 +249,8 @@ class MatchOperatorValidationTest extends TableTestBase {
}
@Test
- def testAggregatesAreNotSupportedInMeasures(): Unit = {
- thrown.expectMessage(
- "Unsupported call: SUM \nIf you think this function should be supported, you can " +
- "create an issue and start a discussion for it.")
- thrown.expect(classOf[CodeGenException])
+ def testDistinctAggregationsNotSupported(): Unit = {
+ thrown.expect(classOf[ValidationException])
val sqlQuery =
s"""
@@ -216,7 +259,7 @@ class MatchOperatorValidationTest extends TableTestBase {
|MATCH_RECOGNIZE (
| ORDER BY proctime
| MEASURES
- | SUM(A.price + A.tax) AS cost
+ | COUNT(DISTINCT A.price) AS price
| PATTERN (A B)
| DEFINE
| A AS A.symbol = 'a'
@@ -225,28 +268,4 @@ class MatchOperatorValidationTest extends TableTestBase {
streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row]
}
-
- @Test
- def testAggregatesAreNotSupportedInDefine(): Unit = {
- thrown.expectMessage(
- "Unsupported call: SUM \nIf you think this function should be supported, you can " +
- "create an issue and start a discussion for it.")
- thrown.expect(classOf[CodeGenException])
-
- val sqlQuery =
- s"""
- |SELECT *
- |FROM Ticker
- |MATCH_RECOGNIZE (
- | ORDER BY proctime
- | MEASURES
- | B.price as bPrice
- | PATTERN (A+ B)
- | DEFINE
- | A AS SUM(A.price + A.tax) < 10
- |) AS T
- |""".stripMargin
-
- streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row]
- }
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/AggFunctionHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/AggFunctionHarnessTest.scala
new file mode 100644
index 00000000000..0549339381d
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/AggFunctionHarnessTest.scala
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.harness
+
+import java.lang.{Integer => JInt}
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import org.apache.flink.api.common.time.Time
+import org.apache.flink.api.scala._
+import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.dataview.MapView
+import org.apache.flink.table.dataview.StateMapView
+import org.apache.flink.table.runtime.aggregate.GroupAggProcessFunction
+import org.apache.flink.table.runtime.harness.HarnessTestBase.TestStreamQueryConfig
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+class AggFunctionHarnessTest extends HarnessTestBase {
+ private val queryConfig = new TestStreamQueryConfig(Time.seconds(0), Time.seconds(0))
+
+ @Test
+ def testCollectAggregate(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val data = new mutable.MutableList[(JInt, String)]
+ val t = env.fromCollection(data).toTable(tEnv, 'a, 'b)
+ tEnv.registerTable("T", t)
+ val sqlQuery = tEnv.sqlQuery(
+ s"""
+ |SELECT
+ | b, collect(a)
+ |FROM (
+ | SELECT a, b
+ | FROM T
+ | GROUP BY a, b
+ |) GROUP BY b
+ |""".stripMargin)
+
+ val testHarness = createHarnessTester[String, CRow, CRow](
+ sqlQuery.toRetractStream[Row](queryConfig), "groupBy")
+
+ testHarness.setStateBackend(getStateBackend)
+ testHarness.open()
+
+ val operator = getOperator(testHarness)
+ val state = getState(
+ operator,
+ "function",
+ classOf[GroupAggProcessFunction],
+ "acc0_map_dataview").asInstanceOf[MapView[JInt, JInt]]
+ assertTrue(state.isInstanceOf[StateMapView[_, _]])
+ assertTrue(operator.getKeyedStateBackend.isInstanceOf[RocksDBKeyedStateBackend[_]])
+
+ val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+ testHarness.processElement(new StreamRecord(CRow(1: JInt, "aaa"), 1))
+ expectedOutput.add(new StreamRecord(CRow("aaa", Map(1 -> 1).asJava), 1))
+
+ testHarness.processElement(new StreamRecord(CRow(1: JInt, "bbb"), 1))
+ expectedOutput.add(new StreamRecord(CRow("bbb", Map(1 -> 1).asJava), 1))
+
+ testHarness.processElement(new StreamRecord(CRow(1: JInt, "aaa"), 1))
+ expectedOutput.add(new StreamRecord(CRow(false, "aaa", Map(1 -> 1).asJava), 1))
+ expectedOutput.add(new StreamRecord(CRow("aaa", Map(1 -> 2).asJava), 1))
+
+ testHarness.processElement(new StreamRecord(CRow(2: JInt, "aaa"), 1))
+ expectedOutput.add(new StreamRecord(CRow(false, "aaa", Map(1 -> 2).asJava), 1))
+ expectedOutput.add(new StreamRecord(CRow("aaa", Map(1 -> 2, 2 -> 1).asJava), 1))
+
+ // remove some state: state may be cleaned up by the state backend
+ // if not accessed beyond ttl time
+ operator.setCurrentKey(Row.of("aaa"))
+ state.remove(2)
+
+ // retract after state has been cleaned up
+ testHarness.processElement(new StreamRecord(CRow(false, 2: JInt, "aaa"), 1))
+
+ val result = testHarness.getOutput
+
+ verify(expectedOutput, result)
+
+ testHarness.close()
+ }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
index e5cceecc560..c37fd0cf5ca 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
@@ -25,27 +25,27 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{LONG_TYPE_INFO, STRIN
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator
+import org.apache.flink.streaming.api.operators.{AbstractUdfStreamOperator, OneInputStreamOperator}
+import org.apache.flink.streaming.api.scala.DataStream
+import org.apache.flink.streaming.api.transformations._
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
-import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil}
+import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, OneInputStreamOperatorTestHarness, TestHarnessUtil}
+import org.apache.flink.table.api.dataview.DataView
import org.apache.flink.table.api.{StreamQueryConfig, Types}
import org.apache.flink.table.codegen.GeneratedAggregationsFunction
import org.apache.flink.table.functions.aggfunctions.{CountAggFunction, IntSumWithRetractAggFunction, LongMaxWithRetractAggFunction, LongMinWithRetractAggFunction}
import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.getAccumulatorTypeOfAggregateFunction
import org.apache.flink.table.functions.{AggregateFunction, UserDefinedFunction}
+import org.apache.flink.table.runtime.aggregate.GeneratedAggregations
import org.apache.flink.table.runtime.harness.HarnessTestBase.{RowResultSortComparator, RowResultSortComparatorWithWatermarks}
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.table.runtime.utils.StreamingWithStateTestBase
import org.apache.flink.table.utils.EncodingUtils
-import org.junit.Rule
-import org.junit.rules.ExpectedException
-class HarnessTestBase {
- // used for accurate exception information checking.
- val expectedException = ExpectedException.none()
+import _root_.scala.collection.JavaConversions._
- @Rule
- def thrown = expectedException
+class HarnessTestBase extends StreamingWithStateTestBase {
val longMinWithRetractAggFunction: String =
EncodingUtils.encodeObjectToString(new LongMinWithRetractAggFunction)
@@ -491,6 +491,68 @@ class HarnessTestBase {
distinctCountFuncName,
distinctCountAggCode)
+ def createHarnessTester[KEY, IN, OUT](
+ dataStream: DataStream[_],
+ prefixOperatorName: String)
+ : KeyedOneInputStreamOperatorTestHarness[KEY, IN, OUT] = {
+
+ val transformation = extractExpectedTransformation(
+ dataStream.javaStream.getTransformation,
+ prefixOperatorName).asInstanceOf[OneInputTransformation[_, _]]
+ if (transformation == null) {
+ throw new Exception("Can not find the expected transformation")
+ }
+
+ val processOperator = transformation.getOperator.asInstanceOf[OneInputStreamOperator[IN, OUT]]
+ val keySelector = transformation.getStateKeySelector.asInstanceOf[KeySelector[IN, KEY]]
+ val keyType = transformation.getStateKeyType.asInstanceOf[TypeInformation[KEY]]
+
+ createHarnessTester(processOperator, keySelector, keyType)
+ .asInstanceOf[KeyedOneInputStreamOperatorTestHarness[KEY, IN, OUT]]
+ }
+
+ private def extractExpectedTransformation(
+ transformation: StreamTransformation[_],
+ prefixOperatorName: String): StreamTransformation[_] = {
+ def extractFromInputs(inputs: StreamTransformation[_]*): StreamTransformation[_] = {
+ for (input <- inputs) {
+ val t = extractExpectedTransformation(input, prefixOperatorName)
+ if (t != null) {
+ return t
+ }
+ }
+ null
+ }
+
+ transformation match {
+ case one: OneInputTransformation[_, _] =>
+ if (one.getName.startsWith(prefixOperatorName)) {
+ one
+ } else {
+ extractExpectedTransformation(one.getInput, prefixOperatorName)
+ }
+ case union: UnionTransformation[_] => extractFromInputs(union.getInputs.toSeq: _*)
+ case p: PartitionTransformation[_] => extractFromInputs(p.getInput)
+ case _: SourceTransformation[_] => null
+ case _ => throw new UnsupportedOperationException("This should not happen.")
+ }
+ }
+
+ def getState(
+ operator: AbstractUdfStreamOperator[_, _],
+ funcName: String,
+ funcClass: Class[_],
+ stateFieldName: String): DataView = {
+ val function = funcClass.getDeclaredField(funcName)
+ function.setAccessible(true)
+ val generatedAggregation =
+ function.get(operator.getUserFunction).asInstanceOf[GeneratedAggregations]
+ val cls = generatedAggregation.getClass
+ val stateField = cls.getDeclaredField(stateFieldName)
+ stateField.setAccessible(true)
+ stateField.get(generatedAggregation).asInstanceOf[DataView]
+ }
+
def createHarnessTester[IN, OUT, KEY](
operator: OneInputStreamOperator[IN, OUT],
keySelector: KeySelector[IN, KEY],
@@ -498,6 +560,14 @@ class HarnessTestBase {
new KeyedOneInputStreamOperatorTestHarness[KEY, IN, OUT](operator, keySelector, keyType)
}
+ def getOperator(testHarness: OneInputStreamOperatorTestHarness[_, _])
+ : AbstractUdfStreamOperator[_, _] = {
+ val operatorField = classOf[OneInputStreamOperatorTestHarness[_, _]]
+ .getDeclaredField("oneInputOperator")
+ operatorField.setAccessible(true)
+ operatorField.get(testHarness).asInstanceOf[AbstractUdfStreamOperator[_, _]]
+ }
+
def verify(expected: JQueue[Object], actual: JQueue[Object]): Unit = {
verify(expected, actual, new RowResultSortComparator)
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
index bd19be83757..4619c759c31 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
@@ -21,21 +21,18 @@ import java.lang.{Integer => JInt, Long => JLong}
import java.util.concurrent.ConcurrentLinkedQueue
import org.apache.flink.api.common.time.Time
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.api.java.operators.join.JoinType
-import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.streaming.api.operators.co.KeyedCoProcessOperator
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
import org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness
-import org.apache.flink.table.api.{StreamQueryConfig, Types}
-import org.apache.flink.table.runtime.harness.HarnessTestBase.{RowResultSortComparator, RowResultSortComparatorWithWatermarks, TestStreamQueryConfig, TupleRowKeySelector}
+import org.apache.flink.table.api.Types
+import org.apache.flink.table.runtime.harness.HarnessTestBase.{TestStreamQueryConfig, TupleRowKeySelector}
import org.apache.flink.table.runtime.join._
import org.apache.flink.table.runtime.operators.KeyedCoProcessOperatorWithWatermarkDelay
-import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.apache.flink.types.Row
-import org.junit.Assert.{assertEquals, assertTrue}
+import org.apache.flink.table.runtime.types.CRow
+import org.junit.Assert.assertEquals
import org.junit.Test
/**
@@ -830,14 +827,6 @@ class JoinHarnessTest extends HarnessTestBase {
@Test
def testNonWindowInnerJoin() {
- val joinReturnType = CRowTypeInfo(new RowTypeInfo(
- Array[TypeInformation[_]](
- INT_TYPE_INFO,
- STRING_TYPE_INFO,
- INT_TYPE_INFO,
- STRING_TYPE_INFO),
- Array("a", "b", "c", "d")))
-
val joinProcessFunc = new NonWindowInnerJoin(
rowType,
rowType,
@@ -879,35 +868,32 @@ class JoinHarnessTest extends HarnessTestBase {
// right stream input and output normally
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi1")))
- assertEquals(6, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(1) timer_key_time(1:5, 2:6)
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello1")))
- assertEquals(8, testHarness.numKeyedStateEntries())
- assertEquals(4, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
+ assertEquals(6, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired left stream record with key value of 1
+ // expired stream record with key value of 1
testHarness.setProcessingTime(5)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi2")))
- assertEquals(6, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
-
- // expired all left stream record
- testHarness.setProcessingTime(6)
- assertEquals(4, testHarness.numKeyedStateEntries())
+ // lkeys(2) rkeys(1, 2) timer_key_time(1:9, 2:6)
+ assertEquals(5, testHarness.numKeyedStateEntries())
assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired right stream record with key value of 2
- testHarness.setProcessingTime(8)
+ // expired all left stream records
+ testHarness.setProcessingTime(6)
+ // lkeys() rkeys(1) timer_key_time(1:9)
assertEquals(2, testHarness.numKeyedStateEntries())
assertEquals(1, testHarness.numProcessingTimeTimers())
- testHarness.setProcessingTime(10)
- assertTrue(testHarness.numKeyedStateEntries() > 0)
- // expired all right stream record
- testHarness.setProcessingTime(11)
+ // expired all stream records
+ testHarness.setProcessingTime(9)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -975,32 +961,37 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
+ assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello1")))
- assertEquals(7, testHarness.numKeyedStateEntries())
- assertEquals(4, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
- // expired left stream record with key value of 1
+ // expired stream records with key value of 1
testHarness.setProcessingTime(5)
+ // lkeys(2) rkeys(2) timer_key_time(2:6)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi2")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi2")))
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
+ assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all left stream record
+ // expired all stream records
testHarness.setProcessingTime(6)
- assertEquals(3, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // lkeys() rkeys() timer_key_time(1:9)
+ assertEquals(1, testHarness.numKeyedStateEntries())
+ assertEquals(1, testHarness.numProcessingTimeTimers())
- // expired right stream record with key value of 2
- testHarness.setProcessingTime(8)
+ // expired all data
+ testHarness.setProcessingTime(9)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1067,32 +1058,36 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
+ assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello1")))
- assertEquals(7, testHarness.numKeyedStateEntries())
- assertEquals(4, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
- // expired left stream record with key value of 1
+ // expired stream records with key value of 1
testHarness.setProcessingTime(5)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi2")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi2")))
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
+ assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all left stream record
+ // expired stream records with key value of 2
testHarness.setProcessingTime(6)
- assertEquals(3, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // lkeys() rkeys() timer_key_time(1:9)
+ assertEquals(1, testHarness.numKeyedStateEntries())
+ assertEquals(1, testHarness.numProcessingTimeTimers())
- // expired right stream record with key value of 2
- testHarness.setProcessingTime(8)
+ // expired all data
+ testHarness.setProcessingTime(9)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1160,7 +1155,7 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "bbb")))
assertEquals(1, testHarness.numProcessingTimeTimers())
- // 1 left timer(5), 1 left key(1), 1 join cnt
+ // lkeys(1) rkeys() timer_key_time(1:5)
assertEquals(3, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
testHarness.processElement1(new StreamRecord(
@@ -1168,7 +1163,8 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "bbb")))
assertEquals(2, testHarness.numProcessingTimeTimers())
- // 2 left timer(5,6), 2 left key(1,2), 2 join cnt
+ // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
+ // l_join_cnt_keys(1, 2)
assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1177,17 +1173,19 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "bbb")))
- // 2 left timer(5,6), 2 left keys(1,2), 2 join cnt, 1 right timer(7), 1 right key(1)
- assertEquals(8, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(1) timer_key_time(1:5, 2:6)
+ // l_join_cnt_keys(1, 2)
+ assertEquals(7, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "ccc")))
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello")))
- // 2 left timer(5,6), 2 left keys(1,2), 2 join cnt, 2 right timer(7,8), 2 right key(1,2)
- assertEquals(10, testHarness.numKeyedStateEntries())
- assertEquals(4, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
+ // l_join_cnt_keys(1, 2)
+ assertEquals(8, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
@@ -1197,22 +1195,29 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "Hi2")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- // expired left stream record with key value of 1
+ // lkeys(1, 2) rkeys(2) timer_key_time(1:8, 2:6)
+ // l_join_cnt_keys(1, 2)
+ assertEquals(7, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(5)
+ // [1]. this will clean up left stream records with expired time of 5
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi3")))
+ // [2]. there are no elements can be connected, since be cleaned by [1]
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi3")))
- // 1 left timer(6), 1 left keys(2), 1 join cnt, 2 right timer(7,8), 1 right key(2)
- assertEquals(6, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(2) timer_key_time(1:8, 2:6)
+ // l_join_cnt_keys(1, 2)
+ assertEquals(7, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all left stream record
+ // expired all records with key value of 2
testHarness.setProcessingTime(6)
+ // lkeys(1) rkeys() timer_key_time(1:8)
+ // l_join_cnt_keys(1)
assertEquals(3, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(1, testHarness.numProcessingTimeTimers())
- // expired right stream record with key value of 2
+ // expired all data
testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1253,6 +1258,12 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "bbb", 1: JInt, "Hi1")))
expectedOutput.add(new StreamRecord(
CRow(1: JInt, "bbb", null: JInt, null)))
+ // processing time of 5
+ // timer of 8, we use only one timer state now
+ expectedOutput.add(new StreamRecord(
+ CRow(false, 1: JInt, "bbb", null: JInt, null)))
+ expectedOutput.add(new StreamRecord(
+ CRow(1: JInt, "bbb", 1: JInt, "Hi3")))
verify(expectedOutput, result)
testHarness.close()
@@ -1305,32 +1316,36 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys() rkeys(1, 2) timer_key_time(1:5, 2:6)
+ assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "Hello1")))
- assertEquals(7, testHarness.numKeyedStateEntries())
- assertEquals(4, testHarness.numProcessingTimeTimers())
+ // lkeys(2) rkeys(1, 2) timer_key_time(1:5, 2:6)
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
- // expired right stream record with key value of 1
+ // expired stream records with key value of 1
testHarness.setProcessingTime(5)
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "Hi2")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi2")))
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
+ assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all right stream record
+ // expired stream records with key value of 2
testHarness.setProcessingTime(6)
- assertEquals(3, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // lkeys() rkeys() timer_key_time(1:9)
+ assertEquals(1, testHarness.numKeyedStateEntries())
+ assertEquals(1, testHarness.numProcessingTimeTimers())
- // expired left stream record with key value of 2
- testHarness.setProcessingTime(8)
+ // expired all data
+ testHarness.setProcessingTime(9)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1398,15 +1413,17 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "bbb")))
assertEquals(1, testHarness.numProcessingTimeTimers())
- // 1 right timer(5), 1 right key(1), 1 join cnt
+ // lkeys() rkeys(1) timer_key_time(1:5)
+ // r_join_cnt_keys(1)
assertEquals(3, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "bbb")))
+ // lkeys() rkeys(1, 2) timer_key_time(1:5, 2:6)
+ // r_join_cnt_keys(1, 2)
assertEquals(2, testHarness.numProcessingTimeTimers())
- // 2 right timer(5,6), 2 right key(1,2), 2 join cnt
assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1415,17 +1432,19 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "bbb")))
- // 2 right timer(5,6), 2 right keys(1,2), 2 join cnt, 1 left timer(7), 1 left key(1)
- assertEquals(8, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(1) rkeys(1, 2) timer_key_time(1:5, 2:6)
+ // r_join_cnt_keys(1, 2)
+ assertEquals(7, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "ccc")))
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "Hello")))
- // 2 right timer(5,6), 2 right keys(1,2), 2 join cnt, 2 left timer(7,8), 2 left key(1,2)
- assertEquals(10, testHarness.numKeyedStateEntries())
- assertEquals(4, testHarness.numProcessingTimeTimers())
+ // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
+ // r_join_cnt_keys(1, 2)
+ assertEquals(8, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
@@ -1435,22 +1454,27 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "Hi2")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- // expired right stream record with key value of 1
+ // lkeys(2) rkeys(1, 2) timer_key_time(1:8, 2:6)
+ // r_join_cnt_keys(1, 2)
+ assertEquals(7, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(5)
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "Hi3")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi3")))
- // 1 right timer(6), 1 right keys(2), 1 join cnt, 2 left timer(7,8), 1 left key(2)
- assertEquals(6, testHarness.numKeyedStateEntries())
- assertEquals(3, testHarness.numProcessingTimeTimers())
+ // lkeys(2) rkeys(1, 2) timer_key_time(1:8, 2:6)
+ // r_join_cnt_keys(1, 2)
+ assertEquals(7, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all right stream record
+ // expired all stream records with key value of 2
+ // lkeys() rkeys(1) timer_key_time(1:8)
+ // r_join_cnt_keys(1)
testHarness.setProcessingTime(6)
assertEquals(3, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(1, testHarness.numProcessingTimeTimers())
- // expired left stream record with key value of 2
+ // expired all data
testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1491,6 +1515,12 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "Hi1", 1: JInt, "bbb")))
expectedOutput.add(new StreamRecord(
CRow(null: JInt, null, 1: JInt, "bbb")))
+ // processing time of 5
+ // timer of 8, we use only one timer state now
+ expectedOutput.add(new StreamRecord(
+ CRow(false, null: JInt, null, 1: JInt, "bbb")))
+ expectedOutput.add(new StreamRecord(
+ CRow(1: JInt, "Hi3", 1: JInt, "bbb")))
verify(expectedOutput, result)
testHarness.close()
@@ -1524,8 +1554,8 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "bbb")))
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "ccc")))
+ // lkeys(1) rkeys() timer_key_time(1:5)
assertEquals(1, testHarness.numProcessingTimeTimers())
- // 1 left timer(5), 1 left key(1)
assertEquals(2, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
@@ -1534,8 +1564,7 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "ccc")))
assertEquals(2, testHarness.numProcessingTimeTimers())
- // 1 left timer(5), 1 left key(1)
- // 1 right timer(6), 1 right key(1)
+ // lkeys(1) rkeys(2) timer_key_time(1:5, 2:6)
assertEquals(4, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1543,18 +1572,16 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(2: JInt, "aaa")))
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "ddd")))
- assertEquals(3, testHarness.numProcessingTimeTimers())
- // 2 left timer(5,7), 2 left key(1,2)
- // 1 right timer(6), 1 right key(1)
- assertEquals(6, testHarness.numKeyedStateEntries())
+ // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "ddd")))
- assertEquals(4, testHarness.numProcessingTimeTimers())
- // 2 left timer(5,7), 2 left key(1,2)
- // 2 right timer(6,7), 2 right key(1,2)
- assertEquals(8, testHarness.numKeyedStateEntries())
+ // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
@@ -1565,28 +1592,26 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "ddd")))
- assertEquals(4, testHarness.numProcessingTimeTimers())
- // 2 left timer(5,7), 1 left key(1)
- // 2 right timer(6,7), 1 right key(2)
- assertEquals(6, testHarness.numKeyedStateEntries())
+ // lkeys(1) rkeys(2) timer_key_time(1:8, 2:6)
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(4, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(5)
- assertEquals(3, testHarness.numProcessingTimeTimers())
- // 1 left timer(7)
- // 2 right timer(6,7), 1 right key(2)
+ assertEquals(2, testHarness.numProcessingTimeTimers())
assertEquals(4, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(6)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- // 1 left timer(7)
- // 2 right timer(7)
+ // lkeys(1) rkeys() timer_key_time(1:8)
+ assertEquals(1, testHarness.numProcessingTimeTimers())
assertEquals(2, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(7)
- assertEquals(0, testHarness.numProcessingTimeTimers())
- assertEquals(0, testHarness.numKeyedStateEntries())
+ assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(2, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(8)
+ assertEquals(0, testHarness.numProcessingTimeTimers())
+ assertEquals(0, testHarness.numKeyedStateEntries())
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "bbb")))
testHarness.processElement2(new StreamRecord(
@@ -1693,8 +1718,9 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "bbb")))
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "ccc")))
+ // lkeys(1) rkeys() timer_key_time(1:5)
+ // l_join_cnt_keys(1) r_join_cnt_keys()
assertEquals(1, testHarness.numProcessingTimeTimers())
- // 1 left timer(5), 1 left key(1), 1 left joincnt key(1)
assertEquals(3, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
@@ -1702,9 +1728,9 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(2: JInt, "bbb")))
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "ccc")))
+ // lkeys(1) rkeys(2) timer_key_time(1:5, 2:6)
+ // l_join_cnt_keys(1) r_join_cnt_keys(2)
assertEquals(2, testHarness.numProcessingTimeTimers())
- // 1 left timer(5), 1 left key(1), 1 left joincnt key(1)
- // 1 right timer(6), 1 right key(1), 1 right joincnt key(1)
assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1712,46 +1738,46 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(2: JInt, "aaa")))
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "ddd")))
- assertEquals(3, testHarness.numProcessingTimeTimers())
- // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
- // 1 right timer(6), 1 right key(1), 1 right joincnt key(1)
- assertEquals(9, testHarness.numKeyedStateEntries())
+ // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
+ // l_join_cnt_keys(1, 2) r_join_cnt_keys(2)
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(8, testHarness.numKeyedStateEntries())
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "ddd")))
- assertEquals(4, testHarness.numProcessingTimeTimers())
- // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
- // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
- assertEquals(12, testHarness.numKeyedStateEntries())
+ // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
+ // l_join_cnt_keys(1, 2) r_join_cnt_keys(1, 2)
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(10, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
CRow(false, 2: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "ddd")))
- assertEquals(4, testHarness.numProcessingTimeTimers())
- // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
- // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
- assertEquals(12, testHarness.numKeyedStateEntries())
+ // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:8, 2:6)
+ // l_join_cnt_keys(1, 2) r_join_cnt_keys(1, 2)
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(10, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(5)
- assertEquals(3, testHarness.numProcessingTimeTimers())
- // 1 left timer(7), 1 left key(2), 1 left joincnt key(2)
- // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
- assertEquals(9, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(10, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(6)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- // 1 left timer(7), 1 left key(2), 1 left joincnt key(2)
- // 1 right timer(7), 1 right key(2), 1 right joincnt key(2)
- assertEquals(6, testHarness.numKeyedStateEntries())
+ // lkeys(1) rkeys(1) timer_key_time(1:8)
+ // l_join_cnt_keys(1) r_join_cnt_keys(1)
+ assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(7)
- assertEquals(0, testHarness.numProcessingTimeTimers())
- assertEquals(0, testHarness.numKeyedStateEntries())
+ assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(8)
+ assertEquals(0, testHarness.numProcessingTimeTimers())
+ assertEquals(0, testHarness.numKeyedStateEntries())
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "bbb")))
testHarness.processElement2(new StreamRecord(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
index 7f9c0ef2553..25395be7528 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
@@ -80,8 +80,8 @@ class StateCleaningCountTriggerHarnessTest {
TriggerResult.CONTINUE,
testHarness.processElement(new StreamRecord(1), GlobalWindow.get))
- // have two timers 6001 and 7002
- assertEquals(2, testHarness.numProcessingTimeTimers)
+ // have one timer 7002
+ assertEquals(1, testHarness.numProcessingTimeTimers)
assertEquals(0, testHarness.numEventTimeTimers)
assertEquals(2, testHarness.numStateEntries)
assertEquals(2, testHarness.numStateEntries(GlobalWindow.get))
@@ -116,9 +116,6 @@ class StateCleaningCountTriggerHarnessTest {
// try to trigger onProcessingTime method via 7002, and all states are cleared
val timesIt = testHarness.advanceProcessingTime(7002).iterator()
- assertEquals(
- TriggerResult.CONTINUE,
- timesIt.next().f1)
assertEquals(
TriggerResult.FIRE_AND_PURGE,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
index fe90a5f3300..1c02889513d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
@@ -110,7 +110,7 @@ private class MockedKeyedProcessFunction(queryConfig: StreamQueryConfig)
out: Collector[String]): Unit = {
val curTime = ctx.timerService().currentProcessingTime()
- registerProcessingCleanupTimer(ctx, curTime)
+ processCleanupTimer(ctx, curTime)
state.update(value._2)
}
@@ -119,8 +119,12 @@ private class MockedKeyedProcessFunction(queryConfig: StreamQueryConfig)
ctx: KeyedProcessFunction[String, (String, String), String]#OnTimerContext,
out: Collector[String]): Unit = {
- if (needToCleanupState(timestamp)) {
- cleanupState(state)
+ if (stateCleaningEnabled) {
+ val cleanupTime = cleanupTimeState.value()
+ if (null != cleanupTime && timestamp == cleanupTime) {
+ // clean up
+ cleanupState(state)
+ }
}
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
index 519b03f59b7..6c0ca1a5012 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
@@ -110,7 +110,7 @@ private class MockedProcessFunction(queryConfig: StreamQueryConfig)
out: Collector[String]): Unit = {
val curTime = ctx.timerService().currentProcessingTime()
- registerProcessingCleanupTimer(ctx, curTime)
+ processCleanupTimer(ctx, curTime)
state.update(value._2)
}
@@ -119,7 +119,7 @@ private class MockedProcessFunction(queryConfig: StreamQueryConfig)
ctx: ProcessFunction[(String, String), String]#OnTimerContext,
out: Collector[String]): Unit = {
- if (needToCleanupState(timestamp)) {
+ if (stateCleaningEnabled) {
cleanupState(state)
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala
index 1706fc8a112..21680e86aee 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala
@@ -549,7 +549,7 @@ class TimeAttributesITCase extends AbstractTestBase {
.fromElements(p1, p2)
.assignTimestampsAndWatermarks(new TimestampWithEqualWatermarkPojo)
// use aliases, swap all attributes, and skip b2
- val table = stream.toTable(tEnv, ('b as 'b).rowtime, 'c as 'c, 'a as 'a)
+ val table = stream.toTable(tEnv, 'b.rowtime as 'b, 'c as 'c, 'a as 'a)
// no aliases, no swapping
val table2 = stream.toTable(tEnv, 'a, 'b.rowtime, 'c)
// use proctime, no skipping
@@ -560,7 +560,7 @@ class TimeAttributesITCase extends AbstractTestBase {
// use aliases, swap all attributes, and skip b2
val table4 = stream.toTable(
tEnv,
- ExpressionParser.parseExpressionList("(b as b).rowtime, c as c, a as a"): _*)
+ ExpressionParser.parseExpressionList("b.rowtime as b, c as c, a as a"): _*)
// no aliases, no swapping
val table5 = stream.toTable(
tEnv,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala
index 8f5a8f335f4..56087bec6fa 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala
@@ -22,12 +22,14 @@ import java.sql.Timestamp
import java.util.TimeZone
import org.apache.flink.api.common.time.Time
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
-import org.apache.flink.table.api.{TableConfig, TableEnvironment}
import org.apache.flink.table.api.scala._
-import org.apache.flink.table.functions.{FunctionContext, ScalarFunction}
+import org.apache.flink.table.api.{TableConfig, TableEnvironment, Types}
+import org.apache.flink.table.functions.{AggregateFunction, FunctionContext, ScalarFunction}
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg
import org.apache.flink.table.runtime.utils.TimeTestUtil.EventTimeSourceFunction
import org.apache.flink.table.runtime.utils.{StreamITCase, StreamingWithStateTestBase, UserDefinedFunctionTestUtils}
import org.apache.flink.types.Row
@@ -39,7 +41,7 @@ import scala.collection.mutable
class MatchRecognizeITCase extends StreamingWithStateTestBase {
@Test
- def testSimpleCEP(): Unit = {
+ def testSimplePattern(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
val tEnv = TableEnvironment.getTableEnvironment(env)
@@ -86,7 +88,7 @@ class MatchRecognizeITCase extends StreamingWithStateTestBase {
}
@Test
- def testSimpleCEPWithNulls(): Unit = {
+ def testSimplePatternWithNulls(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
val tEnv = TableEnvironment.getTableEnvironment(env)
@@ -464,6 +466,132 @@ class MatchRecognizeITCase extends StreamingWithStateTestBase {
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
+ /**
+ * This query checks:
+ *
+ * 1. count(D.price) produces 0, because no rows matched to D
+ * 2. sum(D.price) produces null, because no rows matched to D
+ * 3. aggregates that take multiple parameters work
+ * 4. aggregates with expressions work
+ */
+ @Test
+ def testAggregates(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ tEnv.getConfig.setMaxGeneratedCodeLength(1)
+ StreamITCase.clear
+
+ val data = new mutable.MutableList[(Int, String, Long, Double, Int)]
+ data.+=((1, "a", 1, 0.8, 1))
+ data.+=((2, "z", 2, 0.8, 3))
+ data.+=((3, "b", 1, 0.8, 2))
+ data.+=((4, "c", 1, 0.8, 5))
+ data.+=((5, "d", 4, 0.1, 5))
+ data.+=((6, "a", 2, 1.5, 2))
+ data.+=((7, "b", 2, 0.8, 3))
+ data.+=((8, "c", 1, 0.8, 2))
+ data.+=((9, "h", 4, 0.8, 3))
+ data.+=((10, "h", 4, 0.8, 3))
+ data.+=((11, "h", 2, 0.8, 3))
+ data.+=((12, "h", 2, 0.8, 3))
+
+ val t = env.fromCollection(data)
+ .toTable(tEnv, 'id, 'name, 'price, 'rate, 'weight, 'proctime.proctime)
+ tEnv.registerTable("MyTable", t)
+ tEnv.registerFunction("weightedAvg", new WeightedAvg)
+
+ val sqlQuery =
+ s"""
+ |SELECT *
+ |FROM MyTable
+ |MATCH_RECOGNIZE (
+ | ORDER BY proctime
+ | MEASURES
+ | FIRST(id) as startId,
+ | SUM(A.price) AS sumA,
+ | COUNT(D.price) AS countD,
+ | SUM(D.price) as sumD,
+ | weightedAvg(price, weight) as wAvg,
+ | AVG(B.price) AS avgB,
+ | SUM(B.price * B.rate) as sumExprB,
+ | LAST(id) as endId
+ | AFTER MATCH SKIP PAST LAST ROW
+ | PATTERN (A+ B+ C D? E )
+ | DEFINE
+ | A AS SUM(A.price) < 6,
+ | B AS SUM(B.price * B.rate) < SUM(A.price) AND
+ | SUM(B.price * B.rate) > 0.2 AND
+ | SUM(B.price) >= 1 AND
+ | AVG(B.price) >= 1 AND
+ | weightedAvg(price, weight) > 1
+ |) AS T
+ |""".stripMargin
+
+ val result = tEnv.sqlQuery(sqlQuery).toAppendStream[Row]
+ result.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList("1,5,0,null,2,3,3.4,8", "9,4,0,null,3,4,3.2,12")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testAggregatesWithNullInputs(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ tEnv.getConfig.setMaxGeneratedCodeLength(1)
+ StreamITCase.clear
+
+ val data = new mutable.MutableList[Row]
+ data.+=(Row.of(Int.box(1), "a", Int.box(10)))
+ data.+=(Row.of(Int.box(2), "z", Int.box(10)))
+ data.+=(Row.of(Int.box(3), "b", null))
+ data.+=(Row.of(Int.box(4), "c", null))
+ data.+=(Row.of(Int.box(5), "d", Int.box(3)))
+ data.+=(Row.of(Int.box(6), "c", Int.box(3)))
+ data.+=(Row.of(Int.box(7), "c", Int.box(3)))
+ data.+=(Row.of(Int.box(8), "c", Int.box(3)))
+ data.+=(Row.of(Int.box(9), "c", Int.box(2)))
+
+ val t = env.fromCollection(data)(Types.ROW(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO))
+ .toTable(tEnv, 'id, 'name, 'price, 'proctime.proctime)
+ tEnv.registerTable("MyTable", t)
+ tEnv.registerFunction("weightedAvg", new WeightedAvg)
+
+ val sqlQuery =
+ s"""
+ |SELECT *
+ |FROM MyTable
+ |MATCH_RECOGNIZE (
+ | ORDER BY proctime
+ | MEASURES
+ | SUM(A.price) as sumA,
+ | COUNT(A.id) as countAId,
+ | COUNT(A.price) as countAPrice,
+ | COUNT(*) as countAll,
+ | COUNT(price) as countAllPrice,
+ | LAST(id) as endId
+ | AFTER MATCH SKIP PAST LAST ROW
+ | PATTERN (A+ C)
+ | DEFINE
+ | A AS SUM(A.price) < 30,
+ | C AS C.name = 'c'
+ |) AS T
+ |""".stripMargin
+
+ val result = tEnv.sqlQuery(sqlQuery).toAppendStream[Row]
+ result.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList("29,7,5,8,6,8")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
@Test
def testAccessingProctime(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
@@ -567,9 +695,11 @@ class MatchRecognizeITCase extends StreamingWithStateTestBase {
.toTable(tEnv, 'id, 'name, 'price, 'proctime.proctime)
tEnv.registerTable("MyTable", t)
tEnv.registerFunction("prefix", new PrefixingScalarFunc)
+ tEnv.registerFunction("countFrom", new RichAggFunc)
val prefix = "PREF"
+ val startFrom = 4
UserDefinedFunctionTestUtils
- .setJobParameters(env, Map("prefix" -> prefix))
+ .setJobParameters(env, Map("prefix" -> prefix, "start" -> startFrom.toString))
val sqlQuery =
s"""
@@ -580,11 +710,12 @@ class MatchRecognizeITCase extends StreamingWithStateTestBase {
| MEASURES
| FIRST(id) as firstId,
| prefix(A.name) as prefixedNameA,
+ | countFrom(A.price) as countFromA,
| LAST(id) as lastId
| AFTER MATCH SKIP PAST LAST ROW
| PATTERN (A+ C)
| DEFINE
- | A AS prefix(A.name) = '$prefix:a'
+ | A AS prefix(A.name) = '$prefix:a' AND countFrom(A.price) <= ${startFrom + 4}
|) AS T
|""".stripMargin
@@ -592,7 +723,7 @@ class MatchRecognizeITCase extends StreamingWithStateTestBase {
result.addSink(new StreamITCase.StringSink[Row])
env.execute()
- val expected = mutable.MutableList("1,PREF:a,6", "7,PREF:a,9")
+ val expected = mutable.MutableList("1,PREF:a,8,5", "7,PREF:a,6,9")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
}
@@ -615,3 +746,26 @@ private class PrefixingScalarFunc extends ScalarFunction {
s"$prefix:$value"
}
}
+
+private case class CountAcc(var count: Long)
+
+private class RichAggFunc extends AggregateFunction[Long, CountAcc] {
+
+ private var start : Long = 0
+
+ override def open(context: FunctionContext): Unit = {
+ start = context.getJobParameter("start", "0").toLong
+ }
+
+ override def close(): Unit = {
+ start = 0
+ }
+
+ def accumulate(countAcc: CountAcc, value: Long): Unit = {
+ countAcc.count += value
+ }
+
+ override def createAccumulator(): CountAcc = CountAcc(start)
+
+ override def getValue(accumulator: CountAcc): Long = accumulator.count
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala
index 46dde8e0225..ddc2a687541 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala
@@ -78,6 +78,7 @@ class SqlITCase extends StreamingWithStateTestBase {
val sqlQuery = "SELECT c, " +
" COUNT(DISTINCT b)," +
+ " SUM(DISTINCT b)," +
" SESSION_END(rowtime, INTERVAL '0.005' SECOND) " +
"FROM MyTable " +
"GROUP BY SESSION(rowtime, INTERVAL '0.005' SECOND), c "
@@ -87,9 +88,9 @@ class SqlITCase extends StreamingWithStateTestBase {
env.execute()
val expected = Seq(
- "Hello World,1,1970-01-01 00:00:00.014", // window starts at [9L] till {14L}
- "Hello,1,1970-01-01 00:00:00.021", // window starts at [16L] till {21L}, not merged
- "Hello,3,1970-01-01 00:00:00.015" // window starts at [1L,2L],
+ "Hello World,1,9,1970-01-01 00:00:00.014", // window starts at [9L] till {14L}
+ "Hello,1,16,1970-01-01 00:00:00.021", // window starts at [16L] till {21L}, not merged
+ "Hello,3,6,1970-01-01 00:00:00.015" // window starts at [1L,2L],
// merged with [8L,10L], by [4L], till {15L}
)
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala
index 7483b04d9ca..055501a6a01 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/types/CRowSerializerTest.scala
@@ -18,8 +18,20 @@
package org.apache.flink.table.runtime.types
-import org.apache.flink.util.TestLogger
-import org.junit.Test
+import org.apache.flink.api.common.state.{ListState, ListStateDescriptor}
+import org.apache.flink.api.common.typeinfo.Types
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.runtime.state.heap.HeapKeyedStateBackend
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction
+import org.apache.flink.streaming.api.operators.{AbstractStreamOperator, KeyedProcessOperator}
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
+import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness
+import org.apache.flink.types.Row
+import org.apache.flink.util.{Collector, InstantiationUtil, TestLogger}
+
+import org.junit.{Assert, Test}
class CRowSerializerTest extends TestLogger {
@@ -29,6 +41,70 @@ class CRowSerializerTest extends TestLogger {
@Test
def testDefaultConstructor(): Unit = {
new CRowSerializer.CRowSerializerConfigSnapshot()
+
+ InstantiationUtil.instantiate(classOf[CRowSerializer.CRowSerializerConfigSnapshot])
+ }
+
+ @Test
+ def testStateRestore(): Unit = {
+
+ class IKeyedProcessFunction extends KeyedProcessFunction[Integer, Integer, Integer] {
+ var state: ListState[CRow] = _
+ override def open(parameters: Configuration): Unit = {
+ val stateDesc = new ListStateDescriptor[CRow]("CRow",
+ new CRowTypeInfo(new RowTypeInfo(Types.INT)))
+ state = getRuntimeContext.getListState(stateDesc)
+ }
+ override def processElement(value: Integer,
+ ctx: KeyedProcessFunction[Integer, Integer, Integer]#Context,
+ out: Collector[Integer]): Unit = {
+ state.add(new CRow(Row.of(value), true))
+ }
+ }
+
+ val operator = new KeyedProcessOperator[Integer, Integer, Integer](new IKeyedProcessFunction)
+
+ var testHarness = new KeyedOneInputStreamOperatorTestHarness[Integer, Integer, Integer](
+ operator,
+ new KeySelector[Integer, Integer] {
+ override def getKey(value: Integer): Integer= -1
+ },
+ Types.INT, 1, 1, 0)
+ testHarness.setup()
+ testHarness.open()
+
+ testHarness.processElement(new StreamRecord[Integer](1, 1L))
+ testHarness.processElement(new StreamRecord[Integer](2, 1L))
+ testHarness.processElement(new StreamRecord[Integer](3, 1L))
+
+ Assert.assertEquals(1, numKeyedStateEntries(operator))
+
+ val snapshot = testHarness.snapshot(0L, 0L)
+ testHarness.close()
+
+ testHarness = new KeyedOneInputStreamOperatorTestHarness[Integer, Integer, Integer](
+ operator,
+ new KeySelector[Integer, Integer] {
+ override def getKey(value: Integer): Integer= -1
+ },
+ Types.INT, 1, 1, 0)
+ testHarness.setup()
+
+ testHarness.initializeState(snapshot)
+
+ testHarness.open()
+
+ Assert.assertEquals(1, numKeyedStateEntries(operator))
+
+ testHarness.close()
+ }
+
+ def numKeyedStateEntries(operator: AbstractStreamOperator[_]): Int = {
+ val keyedStateBackend = operator.getKeyedStateBackend
+ keyedStateBackend match {
+ case hksb: HeapKeyedStateBackend[_] => hksb.numKeyValueStateEntries
+ case _ => throw new UnsupportedOperationException
+ }
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java
index dad45eb669c..9e15aebd048 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCounts.java
@@ -18,6 +18,9 @@
package org.apache.flink.runtime.checkpoint;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.Serializable;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -26,6 +29,7 @@
* Counts of checkpoints.
*/
public class CheckpointStatsCounts implements Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(CheckpointStatsCounts.class);
private static final long serialVersionUID = -5229425063269482528L;
@@ -147,9 +151,8 @@ void incrementInProgressCheckpoints() {
* {@link #incrementInProgressCheckpoints()}.
*/
void incrementCompletedCheckpoints() {
- if (--numInProgressCheckpoints < 0) {
- throw new IllegalStateException("Incremented the completed number of checkpoints " +
- "without incrementing the in progress checkpoints before.");
+ if (canDecrementOfInProgressCheckpointsNumber()) {
+ numInProgressCheckpoints--;
}
numCompletedCheckpoints++;
}
@@ -161,9 +164,8 @@ void incrementCompletedCheckpoints() {
* {@link #incrementInProgressCheckpoints()}.
*/
void incrementFailedCheckpoints() {
- if (--numInProgressCheckpoints < 0) {
- throw new IllegalStateException("Incremented the completed number of checkpoints " +
- "without incrementing the in progress checkpoints before.");
+ if (canDecrementOfInProgressCheckpointsNumber()) {
+ numInProgressCheckpoints--;
}
numFailedCheckpoints++;
}
@@ -181,4 +183,14 @@ CheckpointStatsCounts createSnapshot() {
numCompletedCheckpoints,
numFailedCheckpoints);
}
+
+ private boolean canDecrementOfInProgressCheckpointsNumber() {
+ boolean decrementLeadsToNegativeNumber = numInProgressCheckpoints - 1 < 0;
+ if (decrementLeadsToNegativeNumber) {
+ String errorMessage = "Incremented the completed number of checkpoints " +
+ "without incrementing the in progress checkpoints before.";
+ LOG.warn(errorMessage);
+ }
+ return !decrementLeadsToNegativeNumber;
+ }
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ApplicationStatus.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ApplicationStatus.java
index 963e9ab1231..0cc43c0cc02 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ApplicationStatus.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ApplicationStatus.java
@@ -32,7 +32,7 @@
FAILED(1443),
/** Application was canceled or killed on request */
- CANCELED(1444),
+ CANCELED(0),
/** Application status is not known */
UNKNOWN(1445);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 40a675aca31..2fdf79c7cb2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -683,8 +683,12 @@ public void acknowledgeCheckpoint(
}
});
} else {
- log.error("Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator",
- jobGraph.getJobID());
+ String errorMessage = "Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator";
+ if (executionGraph.getState() == JobStatus.RUNNING) {
+ log.error(errorMessage, jobGraph.getJobID());
+ } else {
+ log.debug(errorMessage, jobGraph.getJobID());
+ }
}
}
@@ -702,8 +706,12 @@ public void declineCheckpoint(DeclineCheckpoint decline) {
}
});
} else {
- log.error("Received DeclineCheckpoint message for job {} with no CheckpointCoordinator",
- jobGraph.getJobID());
+ String errorMessage = "Received DeclineCheckpoint message for job {} with no CheckpointCoordinator";
+ if (executionGraph.getState() == JobStatus.RUNNING) {
+ log.error(errorMessage, jobGraph.getJobID());
+ } else {
+ log.debug(errorMessage, jobGraph.getJobID());
+ }
}
}
@@ -956,19 +964,18 @@ public void heartbeatFromResourceManager(final ResourceID resourceID) {
return checkpointCoordinator
.triggerSavepoint(System.currentTimeMillis(), targetDirectory)
.thenApply(CompletedCheckpoint::getExternalPointer)
- .thenApplyAsync(path -> {
- if (cancelJob) {
+ .handleAsync((path, throwable) -> {
+ if (throwable != null) {
+ if (cancelJob) {
+ startCheckpointScheduler(checkpointCoordinator);
+ }
+ throw new CompletionException(throwable);
+ } else if (cancelJob) {
log.info("Savepoint stored in {}. Now cancelling {}.", path, jobGraph.getJobID());
cancel(timeout);
}
return path;
- }, getMainThreadExecutor())
- .exceptionally(throwable -> {
- if (cancelJob) {
- startCheckpointScheduler(checkpointCoordinator);
- }
- throw new CompletionException(throwable);
- });
+ }, getMainThreadExecutor());
}
private void startCheckpointScheduler(final CheckpointCoordinator checkpointCoordinator) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java
index 7188a597c86..244a1ede5ca 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryConfiguration.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.metrics;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.DelegatingConfiguration;
@@ -26,6 +27,8 @@
import org.apache.flink.runtime.metrics.scope.ScopeFormats;
import org.apache.flink.util.Preconditions;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,14 +69,18 @@
// contains for every configured reporter its name and the configuration object
private final List<Tuple2<String, Configuration>> reporterConfigurations;
+ private final long queryServiceMessageSizeLimit;
+
public MetricRegistryConfiguration(
ScopeFormats scopeFormats,
char delimiter,
- List<Tuple2<String, Configuration>> reporterConfigurations) {
+ List<Tuple2<String, Configuration>> reporterConfigurations,
+ long queryServiceMessageSizeLimit) {
this.scopeFormats = Preconditions.checkNotNull(scopeFormats);
this.delimiter = delimiter;
this.reporterConfigurations = Preconditions.checkNotNull(reporterConfigurations);
+ this.queryServiceMessageSizeLimit = queryServiceMessageSizeLimit;
}
// ------------------------------------------------------------------------
@@ -92,6 +99,10 @@ public char getDelimiter() {
return reporterConfigurations;
}
+ public long getQueryServiceMessageSizeLimit() {
+ return queryServiceMessageSizeLimit;
+ }
+
// ------------------------------------------------------------------------
// Static factory methods
// ------------------------------------------------------------------------
@@ -160,7 +171,15 @@ public static MetricRegistryConfiguration fromConfiguration(Configuration config
}
}
- return new MetricRegistryConfiguration(scopeFormats, delim, reporterConfigurations);
+ final String maxFrameSizeStr = configuration.getString(AkkaOptions.FRAMESIZE);
+ final String akkaConfigStr = String.format("akka {remote {netty.tcp {maximum-frame-size = %s}}}", maxFrameSizeStr);
+ final Config akkaConfig = ConfigFactory.parseString(akkaConfigStr);
+ final long maximumFrameSize = akkaConfig.getBytes("akka.remote.netty.tcp.maximum-frame-size");
+
+ // padding to account for serialization overhead
+ final long messageSizeLimitPadding = 256;
+
+ return new MetricRegistryConfiguration(scopeFormats, delim, reporterConfigurations, maximumFrameSize - messageSizeLimitPadding);
}
public static MetricRegistryConfiguration defaultMetricRegistryConfiguration() {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java
index 6b3770907a9..31775e24276 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/MetricRegistryImpl.java
@@ -77,6 +77,8 @@
private final CompletableFuture<Void> terminationFuture;
+ private final long maximumFramesize;
+
@Nullable
private ActorRef queryService;
@@ -91,6 +93,7 @@
* Creates a new MetricRegistry and starts the configured reporter.
*/
public MetricRegistryImpl(MetricRegistryConfiguration config) {
+ this.maximumFramesize = config.getQueryServiceMessageSizeLimit();
this.scopeFormats = config.getScopeFormats();
this.globalDelimiter = config.getDelimiter();
this.delimiters = new ArrayList<>(10);
@@ -184,7 +187,7 @@ public void startQueryService(ActorSystem actorSystem, ResourceID resourceID) {
Preconditions.checkState(!isShutdown(), "The metric registry has already been shut down.");
try {
- queryService = MetricQueryService.startMetricQueryService(actorSystem, resourceID);
+ queryService = MetricQueryService.startMetricQueryService(actorSystem, resourceID, maximumFramesize);
metricQueryServicePath = AkkaUtils.getAkkaURL(actorSystem, queryService);
} catch (Exception e) {
LOG.warn("Could not start MetricDumpActor. No metrics will be submitted to the WebInterface.", e);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java
index 16a885dd345..5456b56cdbd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerialization.java
@@ -73,19 +73,38 @@ private MetricDumpSerialization() {
private static final long serialVersionUID = 6928770855951536906L;
- public final byte[] serializedMetrics;
+ public final byte[] serializedCounters;
+ public final byte[] serializedGauges;
+ public final byte[] serializedMeters;
+ public final byte[] serializedHistograms;
+
public final int numCounters;
public final int numGauges;
public final int numMeters;
public final int numHistograms;
- public MetricSerializationResult(byte[] serializedMetrics, int numCounters, int numGauges, int numMeters, int numHistograms) {
- Preconditions.checkNotNull(serializedMetrics);
+ public MetricSerializationResult(
+ byte[] serializedCounters,
+ byte[] serializedGauges,
+ byte[] serializedMeters,
+ byte[] serializedHistograms,
+ int numCounters,
+ int numGauges,
+ int numMeters,
+ int numHistograms) {
+
+ Preconditions.checkNotNull(serializedCounters);
+ Preconditions.checkNotNull(serializedGauges);
+ Preconditions.checkNotNull(serializedMeters);
+ Preconditions.checkNotNull(serializedHistograms);
Preconditions.checkArgument(numCounters >= 0);
Preconditions.checkArgument(numGauges >= 0);
Preconditions.checkArgument(numMeters >= 0);
Preconditions.checkArgument(numHistograms >= 0);
- this.serializedMetrics = serializedMetrics;
+ this.serializedCounters = serializedCounters;
+ this.serializedGauges = serializedGauges;
+ this.serializedMeters = serializedMeters;
+ this.serializedHistograms = serializedHistograms;
this.numCounters = numCounters;
this.numGauges = numGauges;
this.numMeters = numMeters;
@@ -102,7 +121,10 @@ public MetricSerializationResult(byte[] serializedMetrics, int numCounters, int
*/
public static class MetricDumpSerializer {
- private DataOutputSerializer buffer = new DataOutputSerializer(1024 * 32);
+ private DataOutputSerializer countersBuffer = new DataOutputSerializer(1024 * 8);
+ private DataOutputSerializer gaugesBuffer = new DataOutputSerializer(1024 * 8);
+ private DataOutputSerializer metersBuffer = new DataOutputSerializer(1024 * 8);
+ private DataOutputSerializer histogramsBuffer = new DataOutputSerializer(1024 * 8);
/**
* Serializes the given metrics and returns the resulting byte array.
@@ -126,53 +148,66 @@ public MetricSerializationResult serialize(
Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms,
Map<Meter, Tuple2<QueryScopeInfo, String>> meters) {
- buffer.clear();
-
+ countersBuffer.clear();
int numCounters = 0;
for (Map.Entry<Counter, Tuple2<QueryScopeInfo, String>> entry : counters.entrySet()) {
try {
- serializeCounter(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
+ serializeCounter(countersBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
numCounters++;
} catch (Exception e) {
LOG.debug("Failed to serialize counter.", e);
}
}
+ gaugesBuffer.clear();
int numGauges = 0;
for (Map.Entry<Gauge<?>, Tuple2<QueryScopeInfo, String>> entry : gauges.entrySet()) {
try {
- serializeGauge(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
+ serializeGauge(gaugesBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
numGauges++;
} catch (Exception e) {
LOG.debug("Failed to serialize gauge.", e);
}
}
+ histogramsBuffer.clear();
int numHistograms = 0;
for (Map.Entry<Histogram, Tuple2<QueryScopeInfo, String>> entry : histograms.entrySet()) {
try {
- serializeHistogram(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
+ serializeHistogram(histogramsBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
numHistograms++;
} catch (Exception e) {
LOG.debug("Failed to serialize histogram.", e);
}
}
+ metersBuffer.clear();
int numMeters = 0;
for (Map.Entry<Meter, Tuple2<QueryScopeInfo, String>> entry : meters.entrySet()) {
try {
- serializeMeter(buffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
+ serializeMeter(metersBuffer, entry.getValue().f0, entry.getValue().f1, entry.getKey());
numMeters++;
} catch (Exception e) {
LOG.debug("Failed to serialize meter.", e);
}
}
- return new MetricSerializationResult(buffer.getCopyOfBuffer(), numCounters, numGauges, numMeters, numHistograms);
+ return new MetricSerializationResult(
+ countersBuffer.getCopyOfBuffer(),
+ gaugesBuffer.getCopyOfBuffer(),
+ metersBuffer.getCopyOfBuffer(),
+ histogramsBuffer.getCopyOfBuffer(),
+ numCounters,
+ numGauges,
+ numMeters,
+ numHistograms);
}
public void close() {
- buffer = null;
+ countersBuffer = null;
+ gaugesBuffer = null;
+ metersBuffer = null;
+ histogramsBuffer = null;
}
}
@@ -280,13 +315,16 @@ private static void serializeMeter(DataOutput out, QueryScopeInfo info, String n
* @return A list containing the deserialized metrics.
*/
public List<MetricDump> deserialize(MetricDumpSerialization.MetricSerializationResult data) {
- DataInputView in = new DataInputDeserializer(data.serializedMetrics, 0, data.serializedMetrics.length);
+ DataInputView countersInputView = new DataInputDeserializer(data.serializedCounters, 0, data.serializedCounters.length);
+ DataInputView gaugesInputView = new DataInputDeserializer(data.serializedGauges, 0, data.serializedGauges.length);
+ DataInputView metersInputView = new DataInputDeserializer(data.serializedMeters, 0, data.serializedMeters.length);
+ DataInputView histogramsInputView = new DataInputDeserializer(data.serializedHistograms, 0, data.serializedHistograms.length);
- List<MetricDump> metrics = new ArrayList<>(data.numCounters + data.numGauges + data.numHistograms + data.numMeters);
+ List<MetricDump> metrics = new ArrayList<>(data.numCounters + data.numGauges + data.numMeters + data.numHistograms);
for (int x = 0; x < data.numCounters; x++) {
try {
- metrics.add(deserializeCounter(in));
+ metrics.add(deserializeCounter(countersInputView));
} catch (Exception e) {
LOG.debug("Failed to deserialize counter.", e);
}
@@ -294,25 +332,25 @@ private static void serializeMeter(DataOutput out, QueryScopeInfo info, String n
for (int x = 0; x < data.numGauges; x++) {
try {
- metrics.add(deserializeGauge(in));
+ metrics.add(deserializeGauge(gaugesInputView));
} catch (Exception e) {
LOG.debug("Failed to deserialize gauge.", e);
}
}
- for (int x = 0; x < data.numHistograms; x++) {
+ for (int x = 0; x < data.numMeters; x++) {
try {
- metrics.add(deserializeHistogram(in));
+ metrics.add(deserializeMeter(metersInputView));
} catch (Exception e) {
- LOG.debug("Failed to deserialize histogram.", e);
+ LOG.debug("Failed to deserialize meter.", e);
}
}
- for (int x = 0; x < data.numMeters; x++) {
+ for (int x = 0; x < data.numHistograms; x++) {
try {
- metrics.add(deserializeMeter(in));
+ metrics.add(deserializeHistogram(histogramsInputView));
} catch (Exception e) {
- LOG.debug("Failed to deserialize meter.", e);
+ LOG.debug("Failed to deserialize histogram.", e);
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java
index 8821e0d9f4a..fc69d17503d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/dump/MetricQueryService.java
@@ -55,6 +55,7 @@
private static final Logger LOG = LoggerFactory.getLogger(MetricQueryService.class);
public static final String METRIC_QUERY_SERVICE_NAME = "MetricQueryService";
+ private static final String SIZE_EXCEEDED_LOG_TEMPLATE = "{} will not be reported as the metric dump would exceed the maximum size of {} bytes.";
private static final CharacterFilter FILTER = new CharacterFilter() {
@Override
@@ -70,6 +71,12 @@ public String filterCharacters(String input) {
private final Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>();
private final Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>();
+ private final long messageSizeLimit;
+
+ public MetricQueryService(long messageSizeLimit) {
+ this.messageSizeLimit = messageSizeLimit;
+ }
+
@Override
public void postStop() {
serializer.close();
@@ -109,6 +116,9 @@ public void onReceive(Object message) {
}
} else if (message instanceof CreateDump) {
MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters);
+
+ dump = enforceSizeLimit(dump);
+
getSender().tell(dump, getSelf());
} else {
LOG.warn("MetricQueryServiceActor received an invalid message. " + message.toString());
@@ -119,6 +129,83 @@ public void onReceive(Object message) {
}
}
+ private MetricDumpSerialization.MetricSerializationResult enforceSizeLimit(
+ MetricDumpSerialization.MetricSerializationResult serializationResult) {
+
+ int currentLength = 0;
+ boolean hasExceededBefore = false;
+
+ byte[] serializedCounters = serializationResult.serializedCounters;
+ int numCounters = serializationResult.numCounters;
+ if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedCounters.length)) {
+ logDumpSizeWouldExceedLimit("Counters", hasExceededBefore);
+ hasExceededBefore = true;
+
+ serializedCounters = new byte[0];
+ numCounters = 0;
+ } else {
+ currentLength += serializedCounters.length;
+ }
+
+ byte[] serializedMeters = serializationResult.serializedMeters;
+ int numMeters = serializationResult.numMeters;
+ if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedMeters.length)) {
+ logDumpSizeWouldExceedLimit("Meters", hasExceededBefore);
+ hasExceededBefore = true;
+
+ serializedMeters = new byte[0];
+ numMeters = 0;
+ } else {
+ currentLength += serializedMeters.length;
+ }
+
+ byte[] serializedGauges = serializationResult.serializedGauges;
+ int numGauges = serializationResult.numGauges;
+ if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedGauges.length)) {
+ logDumpSizeWouldExceedLimit("Gauges", hasExceededBefore);
+ hasExceededBefore = true;
+
+ serializedGauges = new byte[0];
+ numGauges = 0;
+ } else {
+ currentLength += serializedGauges.length;
+ }
+
+ byte[] serializedHistograms = serializationResult.serializedHistograms;
+ int numHistograms = serializationResult.numHistograms;
+ if (exceedsMessageSizeLimit(currentLength + serializationResult.serializedHistograms.length)) {
+ logDumpSizeWouldExceedLimit("Histograms", hasExceededBefore);
+ hasExceededBefore = true;
+
+ serializedHistograms = new byte[0];
+ numHistograms = 0;
+ }
+
+ return new MetricDumpSerialization.MetricSerializationResult(
+ serializedCounters,
+ serializedGauges,
+ serializedMeters,
+ serializedHistograms,
+ numCounters,
+ numGauges,
+ numMeters,
+ numHistograms);
+ }
+
+ private boolean exceedsMessageSizeLimit(final int currentSize) {
+ return currentSize > messageSizeLimit;
+ }
+
+ private void logDumpSizeWouldExceedLimit(final String metricType, boolean hasExceededBefore) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(SIZE_EXCEEDED_LOG_TEMPLATE, metricType, messageSizeLimit);
+ } else {
+ if (!hasExceededBefore) {
+ LOG.info(SIZE_EXCEEDED_LOG_TEMPLATE, "Some metrics", messageSizeLimit);
+ }
+ }
+ }
+
/**
* Lightweight method to replace unsupported characters.
* If the string does not contain any unsupported characters, this method creates no
@@ -165,11 +252,16 @@ static String replaceInvalidChars(String str) {
* @param resourceID resource ID to disambiguate the actor name
* @return actor reference to the MetricQueryService
*/
- public static ActorRef startMetricQueryService(ActorSystem actorSystem, ResourceID resourceID) {
+ public static ActorRef startMetricQueryService(
+ ActorSystem actorSystem,
+ ResourceID resourceID,
+ long maximumFramesize) {
+
String actorName = resourceID == null
? METRIC_QUERY_SERVICE_NAME
: METRIC_QUERY_SERVICE_NAME + "_" + resourceID.getResourceIdString();
- return actorSystem.actorOf(Props.create(MetricQueryService.class), actorName);
+
+ return actorSystem.actorOf(Props.create(MetricQueryService.class, maximumFramesize), actorName);
}
/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java
index 7c46af04b55..b99de66122c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java
@@ -102,6 +102,10 @@ protected void channelRead0(final ChannelHandlerContext ctx, final HttpObject ms
checkState(currentUploadDir == null);
currentHttpPostRequestDecoder = new HttpPostRequestDecoder(DATA_FACTORY, httpRequest);
currentHttpRequest = ReferenceCountUtil.retain(httpRequest);
+
+ // make sure that we still have a upload dir in case that it got deleted in the meanwhile
+ RestServerEndpoint.createUploadDir(uploadDir, LOG);
+
currentUploadDir = Files.createDirectory(uploadDir.resolve(UUID.randomUUID().toString()));
} else {
ctx.fireChannelRead(ReferenceCountUtil.retain(msg));
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
index 4702919384f..952dffbc70f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java
@@ -26,7 +26,6 @@
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
-import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer;
import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.fs.FSDataInputStream;
@@ -105,23 +104,10 @@
*/
private final boolean asynchronousSnapshots;
- /**
- * Map of state names to their corresponding restored state meta info.
- *
- * <p>TODO this map can be removed when eager-state registration is in place.
- * TODO we currently need this cached to check state migration strategies when new serializers are registered.
- */
- private final Map<String, StateMetaInfoSnapshot> restoredOperatorStateMetaInfos;
-
- /**
- * Map of state names to their corresponding restored broadcast state meta info.
- */
- private final Map<String, StateMetaInfoSnapshot> restoredBroadcastStateMetaInfos;
-
/**
* Cache of already accessed states.
*
- * <p>In contrast to {@link #registeredOperatorStates} and {@link #restoredOperatorStateMetaInfos} which may be repopulated
+ * <p>In contrast to {@link #registeredOperatorStates} which may be repopulated
* with restored state, this map is always empty at the beginning.
*
* <p>TODO this map should be moved to a base class once we have proper hierarchy for the operator state backends.
@@ -148,8 +134,6 @@ public DefaultOperatorStateBackend(
this.asynchronousSnapshots = asynchronousSnapshots;
this.accessedStatesByName = new HashMap<>();
this.accessedBroadcastStatesByName = new HashMap<>();
- this.restoredOperatorStateMetaInfos = new HashMap<>();
- this.restoredBroadcastStateMetaInfos = new HashMap<>();
this.snapshotStrategy = new DefaultOperatorStateBackendSnapshotStrategy();
}
@@ -226,34 +210,22 @@ public void dispose() {
broadcastState.getStateMetaInfo().getAssignmentMode(),
OperatorStateHandle.Mode.BROADCAST);
- final StateMetaInfoSnapshot metaInfoSnapshot = restoredBroadcastStateMetaInfos.get(name);
+ RegisteredBroadcastStateBackendMetaInfo<K, V> restoredBroadcastStateMetaInfo = broadcastState.getStateMetaInfo();
// check whether new serializers are incompatible
- TypeSerializerSnapshot<K> keySerializerSnapshot = Preconditions.checkNotNull(
- (TypeSerializerSnapshot<K>) metaInfoSnapshot.getTypeSerializerConfigSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER));
-
TypeSerializerSchemaCompatibility<K> keyCompatibility =
- keySerializerSnapshot.resolveSchemaCompatibility(broadcastStateKeySerializer);
+ restoredBroadcastStateMetaInfo.updateKeySerializer(broadcastStateKeySerializer);
if (keyCompatibility.isIncompatible()) {
throw new StateMigrationException("The new key serializer for broadcast state must not be incompatible.");
}
- TypeSerializerSnapshot<V> valueSerializerSnapshot = Preconditions.checkNotNull(
- (TypeSerializerSnapshot<V>) metaInfoSnapshot.getTypeSerializerConfigSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER));
-
TypeSerializerSchemaCompatibility<V> valueCompatibility =
- valueSerializerSnapshot.resolveSchemaCompatibility(broadcastStateValueSerializer);
+ restoredBroadcastStateMetaInfo.updateValueSerializer(broadcastStateValueSerializer);
if (valueCompatibility.isIncompatible()) {
throw new StateMigrationException("The new value serializer for broadcast state must not be incompatible.");
}
- // new serializer is compatible; use it to replace the old serializer
- broadcastState.setStateMetaInfo(
- new RegisteredBroadcastStateBackendMetaInfo<>(
- name,
- OperatorStateHandle.Mode.BROADCAST,
- broadcastStateKeySerializer,
- broadcastStateValueSerializer));
+ broadcastState.setStateMetaInfo(restoredBroadcastStateMetaInfo);
}
accessedBroadcastStatesByName.put(name, broadcastState);
@@ -345,8 +317,6 @@ public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exc
" not be loaded. This is a temporary restriction that will be fixed in future versions.");
}
- restoredOperatorStateMetaInfos.put(restoredSnapshot.getName(), restoredSnapshot);
-
PartitionableListState<?> listState = registeredOperatorStates.get(restoredSnapshot.getName());
if (null == listState) {
@@ -381,8 +351,6 @@ public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exc
" not be loaded. This is a temporary restriction that will be fixed in future versions.");
}
- restoredBroadcastStateMetaInfos.put(restoredSnapshot.getName(), restoredSnapshot);
-
BackendWritableBroadcastState<? ,?> broadcastState = registeredBroadcastStates.get(restoredSnapshot.getName());
if (broadcastState == null) {
@@ -590,25 +558,19 @@ public void addAll(List<S> values) {
partitionableListState.getStateMetaInfo().getAssignmentMode(),
mode);
- StateMetaInfoSnapshot restoredSnapshot = restoredOperatorStateMetaInfos.get(name);
- RegisteredOperatorStateBackendMetaInfo<S> metaInfo =
- new RegisteredOperatorStateBackendMetaInfo<>(restoredSnapshot);
+ RegisteredOperatorStateBackendMetaInfo<S> restoredPartitionableListStateMetaInfo =
+ partitionableListState.getStateMetaInfo();
- // check compatibility to determine if state migration is required
+ // check compatibility to determine if new serializers are incompatible
TypeSerializer<S> newPartitionStateSerializer = partitionStateSerializer.duplicate();
- @SuppressWarnings("unchecked")
- TypeSerializerSnapshot<S> stateSerializerSnapshot = Preconditions.checkNotNull(
- (TypeSerializerSnapshot<S>) restoredSnapshot.getTypeSerializerConfigSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER));
-
TypeSerializerSchemaCompatibility<S> stateCompatibility =
- stateSerializerSnapshot.resolveSchemaCompatibility(newPartitionStateSerializer);
+ restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer);
if (stateCompatibility.isIncompatible()) {
throw new StateMigrationException("The new state serializer for operator state must not be incompatible.");
}
- partitionableListState.setStateMetaInfo(
- new RegisteredOperatorStateBackendMetaInfo<>(name, newPartitionStateSerializer, mode));
+ partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo);
}
accessedStatesByName.put(name, partitionableListState);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java
index 70a14142474..ecc13faa43d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java
@@ -19,11 +19,13 @@
package org.apache.flink.runtime.state;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
import org.apache.flink.util.Preconditions;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.util.Collections;
import java.util.HashMap;
@@ -38,11 +40,11 @@
/** The type serializer for the keys in the map state. */
@Nonnull
- private final TypeSerializer<K> keySerializer;
+ private final StateSerializerProvider<K> keySerializerProvider;
/** The type serializer for the values in the map state. */
@Nonnull
- private final TypeSerializer<V> valueSerializer;
+ private final StateSerializerProvider<V> valueSerializerProvider;
public RegisteredBroadcastStateBackendMetaInfo(
@Nonnull final String name,
@@ -50,19 +52,19 @@ public RegisteredBroadcastStateBackendMetaInfo(
@Nonnull final TypeSerializer<K> keySerializer,
@Nonnull final TypeSerializer<V> valueSerializer) {
- super(name);
- Preconditions.checkArgument(assignmentMode == OperatorStateHandle.Mode.BROADCAST);
- this.assignmentMode = assignmentMode;
- this.keySerializer = keySerializer;
- this.valueSerializer = valueSerializer;
+ this(
+ name,
+ assignmentMode,
+ StateSerializerProvider.fromNewState(keySerializer),
+ StateSerializerProvider.fromNewState(valueSerializer));
}
public RegisteredBroadcastStateBackendMetaInfo(@Nonnull RegisteredBroadcastStateBackendMetaInfo<K, V> copy) {
this(
Preconditions.checkNotNull(copy).name,
copy.assignmentMode,
- copy.keySerializer.duplicate(),
- copy.valueSerializer.duplicate());
+ copy.getKeySerializer().duplicate(),
+ copy.getValueSerializer().duplicate());
}
@SuppressWarnings("unchecked")
@@ -71,10 +73,13 @@ public RegisteredBroadcastStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sn
snapshot.getName(),
OperatorStateHandle.Mode.valueOf(
snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE)),
- (TypeSerializer<K>) Preconditions.checkNotNull(
- snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER)),
- (TypeSerializer<V>) Preconditions.checkNotNull(
- snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)));
+ StateSerializerProvider.fromRestoredState(
+ (TypeSerializerSnapshot<K>) Preconditions.checkNotNull(
+ snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER))),
+ StateSerializerProvider.fromRestoredState(
+ (TypeSerializerSnapshot<V>) Preconditions.checkNotNull(
+ snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))));
+
Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.BROADCAST == snapshot.getBackendStateType());
}
@@ -86,6 +91,19 @@ public RegisteredBroadcastStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sn
return new RegisteredBroadcastStateBackendMetaInfo<>(this);
}
+ private RegisteredBroadcastStateBackendMetaInfo(
+ @Nonnull final String name,
+ @Nonnull final OperatorStateHandle.Mode assignmentMode,
+ @Nonnull final StateSerializerProvider<K> keySerializerProvider,
+ @Nonnull final StateSerializerProvider<V> valueSerializerProvider) {
+
+ super(name);
+ Preconditions.checkArgument(assignmentMode == OperatorStateHandle.Mode.BROADCAST);
+ this.assignmentMode = assignmentMode;
+ this.keySerializerProvider = keySerializerProvider;
+ this.valueSerializerProvider = valueSerializerProvider;
+ }
+
@Nonnull
@Override
public StateMetaInfoSnapshot snapshot() {
@@ -94,12 +112,32 @@ public StateMetaInfoSnapshot snapshot() {
@Nonnull
public TypeSerializer<K> getKeySerializer() {
- return keySerializer;
+ return keySerializerProvider.currentSchemaSerializer();
+ }
+
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<K> updateKeySerializer(TypeSerializer<K> newKeySerializer) {
+ return keySerializerProvider.registerNewSerializerForRestoredState(newKeySerializer);
+ }
+
+ @Nullable
+ public TypeSerializer<K> getPreviousKeySerializer() {
+ return keySerializerProvider.previousSchemaSerializer();
}
@Nonnull
public TypeSerializer<V> getValueSerializer() {
- return valueSerializer;
+ return valueSerializerProvider.currentSchemaSerializer();
+ }
+
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<V> updateValueSerializer(TypeSerializer<V> newValueSerializer) {
+ return valueSerializerProvider.registerNewSerializerForRestoredState(newValueSerializer);
+ }
+
+ @Nullable
+ public TypeSerializer<V> getPreviousValueSerializer() {
+ return valueSerializerProvider.previousSchemaSerializer();
}
@Nonnull
@@ -122,16 +160,16 @@ public boolean equals(Object obj) {
return Objects.equals(name, other.getName())
&& Objects.equals(assignmentMode, other.getAssignmentMode())
- && Objects.equals(keySerializer, other.getKeySerializer())
- && Objects.equals(valueSerializer, other.getValueSerializer());
+ && Objects.equals(getKeySerializer(), other.getKeySerializer())
+ && Objects.equals(getValueSerializer(), other.getValueSerializer());
}
@Override
public int hashCode() {
int result = name.hashCode();
result = 31 * result + assignmentMode.hashCode();
- result = 31 * result + keySerializer.hashCode();
- result = 31 * result + valueSerializer.hashCode();
+ result = 31 * result + getKeySerializer().hashCode();
+ result = 31 * result + getValueSerializer().hashCode();
return result;
}
@@ -139,8 +177,8 @@ public int hashCode() {
public String toString() {
return "RegisteredBroadcastBackendStateMetaInfo{" +
"name='" + name + '\'' +
- ", keySerializer=" + keySerializer +
- ", valueSerializer=" + valueSerializer +
+ ", keySerializer=" + getKeySerializer() +
+ ", valueSerializer=" + getValueSerializer() +
", assignmentMode=" + assignmentMode +
'}';
}
@@ -154,8 +192,12 @@ private StateMetaInfoSnapshot computeSnapshot() {
Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap = new HashMap<>(2);
String keySerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER.toString();
String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString();
+
+ TypeSerializer<K> keySerializer = getKeySerializer();
serializerMap.put(keySerializerKey, keySerializer.duplicate());
serializerConfigSnapshotsMap.put(keySerializerKey, keySerializer.snapshotConfiguration());
+
+ TypeSerializer<V> valueSerializer = getValueSerializer();
serializerMap.put(valueSerializerKey, valueSerializer.duplicate());
serializerConfigSnapshotsMap.put(valueSerializerKey, valueSerializer.snapshotConfiguration());
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java
index d05f31a0c5c..b37c79de026 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java
@@ -20,6 +20,7 @@
import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
import org.apache.flink.util.Preconditions;
@@ -44,18 +45,24 @@
@Nonnull
private final StateDescriptor.Type stateType;
@Nonnull
- private final TypeSerializer<N> namespaceSerializer;
+ private final StateSerializerProvider<N> namespaceSerializerProvider;
@Nonnull
- private final TypeSerializer<S> stateSerializer;
+ private final StateSerializerProvider<S> stateSerializerProvider;
@Nullable
- private final StateSnapshotTransformer<S> snapshotTransformer;
+ private StateSnapshotTransformer<S> snapshotTransformer;
public RegisteredKeyValueStateBackendMetaInfo(
@Nonnull StateDescriptor.Type stateType,
@Nonnull String name,
@Nonnull TypeSerializer<N> namespaceSerializer,
@Nonnull TypeSerializer<S> stateSerializer) {
- this(stateType, name, namespaceSerializer, stateSerializer, null);
+
+ this(
+ stateType,
+ name,
+ StateSerializerProvider.fromNewState(namespaceSerializer),
+ StateSerializerProvider.fromNewState(stateSerializer),
+ null);
}
public RegisteredKeyValueStateBackendMetaInfo(
@@ -65,11 +72,12 @@ public RegisteredKeyValueStateBackendMetaInfo(
@Nonnull TypeSerializer<S> stateSerializer,
@Nullable StateSnapshotTransformer<S> snapshotTransformer) {
- super(name);
- this.stateType = stateType;
- this.namespaceSerializer = namespaceSerializer;
- this.stateSerializer = stateSerializer;
- this.snapshotTransformer = snapshotTransformer;
+ this(
+ stateType,
+ name,
+ StateSerializerProvider.fromNewState(namespaceSerializer),
+ StateSerializerProvider.fromNewState(stateSerializer),
+ snapshotTransformer);
}
@SuppressWarnings("unchecked")
@@ -77,13 +85,31 @@ public RegisteredKeyValueStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sna
this(
StateDescriptor.Type.valueOf(snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE)),
snapshot.getName(),
- (TypeSerializer<N>) Preconditions.checkNotNull(
- snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER)),
- (TypeSerializer<S>) Preconditions.checkNotNull(
- snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)), null);
+ StateSerializerProvider.fromRestoredState(
+ (TypeSerializerSnapshot<N>) Preconditions.checkNotNull(
+ snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER))),
+ StateSerializerProvider.fromRestoredState(
+ (TypeSerializerSnapshot<S>) Preconditions.checkNotNull(
+ snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))),
+ null);
+
Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.KEY_VALUE == snapshot.getBackendStateType());
}
+ private RegisteredKeyValueStateBackendMetaInfo(
+ @Nonnull StateDescriptor.Type stateType,
+ @Nonnull String name,
+ @Nonnull StateSerializerProvider<N> namespaceSerializerProvider,
+ @Nonnull StateSerializerProvider<S> stateSerializerProvider,
+ @Nullable StateSnapshotTransformer<S> snapshotTransformer) {
+
+ super(name);
+ this.stateType = stateType;
+ this.namespaceSerializerProvider = namespaceSerializerProvider;
+ this.stateSerializerProvider = stateSerializerProvider;
+ this.snapshotTransformer = snapshotTransformer;
+ }
+
@Nonnull
public StateDescriptor.Type getStateType() {
return stateType;
@@ -91,12 +117,32 @@ public RegisteredKeyValueStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sna
@Nonnull
public TypeSerializer<N> getNamespaceSerializer() {
- return namespaceSerializer;
+ return namespaceSerializerProvider.currentSchemaSerializer();
+ }
+
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<N> updateNamespaceSerializer(TypeSerializer<N> newNamespaceSerializer) {
+ return namespaceSerializerProvider.registerNewSerializerForRestoredState(newNamespaceSerializer);
+ }
+
+ @Nullable
+ public TypeSerializer<N> getPreviousNamespaceSerializer() {
+ return namespaceSerializerProvider.previousSchemaSerializer();
}
@Nonnull
public TypeSerializer<S> getStateSerializer() {
- return stateSerializer;
+ return stateSerializerProvider.currentSchemaSerializer();
+ }
+
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<S> updateStateSerializer(TypeSerializer<S> newStateSerializer) {
+ return stateSerializerProvider.registerNewSerializerForRestoredState(newStateSerializer);
+ }
+
+ @Nullable
+ public TypeSerializer<S> getPreviousStateSerializer() {
+ return stateSerializerProvider.previousSchemaSerializer();
}
@Nullable
@@ -104,6 +150,10 @@ public RegisteredKeyValueStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sna
return snapshotTransformer;
}
+ public void updateSnapshotTransformer(StateSnapshotTransformer<S> snapshotTransformer) {
+ this.snapshotTransformer = snapshotTransformer;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -133,8 +183,8 @@ public String toString() {
return "RegisteredKeyedBackendStateMetaInfo{" +
"stateType=" + stateType +
", name='" + name + '\'' +
- ", namespaceSerializer=" + namespaceSerializer +
- ", stateSerializer=" + stateSerializer +
+ ", namespaceSerializer=" + getNamespaceSerializer() +
+ ", stateSerializer=" + getStateSerializer() +
'}';
}
@@ -153,34 +203,19 @@ public StateMetaInfoSnapshot snapshot() {
return computeSnapshot();
}
- public static void checkStateMetaInfo(StateMetaInfoSnapshot stateMetaInfoSnapshot, StateDescriptor<?, ?> stateDesc) {
- Preconditions.checkState(
- stateMetaInfoSnapshot != null,
- "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," +
- " but its corresponding restored snapshot cannot be found.");
-
- Preconditions.checkState(stateMetaInfoSnapshot.getBackendStateType()
- == StateMetaInfoSnapshot.BackendStateType.KEY_VALUE,
- "Incompatible state types. " +
- "Was [" + stateMetaInfoSnapshot.getBackendStateType() + "], " +
- "registered as [" + StateMetaInfoSnapshot.BackendStateType.KEY_VALUE + "].");
+ public void checkStateMetaInfo(StateDescriptor<?, ?> stateDesc) {
Preconditions.checkState(
- Objects.equals(stateDesc.getName(), stateMetaInfoSnapshot.getName()),
+ Objects.equals(stateDesc.getName(), getName()),
"Incompatible state names. " +
- "Was [" + stateMetaInfoSnapshot.getName() + "], " +
+ "Was [" + getName() + "], " +
"registered with [" + stateDesc.getName() + "].");
- final StateDescriptor.Type restoredType =
- StateDescriptor.Type.valueOf(
- stateMetaInfoSnapshot.getOption(
- StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE));
-
- if (stateDesc.getType() != StateDescriptor.Type.UNKNOWN && restoredType != StateDescriptor.Type.UNKNOWN) {
+ if (stateDesc.getType() != StateDescriptor.Type.UNKNOWN && getStateType() != StateDescriptor.Type.UNKNOWN) {
Preconditions.checkState(
- stateDesc.getType() == restoredType,
+ stateDesc.getType() == getStateType(),
"Incompatible key/value state types. " +
- "Was [" + restoredType + "], " +
+ "Was [" + getStateType() + "], " +
"registered with [" + stateDesc.getType() + "].");
}
}
@@ -194,8 +229,12 @@ private StateMetaInfoSnapshot computeSnapshot() {
Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap = new HashMap<>(2);
String namespaceSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString();
String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString();
+
+ TypeSerializer<N> namespaceSerializer = getNamespaceSerializer();
serializerMap.put(namespaceSerializerKey, namespaceSerializer.duplicate());
serializerConfigSnapshotsMap.put(namespaceSerializerKey, namespaceSerializer.snapshotConfiguration());
+
+ TypeSerializer<S> stateSerializer = getStateSerializer();
serializerMap.put(valueSerializerKey, stateSerializer.duplicate());
serializerConfigSnapshotsMap.put(valueSerializerKey, stateSerializer.snapshotConfiguration());
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java
index 10ba0296057..921947a4dd0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java
@@ -19,11 +19,13 @@
package org.apache.flink.runtime.state;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
import org.apache.flink.util.Preconditions;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Map;
@@ -46,21 +48,22 @@
* The type serializer for the elements in the state list
*/
@Nonnull
- private final TypeSerializer<S> partitionStateSerializer;
+ private final StateSerializerProvider<S> partitionStateSerializerProvider;
public RegisteredOperatorStateBackendMetaInfo(
@Nonnull String name,
@Nonnull TypeSerializer<S> partitionStateSerializer,
@Nonnull OperatorStateHandle.Mode assignmentMode) {
- super(name);
- this.partitionStateSerializer = partitionStateSerializer;
- this.assignmentMode = assignmentMode;
+ this(
+ name,
+ StateSerializerProvider.fromNewState(partitionStateSerializer),
+ assignmentMode);
}
private RegisteredOperatorStateBackendMetaInfo(@Nonnull RegisteredOperatorStateBackendMetaInfo<S> copy) {
this(
Preconditions.checkNotNull(copy).name,
- copy.partitionStateSerializer.duplicate(),
+ copy.getPartitionStateSerializer().duplicate(),
copy.assignmentMode);
}
@@ -68,13 +71,24 @@ private RegisteredOperatorStateBackendMetaInfo(@Nonnull RegisteredOperatorStateB
public RegisteredOperatorStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot snapshot) {
this(
snapshot.getName(),
- (TypeSerializer<S>) Preconditions.checkNotNull(
- snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)),
+ StateSerializerProvider.fromRestoredState(
+ (TypeSerializerSnapshot<S>) Preconditions.checkNotNull(
+ snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))),
OperatorStateHandle.Mode.valueOf(
snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE)));
+
Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.OPERATOR == snapshot.getBackendStateType());
}
+ private RegisteredOperatorStateBackendMetaInfo(
+ @Nonnull String name,
+ @Nonnull StateSerializerProvider<S> partitionStateSerializerProvider,
+ @Nonnull OperatorStateHandle.Mode assignmentMode) {
+ super(name);
+ this.partitionStateSerializerProvider = partitionStateSerializerProvider;
+ this.assignmentMode = assignmentMode;
+ }
+
/**
* Creates a deep copy of the itself.
*/
@@ -96,7 +110,17 @@ public StateMetaInfoSnapshot snapshot() {
@Nonnull
public TypeSerializer<S> getPartitionStateSerializer() {
- return partitionStateSerializer;
+ return partitionStateSerializerProvider.currentSchemaSerializer();
+ }
+
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<S> updatePartitionStateSerializer(TypeSerializer<S> newPartitionStateSerializer) {
+ return partitionStateSerializerProvider.registerNewSerializerForRestoredState(newPartitionStateSerializer);
+ }
+
+ @Nullable
+ public TypeSerializer<S> getPreviousPartitionStateSerializer() {
+ return partitionStateSerializerProvider.previousSchemaSerializer();
}
@Override
@@ -112,7 +136,7 @@ public boolean equals(Object obj) {
return (obj instanceof RegisteredOperatorStateBackendMetaInfo)
&& name.equals(((RegisteredOperatorStateBackendMetaInfo) obj).getName())
&& assignmentMode.equals(((RegisteredOperatorStateBackendMetaInfo) obj).getAssignmentMode())
- && partitionStateSerializer.equals(((RegisteredOperatorStateBackendMetaInfo) obj).getPartitionStateSerializer());
+ && getPartitionStateSerializer().equals(((RegisteredOperatorStateBackendMetaInfo) obj).getPartitionStateSerializer());
}
@Override
@@ -128,7 +152,7 @@ public String toString() {
return "RegisteredOperatorBackendStateMetaInfo{" +
"name='" + name + "\'" +
", assignmentMode=" + assignmentMode +
- ", partitionStateSerializer=" + partitionStateSerializer +
+ ", partitionStateSerializer=" + getPartitionStateSerializer() +
'}';
}
@@ -138,6 +162,8 @@ private StateMetaInfoSnapshot computeSnapshot() {
StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE.toString(),
assignmentMode.toString());
String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString();
+
+ TypeSerializer<S> partitionStateSerializer = getPartitionStateSerializer();
Map<String, TypeSerializer<?>> serializerMap =
Collections.singletonMap(valueSerializerKey, partitionStateSerializer.duplicate());
Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap =
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java
index 0304b929c6d..961d96fa405 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java
@@ -19,11 +19,13 @@
package org.apache.flink.runtime.state;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
import org.apache.flink.util.Preconditions;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Map;
@@ -34,24 +36,34 @@
public class RegisteredPriorityQueueStateBackendMetaInfo<T> extends RegisteredStateMetaInfoBase {
@Nonnull
- private final TypeSerializer<T> elementSerializer;
+ private final StateSerializerProvider<T> elementSerializerProvider;
public RegisteredPriorityQueueStateBackendMetaInfo(
@Nonnull String name,
@Nonnull TypeSerializer<T> elementSerializer) {
- super(name);
- this.elementSerializer = elementSerializer;
+ this(name, StateSerializerProvider.fromNewState(elementSerializer));
}
@SuppressWarnings("unchecked")
public RegisteredPriorityQueueStateBackendMetaInfo(StateMetaInfoSnapshot snapshot) {
- this(snapshot.getName(),
- (TypeSerializer<T>) Preconditions.checkNotNull(
- snapshot.restoreTypeSerializer(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)));
+ this(
+ snapshot.getName(),
+ StateSerializerProvider.fromRestoredState(
+ (TypeSerializerSnapshot<T>) Preconditions.checkNotNull(
+ snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))));
+
Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE == snapshot.getBackendStateType());
}
+ private RegisteredPriorityQueueStateBackendMetaInfo(
+ @Nonnull String name,
+ @Nonnull StateSerializerProvider<T> elementSerializerProvider) {
+
+ super(name);
+ this.elementSerializerProvider = elementSerializerProvider;
+ }
+
@Nonnull
@Override
public StateMetaInfoSnapshot snapshot() {
@@ -60,10 +72,21 @@ public StateMetaInfoSnapshot snapshot() {
@Nonnull
public TypeSerializer<T> getElementSerializer() {
- return elementSerializer;
+ return elementSerializerProvider.currentSchemaSerializer();
+ }
+
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<T> updateElementSerializer(TypeSerializer<T> newElementSerializer) {
+ return elementSerializerProvider.registerNewSerializerForRestoredState(newElementSerializer);
+ }
+
+ @Nullable
+ public TypeSerializer<T> getPreviousElementSerializer() {
+ return elementSerializerProvider.previousSchemaSerializer();
}
private StateMetaInfoSnapshot computeSnapshot() {
+ TypeSerializer<T> elementSerializer = getElementSerializer();
Map<String, TypeSerializer<?>> serializerMap =
Collections.singletonMap(
StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
@@ -82,6 +105,6 @@ private StateMetaInfoSnapshot computeSnapshot() {
}
public RegisteredPriorityQueueStateBackendMetaInfo deepCopy() {
- return new RegisteredPriorityQueueStateBackendMetaInfo<>(name, elementSerializer.duplicate());
+ return new RegisteredPriorityQueueStateBackendMetaInfo<>(name, getElementSerializer().duplicate());
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java
index 4132d144a4a..b7dff59aef0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredStateMetaInfoBase.java
@@ -42,4 +42,21 @@ public String getName() {
@Nonnull
public abstract StateMetaInfoSnapshot snapshot();
+
+ public static RegisteredStateMetaInfoBase fromMetaInfoSnapshot(@Nonnull StateMetaInfoSnapshot snapshot) {
+
+ final StateMetaInfoSnapshot.BackendStateType backendStateType = snapshot.getBackendStateType();
+ switch (backendStateType) {
+ case KEY_VALUE:
+ return new RegisteredKeyValueStateBackendMetaInfo<>(snapshot);
+ case OPERATOR:
+ return new RegisteredOperatorStateBackendMetaInfo<>(snapshot);
+ case BROADCAST:
+ return new RegisteredBroadcastStateBackendMetaInfo<>(snapshot);
+ case PRIORITY_QUEUE:
+ return new RegisteredPriorityQueueStateBackendMetaInfo<>(snapshot);
+ default:
+ throw new IllegalArgumentException("Unknown backend state type: " + backendStateType);
+ }
+ }
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java
new file mode 100644
index 00000000000..a24f12e42fb
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * A {@link StateSerializerProvider} wraps logic on how to obtain serializers for registered state,
+ * either with the previous schema of state in checkpoints or the current schema of state.
+ *
+ * @param <T> the type of the state.
+ */
+@Internal
+public abstract class StateSerializerProvider<T> {
+
+ /**
+ * The registered serializer for the state.
+ *
+ * <p>In the case that this provider was created from a restored serializer snapshot via
+ * {@link #fromRestoredState(TypeSerializerSnapshot)}, but a new serializer was never registered
+ * for the state (i.e., this is the case if a restored state was never accessed), this would be {@code null}.
+ */
+ @Nullable
+ TypeSerializer<T> registeredSerializer;
+
+ /**
+ * Creates a {@link StateSerializerProvider} for restored state from the previous serializer's snapshot.
+ *
+ * <p>Once a new serializer is registered for the state, it should be provided via
+ * the {@link #registerNewSerializerForRestoredState(TypeSerializer)} method.
+ *
+ * @param stateSerializerSnapshot the previous serializer's snapshot.
+ * @param <T> the type of the state.
+ *
+ * @return a new {@link StateSerializerProvider} for restored state.
+ */
+ public static <T> StateSerializerProvider<T> fromRestoredState(TypeSerializerSnapshot<T> stateSerializerSnapshot) {
+ return new RestoredStateSerializerProvider<>(stateSerializerSnapshot);
+ }
+
+ /**
+ * Creates a {@link StateSerializerProvider} for new state from the registered state serializer.
+ *
+ * @param registeredStateSerializer the new state's registered serializer.
+ * @param <T> the type of the state.
+ *
+ * @return a new {@link StateSerializerProvider} for new state.
+ */
+ public static <T> StateSerializerProvider<T> fromNewState(TypeSerializer<T> registeredStateSerializer) {
+ return new NewStateSerializerProvider<>(registeredStateSerializer);
+ }
+
+ private StateSerializerProvider(@Nullable TypeSerializer<T> stateSerializer) {
+ this.registeredSerializer = stateSerializer;
+ }
+
+ /**
+ * Gets the serializer that recognizes the current serialization schema of the state.
+ * This is the serializer that should be used for regular state serialization and
+ * deserialization after state has been restored.
+ *
+ * <p>If this provider was created from a restored state's serializer snapshot, while a
+ * new serializer (with a new schema) was not registered for the state (i.e., because
+ * the state was never accessed after it was restored), then the schema of state remains
+ * identical. Therefore, in this case, it is guaranteed that the serializer returned by
+ * this method is the same as the one returned by {@link #previousSchemaSerializer()}.
+ *
+ * <p>If this provider was created from new state, then this always returns the
+ * serializer that the new state was registered with.
+ *
+ * @return a serializer that reads and writes in the current schema of the state.
+ */
+ @Nonnull
+ public abstract TypeSerializer<T> currentSchemaSerializer();
+
+ /**
+ * Gets the serializer that recognizes the previous serialization schema of the state.
+ * This is the serializer that should be used for restoring the state, i.e. when the state
+ * is still in the previous serialization schema.
+ *
+ * <p>This method can only be used if this provider was created from a restored state's serializer
+ * snapshot. If this provider was created from new state, then this method is
+ * irrelevant, since there doesn't exist any previous version of the state schema.
+ *
+ * @return a serializer that reads and writes in the previous schema of the state.
+ */
+ @Nonnull
+ public abstract TypeSerializer<T> previousSchemaSerializer();
+
+ /**
+ * For restored state, register a new serializer that potentially has a new serialization schema.
+ *
+ * <p>Users are allowed to register serializers for state only once. Therefore, this method
+ * is irrelevant if this provider was created from new state, since a state serializer had
+ * been registered already.
+ *
+ * <p>For the case where this provider was created from restored state, then this method should
+ * be called at most once. The new serializer will be checked for its schema compatibility with the
+ * previous serializer's schema, and returned to the caller. The caller is responsible for
+ * checking the result and react appropriately to it, as follows:
+ * <ul>
+ * <li>{@link TypeSerializerSchemaCompatibility#isCompatibleAsIs()}: nothing needs to be done.
+ * {@link #currentSchemaSerializer()} now returns the newly registered serializer.</li>
+ * <li>{@link TypeSerializerSchemaCompatibility#isCompatibleAfterMigration()} ()}: state needs to be
+ * migrated before the serializer returned by {@link #currentSchemaSerializer()} can be used.
+ * The migration should be performed by reading the state with {@link #previousSchemaSerializer()},
+ * and then writing it again with {@link #currentSchemaSerializer()}.</li>
+ * <li>{@link TypeSerializerSchemaCompatibility#isIncompatible()}: the registered serializer is
+ * incompatible. {@link #currentSchemaSerializer()} can no longer return a serializer for
+ * the state, and therefore this provider shouldn't be used anymore.</li>
+ * </ul>
+ *
+ * @return the schema compatibility of the new registered serializer, with respect to the previous serializer.
+ */
+ @Nonnull
+ public abstract TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer);
+
+ /**
+ * Implementation of the {@link StateSerializerProvider} for the restored state case.
+ */
+ private static class RestoredStateSerializerProvider<T> extends StateSerializerProvider<T> {
+
+ /**
+ * The snapshot of the previous serializer of the state.
+ */
+ @Nonnull
+ private final TypeSerializerSnapshot<T> previousSerializerSnapshot;
+
+ private boolean isRegisteredWithIncompatibleSerializer = false;
+
+ RestoredStateSerializerProvider(TypeSerializerSnapshot<T> previousSerializerSnapshot) {
+ super(null);
+ this.previousSerializerSnapshot = Preconditions.checkNotNull(previousSerializerSnapshot);
+ }
+
+ /**
+ * The restore serializer, lazily created only when the restore serializer is accessed.
+ *
+ * <p>NOTE: It is important to only create this lazily, so that off-heap
+ * state do not fail eagerly when restoring state that has a
+ * {@link UnloadableDummyTypeSerializer} as the previous serializer. This should
+ * be relevant only for restores from Flink versions prior to 1.7.x.
+ */
+ @Nullable
+ private TypeSerializer<T> cachedRestoredSerializer;
+
+ @Override
+ @Nonnull
+ public TypeSerializer<T> currentSchemaSerializer() {
+ if (registeredSerializer != null) {
+ checkState(
+ !isRegisteredWithIncompatibleSerializer,
+ "Unable to provide a serializer with the current schema, because the restored state was " +
+ "registered with a new serializer that has incompatible schema.");
+
+ return registeredSerializer;
+ }
+
+ // if we are not yet registered with a new serializer,
+ // we can just use the restore serializer to read / write the state.
+ return previousSchemaSerializer();
+ }
+
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) {
+ checkNotNull(newSerializer);
+ if (registeredSerializer != null) {
+ throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed.");
+ }
+
+ TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer);
+ if (result.isIncompatible()) {
+ this.isRegisteredWithIncompatibleSerializer = true;
+ }
+ this.registeredSerializer = newSerializer;
+ return result;
+ }
+
+ @Nonnull
+ public final TypeSerializer<T> previousSchemaSerializer() {
+ if (cachedRestoredSerializer != null) {
+ return cachedRestoredSerializer;
+ }
+
+ this.cachedRestoredSerializer = previousSerializerSnapshot.restoreSerializer();
+ return cachedRestoredSerializer;
+ }
+ }
+
+ /**
+ * Implementation of the {@link StateSerializerProvider} for the new state case.
+ */
+ private static class NewStateSerializerProvider<T> extends StateSerializerProvider<T> {
+
+ NewStateSerializerProvider(TypeSerializer<T> registeredStateSerializer) {
+ super(Preconditions.checkNotNull(registeredStateSerializer));
+ }
+
+ @Override
+ @Nonnull
+ @SuppressWarnings("ConstantConditions")
+ public TypeSerializer<T> currentSchemaSerializer() {
+ return registeredSerializer;
+ }
+
+ @Override
+ @Nonnull
+ public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) {
+ throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed.");
+ }
+
+ @Override
+ @Nonnull
+ public TypeSerializer<T> previousSchemaSerializer() {
+ throw new UnsupportedOperationException("This is a NewStateSerializerProvider; you cannot get a restore serializer because there was no restored state.");
+ }
+ }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index 4eff3a285bb..3f8761b657a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -30,7 +30,6 @@
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
-import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
@@ -79,6 +78,7 @@
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
@@ -125,14 +125,6 @@
*/
private final Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates;
- /**
- * Map of state names to their corresponding restored state meta info.
- *
- * <p>TODO this map can be removed when eager-state registration is in place.
- * TODO we currently need this cached to check state migration strategies when new serializers are registered.
- */
- private final Map<StateUID, StateMetaInfoSnapshot> restoredStateMetaInfo;
-
/**
* The configuration for local recovery.
*/
@@ -173,7 +165,6 @@ public HeapKeyedStateBackend(
this.snapshotStrategy = new HeapSnapshotStrategy(synchronicityTrait);
LOG.info("Initializing heap keyed state backend with stream factory.");
- this.restoredStateMetaInfo = new HashMap<>();
this.priorityQueueSetFactory = priorityQueueSetFactory;
}
@@ -194,23 +185,9 @@ public HeapKeyedStateBackend(
// TODO we implement the simple way of supporting the current functionality, mimicking keyed state
// because this should be reworked in FLINK-9376 and then we should have a common algorithm over
// StateMetaInfoSnapshot that avoids this code duplication.
- StateMetaInfoSnapshot restoredMetaInfoSnapshot =
- restoredStateMetaInfo.get(StateUID.of(stateName, StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE));
-
- Preconditions.checkState(
- restoredMetaInfoSnapshot != null,
- "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," +
- " but its corresponding restored snapshot cannot be found.");
-
- StateMetaInfoSnapshot.CommonSerializerKeys serializerKey =
- StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER;
-
- @SuppressWarnings("unchecked")
- TypeSerializerSnapshot<T> serializerSnapshot = Preconditions.checkNotNull(
- (TypeSerializerSnapshot<T>) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot(serializerKey));
TypeSerializerSchemaCompatibility<T> compatibilityResult =
- serializerSnapshot.resolveSchemaCompatibility(byteOrderedElementSerializer);
+ existingState.getMetaInfo().updateElementSerializer(byteOrderedElementSerializer);
if (compatibilityResult.isIncompatible()) {
throw new FlinkRuntimeException(new StateMigrationException("For heap backends, the new priority queue serializer must not be incompatible."));
@@ -252,57 +229,42 @@ public HeapKeyedStateBackend(
private <N, V> StateTable<K, N, V> tryRegisterStateTable(
TypeSerializer<N> namespaceSerializer,
StateDescriptor<?, V> stateDesc,
- StateSnapshotTransformer<V> snapshotTransformer) throws StateMigrationException {
+ @Nullable StateSnapshotTransformer<V> snapshotTransformer) throws StateMigrationException {
@SuppressWarnings("unchecked")
StateTable<K, N, V> stateTable = (StateTable<K, N, V>) registeredKVStates.get(stateDesc.getName());
TypeSerializer<V> newStateSerializer = stateDesc.getSerializer();
- RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
- stateDesc.getType(),
- stateDesc.getName(),
- namespaceSerializer,
- newStateSerializer,
- snapshotTransformer);
if (stateTable != null) {
- @SuppressWarnings("unchecked")
- StateMetaInfoSnapshot restoredMetaInfoSnapshot =
- restoredStateMetaInfo.get(
- StateUID.of(stateDesc.getName(), StateMetaInfoSnapshot.BackendStateType.KEY_VALUE));
-
- Preconditions.checkState(
- restoredMetaInfoSnapshot != null,
- "Requested to check compatibility of a restored RegisteredKeyedBackendStateMetaInfo," +
- " but its corresponding restored snapshot cannot be found.");
+ RegisteredKeyValueStateBackendMetaInfo<N, V> restoredKvMetaInfo = stateTable.getMetaInfo();
- @SuppressWarnings("unchecked")
- TypeSerializerSnapshot<N> namespaceSerializerSnapshot = Preconditions.checkNotNull(
- (TypeSerializerSnapshot<N>) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot(
- StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString()));
+ restoredKvMetaInfo.updateSnapshotTransformer(snapshotTransformer);
TypeSerializerSchemaCompatibility<N> namespaceCompatibility =
- namespaceSerializerSnapshot.resolveSchemaCompatibility(namespaceSerializer);
- if (namespaceCompatibility.isIncompatible()) {
- throw new StateMigrationException("For heap backends, the new namespace serializer must not be incompatible.");
+ restoredKvMetaInfo.updateNamespaceSerializer(namespaceSerializer);
+ if (!namespaceCompatibility.isCompatibleAsIs()) {
+ throw new StateMigrationException("For heap backends, the new namespace serializer must be compatible.");
}
- @SuppressWarnings("unchecked")
- TypeSerializerSnapshot<V> stateSerializerSnapshot = Preconditions.checkNotNull(
- (TypeSerializerSnapshot<V>) restoredMetaInfoSnapshot.getTypeSerializerConfigSnapshot(
- StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString()));
-
- RegisteredKeyValueStateBackendMetaInfo.checkStateMetaInfo(restoredMetaInfoSnapshot, stateDesc);
+ restoredKvMetaInfo.checkStateMetaInfo(stateDesc);
TypeSerializerSchemaCompatibility<V> stateCompatibility =
- stateSerializerSnapshot.resolveSchemaCompatibility(newStateSerializer);
+ restoredKvMetaInfo.updateStateSerializer(newStateSerializer);
if (stateCompatibility.isIncompatible()) {
throw new StateMigrationException("For heap backends, the new state serializer must not be incompatible.");
}
- stateTable.setMetaInfo(newMetaInfo);
+ stateTable.setMetaInfo(restoredKvMetaInfo);
} else {
+ RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
+ stateDesc.getType(),
+ stateDesc.getName(),
+ namespaceSerializer,
+ newStateSerializer,
+ snapshotTransformer);
+
stateTable = snapshotStrategy.newStateTable(newMetaInfo);
registeredKVStates.put(stateDesc.getName(), stateTable);
}
@@ -536,10 +498,6 @@ private void createOrCheckStateForMetaInfo(
Map<Integer, StateMetaInfoSnapshot> kvStatesById) {
for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
- restoredStateMetaInfo.put(
- StateUID.of(metaInfoSnapshot.getName(), metaInfoSnapshot.getBackendStateType()),
- metaInfoSnapshot);
-
final StateSnapshotRestore registeredState;
switch (metaInfoSnapshot.getBackendStateType()) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java
index 77c267adff1..836edef0aac 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/LegacyStateMetaInfoReaders.java
@@ -132,11 +132,6 @@ public StateMetaInfoSnapshot readStateMetaInfoSnapshot(
static final OperatorBackendStateMetaInfoReaderV2V3 INSTANCE = new OperatorBackendStateMetaInfoReaderV2V3();
- private static final String[] ORDERED_KEY_STRINGS =
- new String[]{
- StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER.toString(),
- StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString()};
-
@Nonnull
@Override
public StateMetaInfoSnapshot readStateMetaInfoSnapshot(
@@ -156,17 +151,25 @@ public StateMetaInfoSnapshot readStateMetaInfoSnapshot(
final int listSize = stateSerializerAndConfigList.size();
StateMetaInfoSnapshot.BackendStateType stateType = listSize == 1 ?
StateMetaInfoSnapshot.BackendStateType.OPERATOR : StateMetaInfoSnapshot.BackendStateType.BROADCAST;
- Map<String, TypeSerializerSnapshot<?>> serializerConfigsMap = new HashMap<>(listSize);
- for (int i = 0; i < listSize; ++i) {
- Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>> serializerAndConf =
- stateSerializerAndConfigList.get(i);
-
- // this particular mapping happens to support both, V2 and V3
- String serializerKey = ORDERED_KEY_STRINGS[ORDERED_KEY_STRINGS.length - 1 - i];
- serializerConfigsMap.put(
- serializerKey,
- serializerAndConf.f1);
+ Map<String, TypeSerializerSnapshot<?>> serializerConfigsMap = new HashMap<>(listSize);
+ switch (stateType) {
+ case OPERATOR:
+ serializerConfigsMap.put(
+ StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
+ stateSerializerAndConfigList.get(0).f1);
+ break;
+ case BROADCAST:
+ serializerConfigsMap.put(
+ StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER.toString(),
+ stateSerializerAndConfigList.get(0).f1);
+
+ serializerConfigsMap.put(
+ StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
+ stateSerializerAndConfigList.get(1).f1);
+ break;
+ default:
+ throw new IllegalStateException("Unknown operator state type " + stateType);
}
return new StateMetaInfoSnapshot(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java
index 1e9d9191079..9b05500e4d0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshot.java
@@ -81,7 +81,7 @@
/** The configurations of all the type serializers used with the state. */
@Nonnull
- private final Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshots;
+ private final Map<String, TypeSerializerSnapshot<?>> serializerSnapshots;
// TODO this will go away once all serializers have the restoreSerializer() factory method properly implemented.
/** The serializers used by the state. */
@@ -92,8 +92,8 @@ public StateMetaInfoSnapshot(
@Nonnull String name,
@Nonnull BackendStateType backendStateType,
@Nonnull Map<String, String> options,
- @Nonnull Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshots) {
- this(name, backendStateType, options, serializerConfigSnapshots, new HashMap<>());
+ @Nonnull Map<String, TypeSerializerSnapshot<?>> serializerSnapshots) {
+ this(name, backendStateType, options, serializerSnapshots, new HashMap<>());
}
/**
@@ -106,12 +106,12 @@ public StateMetaInfoSnapshot(
@Nonnull String name,
@Nonnull BackendStateType backendStateType,
@Nonnull Map<String, String> options,
- @Nonnull Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshots,
+ @Nonnull Map<String, TypeSerializerSnapshot<?>> serializerSnapshots,
@Nonnull Map<String, TypeSerializer<?>> serializers) {
this.name = name;
this.backendStateType = backendStateType;
this.options = options;
- this.serializerConfigSnapshots = serializerConfigSnapshots;
+ this.serializerSnapshots = serializerSnapshots;
this.serializers = serializers;
}
@@ -121,13 +121,13 @@ public BackendStateType getBackendStateType() {
}
@Nullable
- public TypeSerializerSnapshot<?> getTypeSerializerConfigSnapshot(@Nonnull String key) {
- return serializerConfigSnapshots.get(key);
+ public TypeSerializerSnapshot<?> getTypeSerializerSnapshot(@Nonnull String key) {
+ return serializerSnapshots.get(key);
}
@Nullable
- public TypeSerializerSnapshot<?> getTypeSerializerConfigSnapshot(@Nonnull CommonSerializerKeys key) {
- return getTypeSerializerConfigSnapshot(key.toString());
+ public TypeSerializerSnapshot<?> getTypeSerializerSnapshot(@Nonnull CommonSerializerKeys key) {
+ return getTypeSerializerSnapshot(key.toString());
}
@Nullable
@@ -150,20 +150,9 @@ public String getName() {
return name;
}
- @Nullable
- public TypeSerializer<?> restoreTypeSerializer(@Nonnull String key) {
- TypeSerializerSnapshot<?> configSnapshot = getTypeSerializerConfigSnapshot(key);
- return (configSnapshot != null) ? configSnapshot.restoreSerializer() : null;
- }
-
- @Nullable
- public TypeSerializer<?> restoreTypeSerializer(@Nonnull CommonSerializerKeys key) {
- return restoreTypeSerializer(key.toString());
- }
-
@Nonnull
- public Map<String, TypeSerializerSnapshot<?>> getSerializerConfigSnapshotsImmutable() {
- return Collections.unmodifiableMap(serializerConfigSnapshots);
+ public Map<String, TypeSerializerSnapshot<?>> getSerializerSnapshotsImmutable() {
+ return Collections.unmodifiableMap(serializerSnapshots);
}
/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java
index 4408dfcacef..ad1e7be2871 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/metainfo/StateMetaInfoSnapshotReadersWriters.java
@@ -165,7 +165,7 @@ public void writeStateMetaInfoSnapshot(
@Nonnull DataOutputView outputView) throws IOException {
final Map<String, String> optionsMap = snapshot.getOptionsImmutable();
final Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap =
- snapshot.getSerializerConfigSnapshotsImmutable();
+ snapshot.getSerializerSnapshotsImmutable();
outputView.writeUTF(snapshot.getName());
outputView.writeInt(snapshot.getBackendStateType().ordinal());
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java
index cf1e7f7f82d..2d09b46464f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java
@@ -21,15 +21,16 @@
import org.junit.Test;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
+/** Test checkpoint statistics counters. */
public class CheckpointStatsCountsTest {
/**
* Tests that counts are reported correctly.
*/
@Test
- public void testCounts() throws Exception {
+ public void testCounts() {
CheckpointStatsCounts counts = new CheckpointStatsCounts();
assertEquals(0, counts.getNumberOfRestoredCheckpoints());
assertEquals(0, counts.getTotalNumberOfCheckpoints());
@@ -80,19 +81,15 @@ public void testCounts() throws Exception {
* incrementing the in progress checkpoints before throws an Exception.
*/
@Test
- public void testCompleteOrFailWithoutInProgressCheckpoint() throws Exception {
+ public void testCompleteOrFailWithoutInProgressCheckpoint() {
CheckpointStatsCounts counts = new CheckpointStatsCounts();
- try {
- counts.incrementCompletedCheckpoints();
- fail("Did not throw expected Exception");
- } catch (IllegalStateException ignored) {
- }
-
- try {
- counts.incrementFailedCheckpoints();
- fail("Did not throw expected Exception");
- } catch (IllegalStateException ignored) {
- }
+ counts.incrementCompletedCheckpoints();
+ assertTrue("Number of checkpoints in progress should never be negative",
+ counts.getNumberOfInProgressCheckpoints() >= 0);
+
+ counts.incrementFailedCheckpoints();
+ assertTrue("Number of checkpoints in progress should never be negative",
+ counts.getNumberOfInProgressCheckpoints() >= 0);
}
/**
@@ -100,7 +97,7 @@ public void testCompleteOrFailWithoutInProgressCheckpoint() throws Exception {
* parent.
*/
@Test
- public void testCreateSnapshot() throws Exception {
+ public void testCreateSnapshot() {
CheckpointStatsCounts counts = new CheckpointStatsCounts();
counts.incrementRestoredCheckpoints();
counts.incrementRestoredCheckpoints();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ApplicationStatusTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ApplicationStatusTest.java
new file mode 100644
index 00000000000..8fb7cac55a9
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ApplicationStatusTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.clusterframework;
+
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Tests for the {@link ApplicationStatus}.
+ */
+public class ApplicationStatusTest extends TestLogger {
+
+ private static final int SUCCESS_EXIT_CODE = 0;
+
+ @Test
+ public void succeededStatusMapsToSuccessExitCode() {
+ int exitCode = ApplicationStatus.SUCCEEDED.processExitCode();
+ assertThat(exitCode, is(equalTo(SUCCESS_EXIT_CODE)));
+ }
+
+ @Test
+ public void cancelledStatusMapsToSuccessExitCode() {
+ int exitCode = ApplicationStatus.CANCELED.processExitCode();
+ assertThat(exitCode, is(equalTo(SUCCESS_EXIT_CODE)));
+ }
+
+ @Test
+ public void notSucceededNorCancelledStatusMapsToNonSuccessExitCode() {
+ Iterable<Integer> exitCodes = exitCodes(notSucceededNorCancelledStatus());
+ assertThat(exitCodes, not(contains(SUCCESS_EXIT_CODE)));
+ }
+
+ private static Iterable<Integer> exitCodes(Iterable<ApplicationStatus> statuses) {
+ return StreamSupport.stream(statuses.spliterator(), false)
+ .map(ApplicationStatus::processExitCode)
+ .collect(Collectors.toList());
+ }
+
+ private static Iterable<ApplicationStatus> notSucceededNorCancelledStatus() {
+ return Arrays.stream(ApplicationStatus.values())
+ .filter(ApplicationStatusTest::isNotSucceededNorCancelled)
+ .collect(Collectors.toList());
+ }
+
+ private static boolean isNotSucceededNorCancelled(ApplicationStatus status) {
+ return status != ApplicationStatus.SUCCEEDED && status != ApplicationStatus.CANCELED;
+ }
+
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/AsynchronousFileIOChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/AsynchronousFileIOChannelTest.java
index a471e663ea3..e3d59075b3d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/AsynchronousFileIOChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/AsynchronousFileIOChannelTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.io.disk.iomanager;
-import org.apache.flink.core.memory.HeapMemorySegment;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.runtime.io.network.buffer.Buffer;
@@ -428,4 +427,4 @@ int getNumberOfOutstandingRequests() {
return requestsNotReturned.get();
}
}
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
index 2ebaeba7de7..cd369496d40 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
@@ -24,7 +24,6 @@
import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.highavailability.TestingManualHighAvailabilityServices;
import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java
index 5f83e794ff9..1aab6f7de43 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricDumpSerializerTest.java
@@ -70,7 +70,10 @@ public Object getValue() {
Collections.<Meter, Tuple2<QueryScopeInfo, String>>emptyMap());
// no metrics should be serialized
- Assert.assertEquals(0, output.serializedMetrics.length);
+ Assert.assertEquals(0, output.serializedCounters.length);
+ Assert.assertEquals(0, output.serializedGauges.length);
+ Assert.assertEquals(0, output.serializedHistograms.length);
+ Assert.assertEquals(0, output.serializedMeters.length);
List<MetricDump> deserialized = deserializer.deserialize(output);
Assert.assertEquals(0, deserialized.size());
@@ -141,7 +144,8 @@ public long getCount() {
gauges.put(g1, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.TaskQueryScopeInfo("jid", "vid", 2, "D"), "g1"));
histograms.put(h1, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.OperatorQueryScopeInfo("jid", "vid", 2, "opname", "E"), "h1"));
- MetricDumpSerialization.MetricSerializationResult serialized = serializer.serialize(counters, gauges, histograms, meters);
+ MetricDumpSerialization.MetricSerializationResult serialized = serializer.serialize(
+ counters, gauges, histograms, meters);
List<MetricDump> deserialized = deserializer.deserialize(serialized);
// ===== Counters ==============================================================================================
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java
index 3767421b7d6..673409cca5f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/dump/MetricQueryServiceTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.metrics.dump;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
@@ -25,10 +26,10 @@
import org.apache.flink.metrics.Meter;
import org.apache.flink.metrics.SimpleCounter;
import org.apache.flink.metrics.util.TestHistogram;
+import org.apache.flink.metrics.util.TestMeter;
import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
-import org.apache.flink.runtime.metrics.MetricRegistryImpl;
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
+import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;
import org.apache.flink.util.TestLogger;
import akka.actor.ActorRef;
@@ -38,6 +39,10 @@
import akka.testkit.TestActorRef;
import org.junit.Test;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -47,81 +52,119 @@
public class MetricQueryServiceTest extends TestLogger {
@Test
public void testCreateDump() throws Exception {
-
ActorSystem s = AkkaUtils.createLocalActorSystem(new Configuration());
- ActorRef serviceActor = MetricQueryService.startMetricQueryService(s, null);
- TestActorRef testActorRef = TestActorRef.create(s, Props.create(TestActor.class));
- TestActor testActor = (TestActor) testActorRef.underlyingActor();
-
- final Counter c = new SimpleCounter();
- final Gauge<String> g = new Gauge<String>() {
- @Override
- public String getValue() {
- return "Hello";
- }
- };
- final Histogram h = new TestHistogram();
- final Meter m = new Meter() {
+ try {
+ ActorRef serviceActor = MetricQueryService.startMetricQueryService(s, null, Long.MAX_VALUE);
+ TestActorRef testActorRef = TestActorRef.create(s, Props.create(TestActor.class));
+ TestActor testActor = (TestActor) testActorRef.underlyingActor();
- @Override
- public void markEvent() {
- }
+ final Counter c = new SimpleCounter();
+ final Gauge<String> g = () -> "Hello";
+ final Histogram h = new TestHistogram();
+ final Meter m = new TestMeter();
- @Override
- public void markEvent(long n) {
- }
+ final TaskManagerMetricGroup tm = UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup();
- @Override
- public double getRate() {
- return 5;
- }
+ MetricQueryService.notifyOfAddedMetric(serviceActor, c, "counter", tm);
+ MetricQueryService.notifyOfAddedMetric(serviceActor, g, "gauge", tm);
+ MetricQueryService.notifyOfAddedMetric(serviceActor, h, "histogram", tm);
+ MetricQueryService.notifyOfAddedMetric(serviceActor, m, "meter", tm);
+ serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
- @Override
- public long getCount() {
- return 10;
- }
- };
-
- MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());
- final TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
-
- MetricQueryService.notifyOfAddedMetric(serviceActor, c, "counter", tm);
- MetricQueryService.notifyOfAddedMetric(serviceActor, g, "gauge", tm);
- MetricQueryService.notifyOfAddedMetric(serviceActor, h, "histogram", tm);
- MetricQueryService.notifyOfAddedMetric(serviceActor, m, "meter", tm);
- serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
- synchronized (testActor.lock) {
- if (testActor.message == null) {
- testActor.lock.wait();
- }
- }
+ testActor.waitForResult();
- MetricDumpSerialization.MetricSerializationResult dump = (MetricDumpSerialization.MetricSerializationResult) testActor.message;
- testActor.message = null;
- assertTrue(dump.serializedMetrics.length > 0);
+ MetricDumpSerialization.MetricSerializationResult dump = testActor.getSerializationResult();
- MetricQueryService.notifyOfRemovedMetric(serviceActor, c);
- MetricQueryService.notifyOfRemovedMetric(serviceActor, g);
- MetricQueryService.notifyOfRemovedMetric(serviceActor, h);
- MetricQueryService.notifyOfRemovedMetric(serviceActor, m);
+ assertTrue(dump.serializedCounters.length > 0);
+ assertTrue(dump.serializedGauges.length > 0);
+ assertTrue(dump.serializedHistograms.length > 0);
+ assertTrue(dump.serializedMeters.length > 0);
- serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
- synchronized (testActor.lock) {
- if (testActor.message == null) {
- testActor.lock.wait();
- }
+ MetricQueryService.notifyOfRemovedMetric(serviceActor, c);
+ MetricQueryService.notifyOfRemovedMetric(serviceActor, g);
+ MetricQueryService.notifyOfRemovedMetric(serviceActor, h);
+ MetricQueryService.notifyOfRemovedMetric(serviceActor, m);
+
+ serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
+
+ testActor.waitForResult();
+
+ MetricDumpSerialization.MetricSerializationResult emptyDump = testActor.getSerializationResult();
+
+ assertEquals(0, emptyDump.serializedCounters.length);
+ assertEquals(0, emptyDump.serializedGauges.length);
+ assertEquals(0, emptyDump.serializedHistograms.length);
+ assertEquals(0, emptyDump.serializedMeters.length);
+ } finally {
+ s.terminate();
}
+ }
+
+ @Test
+ public void testHandleOversizedMetricMessage() throws Exception {
+ ActorSystem s = AkkaUtils.createLocalActorSystem(new Configuration());
+ try {
+ final long sizeLimit = 200L;
+ ActorRef serviceActor = MetricQueryService.startMetricQueryService(s, null, sizeLimit);
+ TestActorRef testActorRef = TestActorRef.create(s, Props.create(TestActor.class));
+ TestActor testActor = (TestActor) testActorRef.underlyingActor();
+
+ final TaskManagerMetricGroup tm = UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup();
+
+ final String gaugeValue = "Hello";
+ final long requiredGaugesToExceedLimit = sizeLimit / gaugeValue.length() + 1;
+ List<Tuple2<String, Gauge<String>>> gauges = LongStream.range(0, requiredGaugesToExceedLimit)
+ .mapToObj(x -> Tuple2.of("gauge" + x, (Gauge<String>) () -> "Hello" + x))
+ .collect(Collectors.toList());
+ gauges.forEach(gauge -> MetricQueryService.notifyOfAddedMetric(serviceActor, gauge.f1, gauge.f0, tm));
+
+ MetricQueryService.notifyOfAddedMetric(serviceActor, new SimpleCounter(), "counter", tm);
+ MetricQueryService.notifyOfAddedMetric(serviceActor, new TestHistogram(), "histogram", tm);
+ MetricQueryService.notifyOfAddedMetric(serviceActor, new TestMeter(), "meter", tm);
+
+ serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
+ testActor.waitForResult();
- MetricDumpSerialization.MetricSerializationResult emptyDump = (MetricDumpSerialization.MetricSerializationResult) testActor.message;
- testActor.message = null;
- assertEquals(0, emptyDump.serializedMetrics.length);
+ MetricDumpSerialization.MetricSerializationResult dump = testActor.getSerializationResult();
- s.terminate();
+ assertTrue(dump.serializedCounters.length > 0);
+ assertEquals(1, dump.numCounters);
+ assertTrue(dump.serializedMeters.length > 0);
+ assertEquals(1, dump.numMeters);
+
+ // gauges exceeded the size limit and will be excluded
+ assertEquals(0, dump.serializedGauges.length);
+ assertEquals(0, dump.numGauges);
+
+ assertTrue(dump.serializedHistograms.length > 0);
+ assertEquals(1, dump.numHistograms);
+
+ // unregister all but one gauge to ensure gauges are reported again if the remaining fit
+ for (int x = 1; x < gauges.size(); x++) {
+ MetricQueryService.notifyOfRemovedMetric(serviceActor, gauges.get(x).f1);
+ }
+
+ serviceActor.tell(MetricQueryService.getCreateDump(), testActorRef);
+ testActor.waitForResult();
+
+ MetricDumpSerialization.MetricSerializationResult recoveredDump = testActor.getSerializationResult();
+
+ assertTrue(recoveredDump.serializedCounters.length > 0);
+ assertEquals(1, recoveredDump.numCounters);
+ assertTrue(recoveredDump.serializedMeters.length > 0);
+ assertEquals(1, recoveredDump.numMeters);
+ assertTrue(recoveredDump.serializedGauges.length > 0);
+ assertEquals(1, recoveredDump.numGauges);
+ assertTrue(recoveredDump.serializedHistograms.length > 0);
+ assertEquals(1, recoveredDump.numHistograms);
+ } finally {
+ s.terminate();
+ }
}
private static class TestActor extends UntypedActor {
- public Object message;
- public Object lock = new Object();
+ private Object message;
+ private final Object lock = new Object();
@Override
public void onReceive(Object message) throws Exception {
@@ -130,5 +173,19 @@ public void onReceive(Object message) throws Exception {
lock.notifyAll();
}
}
+
+ void waitForResult() throws InterruptedException {
+ synchronized (lock) {
+ if (message == null) {
+ lock.wait();
+ }
+ }
+ }
+
+ MetricDumpSerialization.MetricSerializationResult getSerializationResult() {
+ final MetricDumpSerialization.MetricSerializationResult result = (MetricDumpSerialization.MetricSerializationResult) message;
+ message = null;
+ return result;
+ }
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java
index a94227c23d7..85315b74196 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTableITCase.java
@@ -38,7 +38,6 @@
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memory.MemoryAllocationException;
import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.operators.hash.MutableHashTable.HashBucketIterator;
import org.apache.flink.runtime.operators.testutils.DummyInvokable;
import org.apache.flink.runtime.operators.testutils.UniformIntPairGenerator;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTablePerformanceComparison.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTablePerformanceComparison.java
index f426a9428df..bc9daf27db8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTablePerformanceComparison.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashTablePerformanceComparison.java
@@ -36,7 +36,6 @@
import org.apache.flink.runtime.operators.testutils.types.IntPairSerializer;
import org.apache.flink.util.MutableObjectIterator;
-import org.junit.AfterClass;
import org.junit.Test;
import static org.junit.Assert.*;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/InPlaceMutableHashTableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/InPlaceMutableHashTableTest.java
index beeccecd21c..ad99b5c6a86 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/InPlaceMutableHashTableTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/InPlaceMutableHashTableTest.java
@@ -45,7 +45,6 @@
import java.util.*;
import static org.junit.Assert.*;
-import static org.junit.Assert.fail;
public class InPlaceMutableHashTableTest extends MutableHashTableTestBase {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java
index 16485caf2a3..9b0b1dcd160 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java
@@ -32,7 +32,6 @@
import org.apache.flink.runtime.operators.Driver;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.TaskConfig;
-import org.apache.flink.runtime.state.TestTaskStateManager;
import org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory;
import org.apache.flink.types.Record;
import org.apache.flink.util.InstantiationUtil;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/BloomFilterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/BloomFilterTest.java
index 957d6c3b820..256d4bc4299 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/BloomFilterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/BloomFilterTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.operators.util;
-import org.apache.flink.core.memory.HeapMemorySegment;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.core.memory.MemorySegmentFactory;
@@ -162,4 +161,4 @@ public void testHashcodeInput() {
assertTrue(bloomFilter.testHash(val4));
assertTrue(bloomFilter.testHash(val5));
}
-}
\ No newline at end of file
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java
index 858c6620b3c..771fd8a837c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java
@@ -20,6 +20,7 @@
import org.apache.flink.runtime.io.network.netty.NettyLeakDetectionResource;
import org.apache.flink.runtime.rest.util.RestMapperUtils;
+import org.apache.flink.util.FileUtils;
import org.apache.flink.util.TestLogger;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
@@ -123,6 +124,20 @@ private static Request finalizeRequest(MultipartBody.Builder builder, String hea
return builder.addFormDataPart(attribute, jsonPayload);
}
+ @Test
+ public void testUploadDirectoryRegeneration() throws Exception {
+ OkHttpClient client = new OkHttpClient();
+
+ MultipartUploadResource.MultipartFileHandler fileHandler = MULTIPART_UPLOAD_RESOURCE.getFileHandler();
+
+ FileUtils.deleteDirectory(MULTIPART_UPLOAD_RESOURCE.getUploadDirectory().toFile());
+
+ Request fileRequest = buildFileRequest(fileHandler.getMessageHeaders().getTargetRestEndpointURL());
+ try (Response response = client.newCall(fileRequest).execute()) {
+ assertEquals(fileHandler.getMessageHeaders().getResponseStatusCode().code(), response.code());
+ }
+ }
+
@Test
public void testMixedMultipart() throws Exception {
OkHttpClient client = new OkHttpClient();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java
index 22de8a1dcde..65690c80b09 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/MultipartUploadResource.java
@@ -163,6 +163,10 @@ public MultipartJsonHandler getJsonHandler() {
return jsonHandler;
}
+ public Path getUploadDirectory() {
+ return configuredUploadDir;
+ }
+
public void resetState() {
mixedHandler.lastReceivedRequest = null;
jsonHandler.lastReceivedRequest = null;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java
index da8182a8ede..61c028ffab5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/metrics/MetricFetcherTest.java
@@ -99,7 +99,7 @@ public void testUpdate() throws Exception {
MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
when(jmQueryService.queryMetrics(any(Time.class)))
- .thenReturn(CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], 0, 0, 0, 0)));
+ .thenReturn(CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)));
when(tmQueryService.queryMetrics(any(Time.class)))
.thenReturn(CompletableFuture.completedFuture(requestMetricsAnswer));
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java
index c1f08e06b3c..55aacb23057 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java
@@ -251,7 +251,7 @@ private void assertEqualStateMetaInfoSnapshots(StateMetaInfoSnapshot expected, S
Assert.assertEquals(expected.getBackendStateType(), actual.getBackendStateType());
Assert.assertEquals(expected.getOptionsImmutable(), actual.getOptionsImmutable());
Assert.assertEquals(
- expected.getSerializerConfigSnapshotsImmutable(),
- actual.getSerializerConfigSnapshotsImmutable());
+ expected.getSerializerSnapshotsImmutable(),
+ actual.getSerializerSnapshotsImmutable());
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
index f5f30d5037a..5511792673e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
@@ -34,21 +34,20 @@
import org.apache.flink.runtime.checkpoint.StateObjectCollection;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
-import org.apache.flink.runtime.state.heap.HeapPriorityQueueElement;
+import org.apache.flink.runtime.testutils.statemigration.TestType;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.StateMigrationException;
import org.apache.flink.util.TestLogger;
+
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.Objects;
import java.util.concurrent.RunnableFuture;
/**
@@ -69,17 +68,6 @@
// lazily initialized stream storage
private CheckpointStorageLocation checkpointStorageLocation;
- /**
- * The compatibility behaviour of {@link TestSerializer}.
- * This controls what format the serializer writes in, as well as
- * the result of the compatibility check against the prior serializer snapshot.
- */
- public enum SerializerCompatibilityType {
- COMPATIBLE_AS_IS,
- REQUIRES_MIGRATION,
- INCOMPATIBLE
- }
-
// -------------------------------------------------------------------------------
// Keyed state backend migration tests
// -------------------------------------------------------------------------------
@@ -95,7 +83,7 @@ public void testKeyedValueStateMigration() throws Exception {
try {
ValueStateDescriptor<TestType> kvId = new ValueStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ValueState<TestType> valueState = backend
.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
@@ -113,10 +101,10 @@ public void testKeyedValueStateMigration() throws Exception {
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
- // the new serializer is REQUIRES_MIGRATION, and has a completely new serialization schema.
+ // the new serializer is V2, and has a completely new serialization schema.
kvId = new ValueStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION));
+ new TestType.V2TestTypeSerializer());
valueState = backend
.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
@@ -151,7 +139,7 @@ public void testKeyedListStateMigration() throws Exception {
try {
ListStateDescriptor<TestType> kvId = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ListState<TestType> listState = backend
.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
@@ -174,10 +162,10 @@ public void testKeyedListStateMigration() throws Exception {
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
- // the new serializer is REQUIRES_MIGRATION, and has a completely new serialization schema.
+ // the new serializer is V2, and has a completely new serialization schema.
kvId = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION));
+ new TestType.V2TestTypeSerializer());
listState = backend
.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
@@ -221,7 +209,7 @@ public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatib
try {
ValueStateDescriptor<TestType> kvId = new ValueStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ValueState<TestType> valueState = backend
.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
@@ -241,10 +229,12 @@ public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatib
kvId = new ValueStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE));
+ new TestType.IncompatibleTestTypeSerializer());
// the new serializer is INCOMPATIBLE, so registering the state should fail
backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
+
+ Assert.fail("should have failed");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
}finally {
@@ -263,7 +253,7 @@ public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatibl
try {
ListStateDescriptor<TestType> kvId = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ListState<TestType> listState = backend
.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
@@ -288,10 +278,12 @@ public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatibl
kvId = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION));
+ new TestType.IncompatibleTestTypeSerializer());
// the new serializer is INCOMPATIBLE, so registering the state should fail
backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
+
+ Assert.fail("should have failed");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
} finally {
@@ -308,7 +300,7 @@ public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible()
try {
InternalPriorityQueue<TestType> internalPriorityQueue = backend.create(
- "testPriorityQueue", new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ "testPriorityQueue", new TestType.V1TestTypeSerializer());
internalPriorityQueue.add(new TestType("key-1", 123));
internalPriorityQueue.add(new TestType("key-2", 346));
@@ -321,7 +313,7 @@ public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible()
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
backend.create(
- "testPriorityQueue", new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE));
+ "testPriorityQueue", new TestType.IncompatibleTestTypeSerializer());
Assert.fail("should have failed");
} catch (Exception e) {
@@ -337,7 +329,7 @@ public void testStateBackendCreationFailsIfNewKeySerializerIsNotCompatible() thr
SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
AbstractKeyedStateBackend<TestType> backend = createKeyedBackend(
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
final String stateName = "test-name";
try {
@@ -357,14 +349,18 @@ public void testStateBackendCreationFailsIfNewKeySerializerIsNotCompatible() thr
try {
// the new key serializer is incompatible; this should fail the restore
- restoreKeyedBackend(new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE), snapshot);
+ restoreKeyedBackend(new TestType.IncompatibleTestTypeSerializer(), snapshot);
+
+ Assert.fail("should have failed");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
}
try {
// the new key serializer requires migration; this should fail the restore
- restoreKeyedBackend(new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION), snapshot);
+ restoreKeyedBackend(new TestType.V2TestTypeSerializer(), snapshot);
+
+ Assert.fail("should have failed");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
}
@@ -386,7 +382,7 @@ public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsNotCompatib
ValueState<Integer> valueState = backend
.getPartitionedState(
new TestType("namespace", 123),
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS),
+ new TestType.V1TestTypeSerializer(),
kvId);
backend.setCurrentKey(1);
@@ -397,26 +393,33 @@ public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsNotCompatib
KeyedStateHandle snapshot = runSnapshot(
backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
sharedStateRegistry);
- backend.dispose();
+ // test incompatible namespace serializer; start with a freshly restored backend
+ backend.dispose();
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
-
try {
// the new namespace serializer is incompatible; this should fail the restore
backend.getPartitionedState(
new TestType("namespace", 123),
- new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE),
+ new TestType.IncompatibleTestTypeSerializer(),
kvId);
+
+ Assert.fail("should have failed");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
}
+ // test namespace serializer that requires migration; start with a freshly restored backend
+ backend.dispose();
+ backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
try {
// the new namespace serializer requires migration; this should fail the restore
backend.getPartitionedState(
new TestType("namespace", 123),
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION),
+ new TestType.V2TestTypeSerializer(),
kvId);
+
+ Assert.fail("should have failed");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
}
@@ -439,7 +442,7 @@ public void testOperatorParitionableListStateMigration() throws Exception {
try {
ListStateDescriptor<TestType> descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ListState<TestType> state = backend.getListState(descriptor);
state.add(new TestType("foo", 13));
@@ -453,7 +456,7 @@ public void testOperatorParitionableListStateMigration() throws Exception {
descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION));
+ new TestType.V2TestTypeSerializer());
state = backend.getListState(descriptor);
// the state backend should have decided whether or not it needs to perform state migration;
@@ -478,7 +481,7 @@ public void testUnionListStateMigration() throws Exception {
try {
ListStateDescriptor<TestType> descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ListState<TestType> state = backend.getUnionListState(descriptor);
state.add(new TestType("foo", 13));
@@ -492,7 +495,7 @@ public void testUnionListStateMigration() throws Exception {
descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION));
+ new TestType.V2TestTypeSerializer());
state = backend.getUnionListState(descriptor);
// the state backend should have decided whether or not it needs to perform state migration;
@@ -518,7 +521,7 @@ public void testBroadcastStateValueMigration() throws Exception {
MapStateDescriptor<Integer, TestType> descriptor = new MapStateDescriptor<>(
stateName,
IntSerializer.INSTANCE,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
BroadcastState<Integer, TestType> state = backend.getBroadcastState(descriptor);
state.put(3, new TestType("foo", 13));
@@ -533,7 +536,7 @@ public void testBroadcastStateValueMigration() throws Exception {
descriptor = new MapStateDescriptor<>(
stateName,
IntSerializer.INSTANCE,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION));
+ new TestType.V2TestTypeSerializer());
state = backend.getBroadcastState(descriptor);
// the state backend should have decided whether or not it needs to perform state migration;
@@ -556,7 +559,7 @@ public void testBroadcastStateKeyMigration() throws Exception {
try {
MapStateDescriptor<TestType, Integer> descriptor = new MapStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS),
+ new TestType.V1TestTypeSerializer(),
IntSerializer.INSTANCE);
BroadcastState<TestType, Integer> state = backend.getBroadcastState(descriptor);
@@ -571,7 +574,7 @@ public void testBroadcastStateKeyMigration() throws Exception {
descriptor = new MapStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION),
+ new TestType.V2TestTypeSerializer(),
IntSerializer.INSTANCE);
state = backend.getBroadcastState(descriptor);
@@ -595,7 +598,7 @@ public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsI
try {
ListStateDescriptor<TestType> descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ListState<TestType> state = backend.getListState(descriptor);
state.add(new TestType("foo", 13));
@@ -609,7 +612,7 @@ public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsI
descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE));
+ new TestType.IncompatibleTestTypeSerializer());
// the new serializer is INCOMPATIBLE, so registering the state should fail
backend.getListState(descriptor);
@@ -632,7 +635,7 @@ public void testUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() t
try {
ListStateDescriptor<TestType> descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
ListState<TestType> state = backend.getUnionListState(descriptor);
state.add(new TestType("foo", 13));
@@ -646,7 +649,7 @@ public void testUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() t
descriptor = new ListStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE));
+ new TestType.IncompatibleTestTypeSerializer());
// the new serializer is INCOMPATIBLE, so registering the state should fail
backend.getUnionListState(descriptor);
@@ -670,7 +673,7 @@ public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatibl
MapStateDescriptor<Integer, TestType> descriptor = new MapStateDescriptor<>(
stateName,
IntSerializer.INSTANCE,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS));
+ new TestType.V1TestTypeSerializer());
BroadcastState<Integer, TestType> state = backend.getBroadcastState(descriptor);
state.put(3, new TestType("foo", 13));
@@ -685,10 +688,12 @@ public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatibl
descriptor = new MapStateDescriptor<>(
stateName,
IntSerializer.INSTANCE,
- new TestSerializer(SerializerCompatibilityType.REQUIRES_MIGRATION));
+ new TestType.IncompatibleTestTypeSerializer());
// the new value serializer is INCOMPATIBLE, so registering the state should fail
backend.getBroadcastState(descriptor);
+
+ Assert.fail("should have failed.");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
} finally {
@@ -706,7 +711,7 @@ public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible(
try {
MapStateDescriptor<TestType, Integer> descriptor = new MapStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS),
+ new TestType.V1TestTypeSerializer(),
IntSerializer.INSTANCE);
BroadcastState<TestType, Integer> state = backend.getBroadcastState(descriptor);
@@ -721,11 +726,13 @@ public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible(
descriptor = new MapStateDescriptor<>(
stateName,
- new TestSerializer(SerializerCompatibilityType.INCOMPATIBLE),
+ new TestType.IncompatibleTestTypeSerializer(),
IntSerializer.INSTANCE);
// the new key serializer is INCOMPATIBLE, so registering the state should fail
backend.getBroadcastState(descriptor);
+
+ Assert.fail("should have failed.");
} catch (Exception e) {
Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
} finally {
@@ -737,223 +744,6 @@ public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible(
// Test types, serializers, and serializer snapshots
// -------------------------------------------------------------------------------
- /**
- * The type used as state under tests.
- *
- * <p>This is implemented so that the type can also be used as keyed priority queue state.
- */
- private static class TestType implements HeapPriorityQueueElement, PriorityComparable<TestType>, Keyed<String> {
-
- private int index;
-
- private final int value;
- private final String key;
-
- public TestType(String key, int value) {
- this.key = key;
- this.value = value;
- }
-
- @Override
- public String getKey() {
- return key;
- }
-
- @Override
- public int comparePriorityTo(@Nonnull TestType other) {
- return Integer.compare(value, other.value);
- }
-
- @Override
- public int getInternalIndex() {
- return index;
- }
-
- @Override
- public void setInternalIndex(int newIndex) {
- this.index = newIndex;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null || !(obj instanceof StateBackendMigrationTestBase.TestType)) {
- return false;
- }
-
- if (obj == this) {
- return true;
- }
-
- TestType other = (TestType) obj;
- return Objects.equals(key, other.key) && value == other.value;
- }
-
- @Override
- public int hashCode() {
- return 31 * key.hashCode() + value;
- }
- }
-
- private static class TestSerializer extends TypeSerializer<TestType> {
-
- private static final String MIGRATION_PAYLOAD = "random-migration-payload";
-
- private final SerializerCompatibilityType compatibilityType;
-
- TestSerializer(SerializerCompatibilityType compatibilityType) {
- this.compatibilityType = compatibilityType;
- }
-
- // --------------------------------------------------------------------------------
- // State serialization relevant methods
- // --------------------------------------------------------------------------------
-
- @Override
- public void serialize(TestType record, DataOutputView target) throws IOException {
- switch (compatibilityType) {
- case COMPATIBLE_AS_IS:
- target.writeUTF(record.getKey());
- target.writeInt(record.value);
- break;
-
- case REQUIRES_MIGRATION:
- target.writeUTF(record.getKey());
- target.writeUTF(MIGRATION_PAYLOAD);
- target.writeInt(record.value);
- target.writeBoolean(true);
- break;
-
- case INCOMPATIBLE:
- // the serializer shouldn't be used in this case
- throw new UnsupportedOperationException();
- }
- }
-
- @Override
- public TestType deserialize(DataInputView source) throws IOException {
- String key;
- int value;
-
- switch (compatibilityType) {
- case COMPATIBLE_AS_IS:
- key = source.readUTF();
- value = source.readInt();
- break;
-
- case REQUIRES_MIGRATION:
- key = source.readUTF();
- Assert.assertEquals(MIGRATION_PAYLOAD, source.readUTF());
- value = source.readInt();
- Assert.assertTrue(source.readBoolean());
- break;
-
- default:
- case INCOMPATIBLE:
- // the serializer shouldn't be used in this case
- throw new UnsupportedOperationException();
- }
-
- return new TestType(key, value);
- }
-
- @Override
- public TestType copy(TestType from) {
- return new TestType(from.key, from.value);
- }
-
- @Override
- public TypeSerializerSnapshot<TestType> snapshotConfiguration() {
- return new TestSerializerSnapshot();
- }
-
- // --------------------------------------------------------------------------------
- // Miscellaneous serializer methods
- // --------------------------------------------------------------------------------
-
- @Override
- public void copy(DataInputView source, DataOutputView target) throws IOException {
- serialize(deserialize(source), target);
- }
-
- @Override
- public TestType deserialize(TestType reuse, DataInputView source) throws IOException {
- return deserialize(source);
- }
-
- @Override
- public TestType copy(TestType from, TestType reuse) {
- return copy(from);
- }
-
- @Override
- public TestType createInstance() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TypeSerializer<TestType> duplicate() {
- return this;
- }
-
- @Override
- public boolean isImmutableType() {
- return false;
- }
-
- @Override
- public int getLength() {
- return -1;
- }
-
- @Override
- public boolean canEqual(Object obj) {
- return getClass().equals(obj.getClass());
- }
-
- @Override
- public int hashCode() {
- return getClass().hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- return obj == this;
- }
- }
-
- public static class TestSerializerSnapshot implements TypeSerializerSnapshot<TestType> {
-
- @Override
- public int getCurrentVersion() {
- return 1;
- }
-
- @Override
- public TypeSerializer<TestType> restoreSerializer() {
- return new TestSerializer(SerializerCompatibilityType.COMPATIBLE_AS_IS);
- }
-
- @Override
(This diff was longer than 20,000 lines, and has been truncated...)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services