You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by "Yao Zhang (Jira)" <ji...@apache.org> on 2021/09/14 08:25:00 UTC
[jira] [Comment Edited] (FLINK-24224) Table change to stream for CEP , only event is ROW datatype can works on CEP, but other POJOs、maps、 JsonObject datatype event do not work. Any datatype for event to CEP is OK by only stream api
[ https://issues.apache.org/jira/browse/FLINK-24224?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17414802#comment-17414802 ]
Yao Zhang edited comment on FLINK-24224 at 9/14/21, 8:24 AM:
-------------------------------------------------------------
Hi [~YUJIANBO],
I failed to reproduce your issue.
I posted my codes in the block below:
{code:java}
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.cep.CEP;
import org.apache.flink.cep.PatternSelectFunction;
import org.apache.flink.cep.PatternStream;
import org.apache.flink.cep.functions.PatternProcessFunction;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List;
import java.util.Map;
public class CEPProblem {
public static void main(String[] args) throws Exception {
// main1();
main2();
}
// Row data type
public static void main1() throws Exception {
StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
environment.setParallelism(1);
StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);
tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");
TableResult tableResult = tEnv.executeSql("select * from temperature");
tableResult.print();
DataStream<Row> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class);
SingleOutputStreamOperator<Row> ts = temperatureStream.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Row>(Time.seconds(1)) {
@Override
public long extractTimestamp(Row element) {
return ((LocalDateTime) element.getField("ts")).toEpochSecond(ZoneOffset.ofHours(8));
}
});
Pattern<Row, Row> pattern = Pattern.<Row>begin("high").where(new IterativeCondition<Row>() {
@Override
public boolean filter(Row row, Context<Row> context) throws Exception {
int temp = (int) row.getField("temp");
return temp >= 50;
}
}).timesOrMore(3).within(Time.seconds(5));
PatternStream<Row> patternStream = CEP.pattern(ts, pattern);
patternStream.process(new PatternProcessFunction<Row, Object>() {
@Override
public void processMatch(Map<String, List<Row>> map, Context context, Collector<Object> collector) throws Exception {
System.out.println(map);
}
});
environment.execute();
}
// Pojo Data type
public static void main2() throws Exception {
StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
environment.setParallelism(1);
StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);
tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");
TableResult tableResult = tEnv.executeSql("select * from temperature");
tableResult.print();
DataStream<Event> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class).map(new MapFunction<Row, Event>() {
@Override
public Event map(Row row) throws Exception {
return new Event((LocalDateTime) row.getField("ts"), (int) row.getField("temp"));
}
}).assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Event>(Time.seconds(1)) {
@Override
public long extractTimestamp(Event element) {
return element.getTs().toEpochSecond(ZoneOffset.ofHours(8));
}
});
Pattern<Event, Event> pattern = Pattern.<Event>begin("high").where(new IterativeCondition<Event>() {
@Override
public boolean filter(Event row, Context<Event> context) throws Exception {
return row.getTemp() >= 50;
}
}).timesOrMore(3).within(Time.seconds(5));
PatternStream<Event> patternStream = CEP.pattern(temperatureStream, pattern);
patternStream.process(new PatternProcessFunction<Event, Object>() {
@Override
public void processMatch(Map<String, List<Event>> map, Context context, Collector<Object> collector) throws Exception {
System.out.println(map);
}
});
environment.execute();
}
}
class Event {
private LocalDateTime ts;
private int temp;
public Event() {
}
public Event(LocalDateTime ts, int temp) {
this.ts = ts;
this.temp = temp;
}
public LocalDateTime getTs() {
return ts;
}
public void setTs(LocalDateTime ts) {
this.ts = ts;
}
public int getTemp() {
return temp;
}
public void setTemp(int temp) {
this.temp = temp;
}
@Override
public String toString() {
return "Event{" +
"ts=" + ts +
", temp=" + temp +
'}';
}
}
{code}
The content of file /path/to/temperature.txt is:
{noformat}
2021-09-14 15:37:03.3,30
2021-09-14 15:37:13.3,50
2021-09-14 15:37:23.3,55
2021-09-14 15:37:33.3,60
2021-09-14 15:37:43.3,55
2021-09-14 15:37:53.3,50
2021-09-14 15:38:03.3,45
{noformat}
No matter I ran main1 or main2, CEP will always print the correct output.
Could you please post all your codes here with sample data and your configurations?
was (Author: paul8263):
Hi [~YUJIANBO],
I failed to reproduce your issue.
I posted my codes in the block below:
{code:java}
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.cep.CEP;
import org.apache.flink.cep.PatternSelectFunction;
import org.apache.flink.cep.PatternStream;
import org.apache.flink.cep.functions.PatternProcessFunction;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List;
import java.util.Map;
public class CEPProblem2 {
public static void main(String[] args) throws Exception {
// main1();
main2();
}
// Row data type
public static void main1() throws Exception {
StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
environment.setParallelism(1);
StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);
tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");
TableResult tableResult = tEnv.executeSql("select * from temperature");
tableResult.print();
DataStream<Row> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class);
SingleOutputStreamOperator<Row> ts = temperatureStream.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Row>(Time.seconds(1)) {
@Override
public long extractTimestamp(Row element) {
return ((LocalDateTime) element.getField("ts")).toEpochSecond(ZoneOffset.ofHours(8));
}
});
Pattern<Row, Row> pattern = Pattern.<Row>begin("high").where(new IterativeCondition<Row>() {
@Override
public boolean filter(Row row, Context<Row> context) throws Exception {
int temp = (int) row.getField("temp");
return temp >= 50;
}
}).timesOrMore(3).within(Time.seconds(5));
PatternStream<Row> patternStream = CEP.pattern(ts, pattern);
patternStream.process(new PatternProcessFunction<Row, Object>() {
@Override
public void processMatch(Map<String, List<Row>> map, Context context, Collector<Object> collector) throws Exception {
System.out.println(map);
}
});
environment.execute();
}
// Pojo Data type
public static void main2() throws Exception {
StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
environment.setParallelism(1);
StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);
tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");
TableResult tableResult = tEnv.executeSql("select * from temperature");
tableResult.print();
DataStream<Event> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class).map(new MapFunction<Row, Event>() {
@Override
public Event map(Row row) throws Exception {
return new Event((LocalDateTime) row.getField("ts"), (int) row.getField("temp"));
}
}).assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Event>(Time.seconds(1)) {
@Override
public long extractTimestamp(Event element) {
return element.getTs().toEpochSecond(ZoneOffset.ofHours(8));
}
});
Pattern<Event, Event> pattern = Pattern.<Event>begin("high").where(new IterativeCondition<Event>() {
@Override
public boolean filter(Event row, Context<Event> context) throws Exception {
return row.getTemp() >= 50;
}
}).timesOrMore(3).within(Time.seconds(5));
PatternStream<Event> patternStream = CEP.pattern(temperatureStream, pattern);
patternStream.process(new PatternProcessFunction<Event, Object>() {
@Override
public void processMatch(Map<String, List<Event>> map, Context context, Collector<Object> collector) throws Exception {
System.out.println(map);
}
});
environment.execute();
}
}
class Event {
private LocalDateTime ts;
private int temp;
public Event() {
}
public Event(LocalDateTime ts, int temp) {
this.ts = ts;
this.temp = temp;
}
public LocalDateTime getTs() {
return ts;
}
public void setTs(LocalDateTime ts) {
this.ts = ts;
}
public int getTemp() {
return temp;
}
public void setTemp(int temp) {
this.temp = temp;
}
@Override
public String toString() {
return "Event{" +
"ts=" + ts +
", temp=" + temp +
'}';
}
}
{code}
The content of file /path/to/temperature.txt is:
{noformat}
2021-09-14 15:37:03.3,30
2021-09-14 15:37:13.3,50
2021-09-14 15:37:23.3,55
2021-09-14 15:37:33.3,60
2021-09-14 15:37:43.3,55
2021-09-14 15:37:53.3,50
2021-09-14 15:38:03.3,45
{noformat}
No matter I ran main1 or main2, CEP will always print the correct output.
Could you please post all your codes here with sample data and your configurations?
> Table change to stream for CEP , only event is ROW datatype can works on CEP, but other POJOs、maps、 JsonObject datatype event do not work. Any datatype for event to CEP is OK by only stream api
> -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: FLINK-24224
> URL: https://issues.apache.org/jira/browse/FLINK-24224
> Project: Flink
> Issue Type: Bug
> Components: Library / CEP
> Affects Versions: 1.11.2, 1.12.0, 1.13.2
> Reporter: YUJIANBO
> Priority: Major
>
>
> 1、problem:*Table change to stream for CEP* , only event is *ROW* datatype *can works* on CEP, but *othe*r POJOs、maps、 JsonObject datatype event *do not work*.
> *Any datatype for event to CEP is OK by only stream api*
> 2、version: I have tried 3 versions,such as 1.11.2、1.12.0、1.13.2
> 3、code:
> (1)table to Stream to CEP (only row datatype is ok, other datatype Stream to CEP has no data print and it has no error message)
> {code:java}
> tableEnv.executeSql(creat_kafka_source);
> tableEnv.executeSql(calculateSql);
> Table tb = tableEnv.from("calculateSql");
> String[] fieldNames = tb.getSchema().getFieldNames();
> DataType[] fieldDataTypes = tb.getSchema().getFieldDataTypes();
> KeyedStream<JSONObject, String> ds = tableEnv
> .toAppendStream(tb, Row.class)
> .map(new RichMapFunction<Row, JSONObject>() {
> Map<String, Object> map = new HashMap<>();
> @Override
> public void open(Configuration parameters) throws Exception {
> super.open(parameters);
> if (null == map) {
> map = new HashMap<>();
> }
> }
> @Override
> public JSONObject map(Row value) throws Exception {
> //将数据key和value添加到map中
> RowParseUtil.setFieldAndValue2Map(map, fieldDataTypes, fieldNames, value);
> JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(map));
> map.clear();
> return jsonObject;
> }
> })
> .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<JSONObject>(Time.seconds(0)) {
> @Override
> public long extractTimestamp(JSONObject element) {
> return element.getLongValue("wStart") * 1000;
> }
> }).keyBy(x -> x.getString("x_forwarded_for"));
> //it has data to print
> ds.print();
> Pattern<JSONObject, JSONObject> pattern = Pattern.<JSONObject>begin("begin")
> .where(new SimpleCondition<JSONObject>() {
> @Override
> public boolean filter(JSONObject value) throws Exception {
> log.info("===================>" + value);
> return true;
> }
> }).timesOrMore(1).within(Time.seconds(10));
> PatternStream<JSONObject> patternStream = CEP.pattern(ds, pattern);
> //it has no data to print
> patternStream.process(new PatternProcessFunction<JSONObject, String>() {
> @Override
> public void processMatch(Map<String, List<JSONObject>> match, Context ctx, Collector<String> out) throws Exception {
> out.collect("==========>>>>>>>" + match.toString());
> }
> }).print();
> {code}
> (2) *Olny Stream API to CEP* ( Any datatype , it is OK)
> {code:java}
> Properties proPs = kafkaUtil.getReceiveKfkProPs(receive_brokers, groupid);
> FlinkKafkaConsumer<ConsumerRecord<String, String>> consumer =
> new FlinkKafkaConsumer<>(receive_topic, new KafkaRecordSchema(), proPs);
> consumer.setStartFromEarliest();
> SingleOutputStreamOperator<JSONObject> input = env.addSource(consumer)
> .map(x -> {
> return JSON.parseObject(x.value());
> })
> .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<JSONObject>(Time.milliseconds(10)) {
> @Override
> public long extractTimestamp(JSONObject element) {
> return element.getLongValue("ts");
> }
> })
> .keyBy(x -> x.getString("x_forwarded_for")+x.getString("request_uri"))
> .timeWindow(Time.seconds(1)).apply(new WindowFunction<JSONObject, JSONObject, String, TimeWindow>() {
> @Override
> public void apply(String s, TimeWindow window, Iterable<JSONObject> input, Collector<JSONObject> out) throws Exception {
> Iterator<JSONObject> iterator = input.iterator();
> ArrayList<JSONObject> list = new ArrayList<>();
> int n = 0;
> while (iterator.hasNext()) {
> n++;
> JSONObject next = iterator.next();
> list.add(next);
> }
> JSONObject jsonObject = list.get(0);
> jsonObject.put("ct",n);
> jsonObject.remove("ts");
> out.collect(jsonObject);
> }
> });
> input.print();
> //it is ok
> Pattern<JSONObject, JSONObject> minInterval = Pattern
> .<JSONObject>begin("begin").where(new SimpleCondition<JSONObject>() {
> @Override
> public boolean filter(JSONObject jsonObject) throws Exception {
> return true;
> }
> }).timesOrMore(1).within(Time.seconds(10));
> PatternStream<JSONObject> pattern = CEP.pattern(input, minInterval);
> pattern.process(new PatternProcessFunction<JSONObject, String>() {
> @Override
> public void processMatch(Map<String, List<JSONObject>> map, Context context, Collector<String> out) throws Exception {
> out.collect("这个用户有嫌疑:====================>" + map.toString());
> }
> }).print();
> {code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)