You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by "Yao Zhang (Jira)" <ji...@apache.org> on 2021/09/14 08:25:00 UTC

[jira] [Comment Edited] (FLINK-24224) Table change to stream for CEP , only event  is ROW  datatype can works on CEP,  but  other POJOs、maps、 JsonObject datatype event  do not work.    Any datatype for event to CEP  is OK by  only stream api 

    [ https://issues.apache.org/jira/browse/FLINK-24224?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17414802#comment-17414802 ] 

Yao Zhang edited comment on FLINK-24224 at 9/14/21, 8:24 AM:
-------------------------------------------------------------

Hi [~YUJIANBO],

I failed to reproduce your issue.

I posted my codes in the block below:
{code:java}
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.cep.CEP;
import org.apache.flink.cep.PatternSelectFunction;
import org.apache.flink.cep.PatternStream;
import org.apache.flink.cep.functions.PatternProcessFunction;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;

import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List;
import java.util.Map;

public class CEPProblem {
    public static void main(String[] args) throws Exception {
//      main1();
        main2();
    }

    // Row data type
    public static void main1() throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
        environment.setParallelism(1);
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);

        tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");

        TableResult tableResult = tEnv.executeSql("select * from temperature");
        tableResult.print();

        DataStream<Row> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class);

        SingleOutputStreamOperator<Row> ts = temperatureStream.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Row>(Time.seconds(1)) {
            @Override
            public long extractTimestamp(Row element) {
                return ((LocalDateTime) element.getField("ts")).toEpochSecond(ZoneOffset.ofHours(8));
            }
        });

        Pattern<Row, Row> pattern = Pattern.<Row>begin("high").where(new IterativeCondition<Row>() {
            @Override
            public boolean filter(Row row, Context<Row> context) throws Exception {
                int temp = (int) row.getField("temp");
                return temp >= 50;
            }
        }).timesOrMore(3).within(Time.seconds(5));

        PatternStream<Row> patternStream = CEP.pattern(ts, pattern);

        patternStream.process(new PatternProcessFunction<Row, Object>() {
            @Override
            public void processMatch(Map<String, List<Row>> map, Context context, Collector<Object> collector) throws Exception {
                System.out.println(map);
            }
        });

        environment.execute();
    }

    // Pojo Data type
    public static void main2() throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
        environment.setParallelism(1);
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);

        tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");

        TableResult tableResult = tEnv.executeSql("select * from temperature");
        tableResult.print();

        DataStream<Event> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class).map(new MapFunction<Row, Event>() {
            @Override
            public Event map(Row row) throws Exception {
                return new Event((LocalDateTime) row.getField("ts"), (int) row.getField("temp"));
            }
        }).assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Event>(Time.seconds(1)) {
            @Override
            public long extractTimestamp(Event element) {
                return element.getTs().toEpochSecond(ZoneOffset.ofHours(8));
            }
        });

        Pattern<Event, Event> pattern = Pattern.<Event>begin("high").where(new IterativeCondition<Event>() {
            @Override
            public boolean filter(Event row, Context<Event> context) throws Exception {
                return row.getTemp() >= 50;
            }
        }).timesOrMore(3).within(Time.seconds(5));

        PatternStream<Event> patternStream = CEP.pattern(temperatureStream, pattern);

        patternStream.process(new PatternProcessFunction<Event, Object>() {
            @Override
            public void processMatch(Map<String, List<Event>> map, Context context, Collector<Object> collector) throws Exception {
                System.out.println(map);
            }
        });

        environment.execute();
    }
}

class Event {
    private LocalDateTime ts;
    private int temp;

    public Event() {
    }

    public Event(LocalDateTime ts, int temp) {
        this.ts = ts;
        this.temp = temp;
    }

    public LocalDateTime getTs() {
        return ts;
    }

    public void setTs(LocalDateTime ts) {
        this.ts = ts;
    }

    public int getTemp() {
        return temp;
    }

    public void setTemp(int temp) {
        this.temp = temp;
    }

    @Override
    public String toString() {
        return "Event{" +
                "ts=" + ts +
                ", temp=" + temp +
                '}';
    }
}
{code}
The content of file /path/to/temperature.txt is:
{noformat}
2021-09-14 15:37:03.3,30
2021-09-14 15:37:13.3,50
2021-09-14 15:37:23.3,55
2021-09-14 15:37:33.3,60
2021-09-14 15:37:43.3,55
2021-09-14 15:37:53.3,50
2021-09-14 15:38:03.3,45
{noformat}
No matter I ran main1 or main2, CEP will always print the correct output.

Could you please post all your codes here with sample data and your configurations?


was (Author: paul8263):
Hi [~YUJIANBO],

I failed to reproduce your issue.

I posted my codes in the block below:
{code:java}
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.cep.CEP;
import org.apache.flink.cep.PatternSelectFunction;
import org.apache.flink.cep.PatternStream;
import org.apache.flink.cep.functions.PatternProcessFunction;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;

import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List;
import java.util.Map;

public class CEPProblem2 {
    public static void main(String[] args) throws Exception {
//      main1();
        main2();
    }

    // Row data type
    public static void main1() throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
        environment.setParallelism(1);
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);

        tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");

        TableResult tableResult = tEnv.executeSql("select * from temperature");
        tableResult.print();

        DataStream<Row> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class);

        SingleOutputStreamOperator<Row> ts = temperatureStream.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Row>(Time.seconds(1)) {
            @Override
            public long extractTimestamp(Row element) {
                return ((LocalDateTime) element.getField("ts")).toEpochSecond(ZoneOffset.ofHours(8));
            }
        });

        Pattern<Row, Row> pattern = Pattern.<Row>begin("high").where(new IterativeCondition<Row>() {
            @Override
            public boolean filter(Row row, Context<Row> context) throws Exception {
                int temp = (int) row.getField("temp");
                return temp >= 50;
            }
        }).timesOrMore(3).within(Time.seconds(5));

        PatternStream<Row> patternStream = CEP.pattern(ts, pattern);

        patternStream.process(new PatternProcessFunction<Row, Object>() {
            @Override
            public void processMatch(Map<String, List<Row>> map, Context context, Collector<Object> collector) throws Exception {
                System.out.println(map);
            }
        });

        environment.execute();
    }

    // Pojo Data type
    public static void main2() throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
        environment.setParallelism(1);
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(environment);

        tEnv.executeSql("create table temperature (ts timestamp(3), temp int) with ('connector'='filesystem', 'path'='/path/to/temperature.txt', 'format'='csv')");

        TableResult tableResult = tEnv.executeSql("select * from temperature");
        tableResult.print();

        DataStream<Event> temperatureStream = tEnv.toAppendStream(tEnv.from("temperature"), Row.class).map(new MapFunction<Row, Event>() {
            @Override
            public Event map(Row row) throws Exception {
                return new Event((LocalDateTime) row.getField("ts"), (int) row.getField("temp"));
            }
        }).assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Event>(Time.seconds(1)) {
            @Override
            public long extractTimestamp(Event element) {
                return element.getTs().toEpochSecond(ZoneOffset.ofHours(8));
            }
        });

        Pattern<Event, Event> pattern = Pattern.<Event>begin("high").where(new IterativeCondition<Event>() {
            @Override
            public boolean filter(Event row, Context<Event> context) throws Exception {
                return row.getTemp() >= 50;
            }
        }).timesOrMore(3).within(Time.seconds(5));

        PatternStream<Event> patternStream = CEP.pattern(temperatureStream, pattern);

        patternStream.process(new PatternProcessFunction<Event, Object>() {
            @Override
            public void processMatch(Map<String, List<Event>> map, Context context, Collector<Object> collector) throws Exception {
                System.out.println(map);
            }
        });

        environment.execute();
    }
}

class Event {
    private LocalDateTime ts;
    private int temp;

    public Event() {
    }

    public Event(LocalDateTime ts, int temp) {
        this.ts = ts;
        this.temp = temp;
    }

    public LocalDateTime getTs() {
        return ts;
    }

    public void setTs(LocalDateTime ts) {
        this.ts = ts;
    }

    public int getTemp() {
        return temp;
    }

    public void setTemp(int temp) {
        this.temp = temp;
    }

    @Override
    public String toString() {
        return "Event{" +
                "ts=" + ts +
                ", temp=" + temp +
                '}';
    }
}
{code}
The content of file /path/to/temperature.txt is:
{noformat}
2021-09-14 15:37:03.3,30
2021-09-14 15:37:13.3,50
2021-09-14 15:37:23.3,55
2021-09-14 15:37:33.3,60
2021-09-14 15:37:43.3,55
2021-09-14 15:37:53.3,50
2021-09-14 15:38:03.3,45
{noformat}
No matter I ran main1 or main2, CEP will always print the correct output.

Could you please post all your codes here with sample data and your configurations?

> Table change to stream for CEP , only event  is ROW  datatype can works on CEP,  but  other POJOs、maps、 JsonObject datatype event  do not work.     Any datatype for event to CEP  is OK by  only stream api 
> -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
>                 Key: FLINK-24224
>                 URL: https://issues.apache.org/jira/browse/FLINK-24224
>             Project: Flink
>          Issue Type: Bug
>          Components: Library / CEP
>    Affects Versions: 1.11.2, 1.12.0, 1.13.2
>            Reporter: YUJIANBO
>            Priority: Major
>
>  
> 1、problem:*Table change to stream for CEP* , only event  is *ROW*  datatype *can works* on CEP,  but  *othe*r POJOs、maps、 JsonObject datatype event  *do not work*.
>                        *Any datatype for event to CEP  is OK by  only stream api* 
> 2、version: I have tried 3 versions,such as 1.11.2、1.12.0、1.13.2
> 3、code:
> (1)table to Stream  to  CEP   (only row datatype is ok,  other datatype Stream to CEP has no data print and it has no error message)
> {code:java}
> tableEnv.executeSql(creat_kafka_source);
> tableEnv.executeSql(calculateSql);
> Table tb = tableEnv.from("calculateSql");
> String[] fieldNames = tb.getSchema().getFieldNames();
> DataType[] fieldDataTypes = tb.getSchema().getFieldDataTypes();
> KeyedStream<JSONObject, String> ds = tableEnv
>         .toAppendStream(tb, Row.class)
>         .map(new RichMapFunction<Row, JSONObject>() {
>             Map<String, Object> map = new HashMap<>();
>             @Override
>             public void open(Configuration parameters) throws Exception {
>                 super.open(parameters);
>                 if (null == map) {
>                     map = new HashMap<>();
>                 }
>             }
>             @Override
>             public JSONObject map(Row value) throws Exception {
>                 //将数据key和value添加到map中
>                 RowParseUtil.setFieldAndValue2Map(map, fieldDataTypes, fieldNames, value);
>                 JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(map));
>                 map.clear();
>                 return jsonObject;
>             }
>         })
>         .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<JSONObject>(Time.seconds(0)) {
>             @Override
>             public long extractTimestamp(JSONObject element) {
>                 return element.getLongValue("wStart") * 1000;
>             }
>         }).keyBy(x -> x.getString("x_forwarded_for"));
> //it has data to print
> ds.print();
> Pattern<JSONObject, JSONObject> pattern = Pattern.<JSONObject>begin("begin")
>         .where(new SimpleCondition<JSONObject>() {
>             @Override
>             public boolean filter(JSONObject value) throws Exception {
>                 log.info("===================>" + value);
>                 return true;
>             }
>         }).timesOrMore(1).within(Time.seconds(10));
> PatternStream<JSONObject> patternStream = CEP.pattern(ds, pattern);
> //it has no data to print
> patternStream.process(new PatternProcessFunction<JSONObject, String>() {
>     @Override
>     public void processMatch(Map<String, List<JSONObject>> match, Context ctx, Collector<String> out) throws Exception {
>         out.collect("==========>>>>>>>" + match.toString());
>     }
> }).print();
> {code}
> (2) *Olny Stream API  to CEP* ( Any datatype ,  it is OK)
> {code:java}
> Properties proPs = kafkaUtil.getReceiveKfkProPs(receive_brokers, groupid);
> FlinkKafkaConsumer<ConsumerRecord<String, String>> consumer =
>         new FlinkKafkaConsumer<>(receive_topic, new KafkaRecordSchema(), proPs);
> consumer.setStartFromEarliest();
> SingleOutputStreamOperator<JSONObject> input = env.addSource(consumer)
>         .map(x -> {
>             return JSON.parseObject(x.value());
>         })
>         .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<JSONObject>(Time.milliseconds(10)) {
>             @Override
>             public long extractTimestamp(JSONObject element) {
>                 return element.getLongValue("ts");
>             }
>         })
>         .keyBy(x -> x.getString("x_forwarded_for")+x.getString("request_uri"))
>         .timeWindow(Time.seconds(1)).apply(new WindowFunction<JSONObject, JSONObject, String, TimeWindow>() {
>             @Override
>             public void apply(String s, TimeWindow window, Iterable<JSONObject> input, Collector<JSONObject> out) throws Exception {
>                 Iterator<JSONObject> iterator = input.iterator();
>                 ArrayList<JSONObject> list = new ArrayList<>();
>                 int n = 0;
>                 while (iterator.hasNext()) {
>                     n++;
>                     JSONObject next = iterator.next();
>                     list.add(next);
>                 }
>                 JSONObject jsonObject = list.get(0);
>                 jsonObject.put("ct",n);
>                 jsonObject.remove("ts");
>                 out.collect(jsonObject);
>             }
>         });
> input.print();
> //it is ok
> Pattern<JSONObject, JSONObject> minInterval = Pattern
>         .<JSONObject>begin("begin").where(new SimpleCondition<JSONObject>() {
>             @Override
>             public boolean filter(JSONObject jsonObject) throws Exception {
>                 return true;
>             }
>         }).timesOrMore(1).within(Time.seconds(10));
> PatternStream<JSONObject> pattern = CEP.pattern(input, minInterval);
> pattern.process(new PatternProcessFunction<JSONObject, String>() {
>     @Override
>     public void processMatch(Map<String, List<JSONObject>> map, Context context, Collector<String> out) throws Exception {
>         out.collect("这个用户有嫌疑:====================>" + map.toString());
>     }
> }).print();
> {code}
>  



--
This message was sent by Atlassian Jira
(v8.3.4#803005)