You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/09 04:49:00 UTC

[3/3] incubator-kylin git commit: KYLIN-1018 enhance and make TimedJsonStreamParser default parser

KYLIN-1018 enhance and make TimedJsonStreamParser default parser


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/02178b1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/02178b1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/02178b1c

Branch: refs/heads/2.x-staging
Commit: 02178b1cac3694bc7b9f983423f9ec89cf49c38d
Parents: 70c700d
Author: honma <ho...@ebay.com>
Authored: Tue Sep 8 22:04:55 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Sep 9 10:50:52 2015 +0800

----------------------------------------------------------------------
 .../kylin/job/streaming/StreamingBootstrap.java |  4 +-
 .../java/org/apache/kylin/job/DeployUtil.java   |  3 +-
 .../apache/kylin/streaming/KafkaRequester.java  |  3 +-
 .../apache/kylin/streaming/StreamingConfig.java | 12 ++++++
 .../kylin/streaming/TimedJsonStreamParser.java  | 41 ++++++++++++++------
 5 files changed, 47 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index b7a8335..da7b20a 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -252,8 +252,8 @@ public class StreamingBootstrap {
     private StreamParser getStreamParser(StreamingConfig streamingConfig, List<TblColRef> columns) throws Exception {
         if (!StringUtils.isEmpty(streamingConfig.getParserName())) {
             Class clazz = Class.forName(streamingConfig.getParserName());
-            Constructor constructor = clazz.getConstructor(List.class);
-            return (StreamParser) constructor.newInstance(columns);
+            Constructor constructor = clazz.getConstructor(List.class, String.class);
+            return (StreamParser) constructor.newInstance(columns, streamingConfig.getParserProperties());
         } else {
             return new JsonStreamParser(columns);
         }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 351e176..8c92f87 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -154,13 +154,14 @@ public class DeployUtil {
 
         //load into kafka
         KafkaDataLoader.loadIntoKafka(streamingConfig, data);
+        logger.info("Write {} messages into topic {}", data.size(), streamingConfig.getTopic());
 
         //csv data for H2 use
         List<TblColRef> tableColumns = Lists.newArrayList();
         for (ColumnDesc columnDesc : tableDesc.getColumns()) {
             tableColumns.add(new TblColRef(columnDesc));
         }
-        TimedJsonStreamParser timedJsonStreamParser = new TimedJsonStreamParser(tableColumns, true);
+        TimedJsonStreamParser timedJsonStreamParser = new TimedJsonStreamParser(tableColumns, "formatTs=true");
         StringBuilder sb = new StringBuilder();
         for (String json : data) {
             List<String> rowColumns = timedJsonStreamParser.parse(new StreamMessage(0, json.getBytes())).getStreamMessage();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
index 4af3bf4..218fe6e 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
@@ -116,8 +116,7 @@ public final class KafkaRequester {
             });
             return new TopicMeta(kafkaClusterConfig.getTopic(), partitionIds);
         }
-        logger.debug("cannot find topic:" + kafkaClusterConfig.getTopic());
-        return null;
+        throw new IllegalArgumentException("cannot find topic:" + kafkaClusterConfig.getTopic());
     }
 
     public static PartitionMetadata getPartitionMetadata(String topic, int partitionId, List<Broker> brokers, KafkaClusterConfig kafkaClusterConfig) {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
index 6a3e4fc..320768b 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
@@ -83,6 +83,10 @@ public class StreamingConfig extends RootPersistentEntity {
 
     @JsonProperty("parserName")
     private String parserName;
+    
+    //"configA=1;configB=2"
+    @JsonProperty("parserProperties")
+    private String parserProperties;
 
     @JsonProperty("margin")
     private long margin;
@@ -163,6 +167,14 @@ public class StreamingConfig extends RootPersistentEntity {
         this.margin = margin;
     }
 
+    public String getParserProperties() {
+        return parserProperties;
+    }
+
+    public void setParserProperties(String parserProperties) {
+        this.parserProperties = parserProperties;
+    }
+
     @Override
     public StreamingConfig clone() {
         try {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
index b7ea34e..6c4146f 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
@@ -61,29 +61,48 @@ public final class TimedJsonStreamParser implements StreamParser {
 
     private static final Logger logger = LoggerFactory.getLogger(TimedJsonStreamParser.class);
 
-    private final List<TblColRef> allColumns;
-    private boolean formatTs;
+    private List<TblColRef> allColumns;
+    private boolean formatTs = false;
     private final ObjectMapper mapper = new ObjectMapper();
+    private String tsColName = "timestamp";
     private final JavaType mapType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
 
-    @SuppressWarnings("unused")
-    public TimedJsonStreamParser(List<TblColRef> allColumns) {
+    public TimedJsonStreamParser(List<TblColRef> allColumns, String propertiesStr) {
         this.allColumns = allColumns;
-        this.formatTs = false;
-    }
+        if (!StringUtils.isEmpty(propertiesStr)) {
+            String[] properties = propertiesStr.split(";");
+            for (String prop : properties) {
+                try {
+                    String[] parts = prop.split("=");
+                    if (parts.length == 2) {
+                        switch (parts[0]) {
+                        case "formatTs":
+                            this.formatTs = Boolean.valueOf(parts[1]);
+                            break;
+                        case "tsColName":
+                            this.tsColName = parts[1];
+                            break;  
+                        default:
+                            break;
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.error("Failed to parse property " + prop);
+                    //ignore
+                }
+            }
+        }
 
-    public TimedJsonStreamParser(List<TblColRef> allColumns, boolean formatTs) {
-        this.allColumns = allColumns;
-        this.formatTs = formatTs;
+        logger.info("TimedJsonStreamParser with formatTs {} tsColName {}", formatTs, tsColName);
     }
 
     @Override
     public ParsedStreamMessage parse(StreamMessage stream) {
         try {
             Map<String, String> root = mapper.readValue(stream.getRawData(), mapType);
-            String tsStr = root.get("timestamp");
+            String tsStr = root.get(tsColName);
             Preconditions.checkArgument(!StringUtils.isEmpty(tsStr), "Timestamp field cannot be null");
-            long t = Long.valueOf(root.get("timestamp"));
+            long t = Long.valueOf(root.get(tsColName));
             ArrayList<String> result = Lists.newArrayList();
 
             for (TblColRef column : allColumns) {