You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/09 04:49:00 UTC
[3/3] incubator-kylin git commit: KYLIN-1018 enhance and make
TimedJsonStreamParser default parser
KYLIN-1018 enhance and make TimedJsonStreamParser default parser
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/02178b1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/02178b1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/02178b1c
Branch: refs/heads/2.x-staging
Commit: 02178b1cac3694bc7b9f983423f9ec89cf49c38d
Parents: 70c700d
Author: honma <ho...@ebay.com>
Authored: Tue Sep 8 22:04:55 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Sep 9 10:50:52 2015 +0800
----------------------------------------------------------------------
.../kylin/job/streaming/StreamingBootstrap.java | 4 +-
.../java/org/apache/kylin/job/DeployUtil.java | 3 +-
.../apache/kylin/streaming/KafkaRequester.java | 3 +-
.../apache/kylin/streaming/StreamingConfig.java | 12 ++++++
.../kylin/streaming/TimedJsonStreamParser.java | 41 ++++++++++++++------
5 files changed, 47 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index b7a8335..da7b20a 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -252,8 +252,8 @@ public class StreamingBootstrap {
private StreamParser getStreamParser(StreamingConfig streamingConfig, List<TblColRef> columns) throws Exception {
if (!StringUtils.isEmpty(streamingConfig.getParserName())) {
Class clazz = Class.forName(streamingConfig.getParserName());
- Constructor constructor = clazz.getConstructor(List.class);
- return (StreamParser) constructor.newInstance(columns);
+ Constructor constructor = clazz.getConstructor(List.class, String.class);
+ return (StreamParser) constructor.newInstance(columns, streamingConfig.getParserProperties());
} else {
return new JsonStreamParser(columns);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 351e176..8c92f87 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -154,13 +154,14 @@ public class DeployUtil {
//load into kafka
KafkaDataLoader.loadIntoKafka(streamingConfig, data);
+ logger.info("Write {} messages into topic {}", data.size(), streamingConfig.getTopic());
//csv data for H2 use
List<TblColRef> tableColumns = Lists.newArrayList();
for (ColumnDesc columnDesc : tableDesc.getColumns()) {
tableColumns.add(new TblColRef(columnDesc));
}
- TimedJsonStreamParser timedJsonStreamParser = new TimedJsonStreamParser(tableColumns, true);
+ TimedJsonStreamParser timedJsonStreamParser = new TimedJsonStreamParser(tableColumns, "formatTs=true");
StringBuilder sb = new StringBuilder();
for (String json : data) {
List<String> rowColumns = timedJsonStreamParser.parse(new StreamMessage(0, json.getBytes())).getStreamMessage();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
index 4af3bf4..218fe6e 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaRequester.java
@@ -116,8 +116,7 @@ public final class KafkaRequester {
});
return new TopicMeta(kafkaClusterConfig.getTopic(), partitionIds);
}
- logger.debug("cannot find topic:" + kafkaClusterConfig.getTopic());
- return null;
+ throw new IllegalArgumentException("cannot find topic:" + kafkaClusterConfig.getTopic());
}
public static PartitionMetadata getPartitionMetadata(String topic, int partitionId, List<Broker> brokers, KafkaClusterConfig kafkaClusterConfig) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
index 6a3e4fc..320768b 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamingConfig.java
@@ -83,6 +83,10 @@ public class StreamingConfig extends RootPersistentEntity {
@JsonProperty("parserName")
private String parserName;
+
+ //"configA=1;configB=2"
+ @JsonProperty("parserProperties")
+ private String parserProperties;
@JsonProperty("margin")
private long margin;
@@ -163,6 +167,14 @@ public class StreamingConfig extends RootPersistentEntity {
this.margin = margin;
}
+ public String getParserProperties() {
+ return parserProperties;
+ }
+
+ public void setParserProperties(String parserProperties) {
+ this.parserProperties = parserProperties;
+ }
+
@Override
public StreamingConfig clone() {
try {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02178b1c/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
index b7ea34e..6c4146f 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/TimedJsonStreamParser.java
@@ -61,29 +61,48 @@ public final class TimedJsonStreamParser implements StreamParser {
private static final Logger logger = LoggerFactory.getLogger(TimedJsonStreamParser.class);
- private final List<TblColRef> allColumns;
- private boolean formatTs;
+ private List<TblColRef> allColumns;
+ private boolean formatTs = false;
private final ObjectMapper mapper = new ObjectMapper();
+ private String tsColName = "timestamp";
private final JavaType mapType = MapType.construct(HashMap.class, SimpleType.construct(String.class), SimpleType.construct(String.class));
- @SuppressWarnings("unused")
- public TimedJsonStreamParser(List<TblColRef> allColumns) {
+ public TimedJsonStreamParser(List<TblColRef> allColumns, String propertiesStr) {
this.allColumns = allColumns;
- this.formatTs = false;
- }
+ if (!StringUtils.isEmpty(propertiesStr)) {
+ String[] properties = propertiesStr.split(";");
+ for (String prop : properties) {
+ try {
+ String[] parts = prop.split("=");
+ if (parts.length == 2) {
+ switch (parts[0]) {
+ case "formatTs":
+ this.formatTs = Boolean.valueOf(parts[1]);
+ break;
+ case "tsColName":
+ this.tsColName = parts[1];
+ break;
+ default:
+ break;
+ }
+ }
+ } catch (Exception e) {
+ logger.error("Failed to parse property " + prop);
+ //ignore
+ }
+ }
+ }
- public TimedJsonStreamParser(List<TblColRef> allColumns, boolean formatTs) {
- this.allColumns = allColumns;
- this.formatTs = formatTs;
+ logger.info("TimedJsonStreamParser with formatTs {} tsColName {}", formatTs, tsColName);
}
@Override
public ParsedStreamMessage parse(StreamMessage stream) {
try {
Map<String, String> root = mapper.readValue(stream.getRawData(), mapType);
- String tsStr = root.get("timestamp");
+ String tsStr = root.get(tsColName);
Preconditions.checkArgument(!StringUtils.isEmpty(tsStr), "Timestamp field cannot be null");
- long t = Long.valueOf(root.get("timestamp"));
+ long t = Long.valueOf(root.get(tsColName));
ArrayList<String> result = Lists.newArrayList();
for (TblColRef column : allColumns) {