You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2017/04/11 09:37:00 UTC
eagle git commit: [EAGLE-999] JobConfigSerDeser fails to
serialize/deserialize data with long string
Repository: eagle
Updated Branches:
refs/heads/master 20b4bea3e -> 7c0d83f3c
[EAGLE-999] JobConfigSerDeser fails to serialize/deserialize data with long string
https://issues.apache.org/jira/browse/EAGLE-999
Author: Zhao, Qingwen <qi...@apache.org>
Closes #909 from qingwen220/EAGLE-999.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/7c0d83f3
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/7c0d83f3
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/7c0d83f3
Branch: refs/heads/master
Commit: 7c0d83f3c4847dac450ba505153a5bf17d30fd73
Parents: 20b4bea
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Tue Apr 11 17:36:50 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Tue Apr 11 17:36:50 2017 +0800
----------------------------------------------------------------------
...doop.queue.HadoopQueueRunningAppProvider.xml | 2 +-
eagle-jpm/eagle-jpm-entity/pom.xml | 3 +-
.../jpm/mr/historyentity/JobConfigSerDeser.java | 41 ++++--------
.../eagle/jpm/mr/JobConfigSerDeserTest.java | 65 ++++++++++++++++++++
....running.MRRunningJobApplicationProvider.xml | 2 +-
...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 4 +-
...ity.oozie.parse.OozieAuditLogAppProvider.xml | 2 +-
....eagle.topology.TopologyCheckAppProvider.xml | 2 +-
8 files changed, 83 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
index 2d68da8..f33797f 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
@@ -57,7 +57,7 @@
<property>
<name>dataSinkConfig.HADOOP_QUEUE_STREAM.topic</name>
<displayName>Destination(Kafka Topic) Of Queue Stream Data</displayName>
- <value>yarn_queue</value>
+ <value>yarn_queue_${siteId}</value>
<description>topic for kafka data sink</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/pom.xml b/eagle-jpm/eagle-jpm-entity/pom.xml
index 98982f9..9a38725 100644
--- a/eagle-jpm/eagle-jpm-entity/pom.xml
+++ b/eagle-jpm/eagle-jpm-entity/pom.xml
@@ -42,10 +42,9 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>3.8.1</version>
+ <version>${junit.version}</version>
<scope>test</scope>
</dependency>
-
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
index cfa50f9..f4f58d8 100644
--- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
@@ -19,45 +19,26 @@
package org.apache.eagle.jpm.mr.historyentity;
import org.apache.eagle.log.entity.meta.EntitySerDeser;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.eagle.log.entity.meta.MapSerDeser;
import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
+
public class JobConfigSerDeser implements EntitySerDeser<JobConfig> {
+ private static final MapSerDeser INSTANCE = new MapSerDeser();
+
@Override
public JobConfig deserialize(byte[] bytes) {
- JobConfig jc = new JobConfig();
- Map<String, String> map = new TreeMap<String, String>();
- jc.setConfig(map);
- String sb = Bytes.toString(bytes);
- String[] keyValue = sb.split(",");
- for (String pair : keyValue) {
- String[] str = pair.split(":");
- if (pair.equals("") || str[0].equals("")) {
- continue;
- }
- String key = str[0];
- String value = "";
- if (str.length == 2) {
- value = str[1];
- }
- map.put(key, value);
- }
- return jc;
+ Map map = INSTANCE.deserialize(bytes);
+ JobConfig config = new JobConfig();
+ config.setConfig(map);
+ return config;
}
-
+
@Override
- public byte[] serialize(JobConfig conf) {
- Map<String, String> map = conf.getConfig();
- StringBuilder sb = new StringBuilder();
- for (Entry<String, String> entry : map.entrySet()) {
- sb.append(entry.getKey() + ":" + entry.getValue() + ",");
- }
- sb.deleteCharAt(sb.length() - 1);
- return sb.toString().getBytes();
+ public byte[] serialize(JobConfig jobConfig) {
+ return INSTANCE.serialize(jobConfig.getConfig());
}
@Override
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java b/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java
new file mode 100644
index 0000000..a4b5307
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.mr;
+
+import org.apache.eagle.jpm.mr.historyentity.JobConfig;
+import org.apache.eagle.jpm.mr.historyentity.JobConfigSerDeser;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class JobConfigSerDeserTest {
+
+ @Test
+ public void test() {
+ String value = "INSERT OVERWRITE TABLE kylin_intermediate_KYLIN_HIVE_METRICS_QUERY_CUBE SELECT\n" +
+ "HIVE_METRICS_QUERY_CUBE.`CUBE_NAME`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`SEGMENT_NAME`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`CUBOID_SOURCE`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`CUBOID_TARGET`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`IF_MATCH`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`IF_SUCCESS`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`KYEAR_BEGIN_DATE`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`KMONTH_BEGIN_DATE`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`KWEEK_BEGIN_DATE`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`KDAY_DATE`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`WEIGHT_PER_HIT`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_COUNT`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_TIME_SUM`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_TIME_MAX`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_COUNT_SKIP`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_SCAN`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_RETURN`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_AGGREGATE_FILTER`\n" +
+ ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_AGGREGATE`\n" +
+ "FROM KYLIN.HIVE_METRICS_QUERY_CUBE as HIVE_METRICS_QUERY_CUBE\n" +
+ "WHERE (((HIVE_METRICS_QUERY_CUBE.KDAY_DATE = '2017-04-06' AND HIVE_METRICS_QUERY_CUBE.KDAY_TIME >= '18:00:00') OR (HIVE_METRICS_QUERY_CUBE.KDAY_DATE > '2017-04-06')) AND ((HIVE_METRICS_QUERY_CUBE.KDAY_DATE = '2017-04-06' AND HIVE_METRICS_QUERY_CUBE.KDAY_TIME < '20:00:00') OR (HIVE_METRICS_QUERY_CUBE.KDAY_DATE < '2017-04-06')))";
+
+ Map<String, String> conf = new HashMap<>();
+ conf.put("test1", value);
+ JobConfig source = new JobConfig();
+ source.setConfig(conf);
+ JobConfigSerDeser serDeser = new JobConfigSerDeser();
+ byte[] bytes = serDeser.serialize(source);
+ JobConfig target = serDeser.deserialize(bytes);
+
+ Assert.assertTrue(target.getConfig().size() == 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
index fe3ab83..907e35a 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
@@ -104,7 +104,7 @@
<property>
<name>dataSinkConfig.ACCEPTED_APP_STREAM.topic</name>
<displayName>Destination(Kafka Topic) Of App Stream Data</displayName>
- <value>yarn_accepted_app_{site}</value>
+ <value>yarn_accepted_app_{siteId}</value>
<description>topic for kafka data sink</description>
<required>true</required>
</property>
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
index d48340c..d453cd6 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
@@ -130,14 +130,14 @@
<property>
<name>dataSinkConfig.HDFS_AUDIT_LOG_ENRICHED_STREAM.topic</name>
<displayName>Kafka Topic for Auditlog Event Sink</displayName>
- <value>hdfs_audit_event_${site}</value>
+ <value>hdfs_audit_event_${siteId}</value>
<description>topic for kafka data sink</description>
<required>true</required>
</property>
<property>
<name>dataSinkConfig.HDFS_AUDIT_LOG_TRAFFIC_STREAM.topic</name>
<displayName>Kafka Topic for Traffic Data Sink</displayName>
- <value>hdfs_traffic_event_${site}</value>
+ <value>hdfs_traffic_event_${siteId}</value>
<description>topic for kafka data sink</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
index ee1715c..5434a9d 100644
--- a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
@@ -69,7 +69,7 @@
<property>
<name>dataSourceConfig.topic</name>
<displayName>Kafka Topic for Data Consumption</displayName>
- <value>oozie_audit_log</value>
+ <value>oozie_audit_log_${siteId}</value>
<description>data source topic</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
index 4bdb1c4..aa36c53 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
+++ b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
@@ -165,7 +165,7 @@
<property>
<name>dataSinkConfig.topic</name>
<displayName>Topic For Kafka Data Sink</displayName>
- <value>topology_check</value>
+ <value>topology_check_${siteId}</value>
<description>topic For kafka data sink</description>
</property>
<property>