You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/06 09:31:16 UTC
[40/49] kylin git commit: minor change on error messages
minor change on error messages
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c587b2ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c587b2ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c587b2ed
Branch: refs/heads/KYLIN-1971
Commit: c587b2ed3a968262a794e893f6af8fc109b02730
Parents: 235b123
Author: shaofengshi <sh...@apache.org>
Authored: Fri Nov 4 13:33:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Nov 4 13:33:19 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 8 ++++----
.../apache/kylin/engine/mr/common/HadoopStatusGetter.java | 2 +-
.../main/java/org/apache/kylin/source/kafka/KafkaSource.java | 8 ++++++++
3 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 77791ce..21bb10e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -204,7 +204,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
StringUtil.appendWithSeparator(kylinDependency, filteredHive);
} else {
- logger.info("No hive dependency jars set in the environment, will find them from jvm:");
+ logger.info("No hive dependency jars set in the environment, will find them from classpath:");
try {
String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
@@ -227,17 +227,17 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for kafka dependencies
if (kylinKafkaDependency != null) {
kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
- logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency);
+ logger.info("Kafka Dependencies: " + kylinKafkaDependency);
StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency);
} else {
- logger.info("No Kafka dependency jars set in the environment, will find them from jvm:");
+ logger.info("No Kafka dependency jar set in the environment, will find them from classpath:");
try {
String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath);
logger.info("kafka jar file: " + kafkaClientJarPath);
} catch (ClassNotFoundException e) {
- logger.error("Cannot found kafka dependency jars: " + e);
+ logger.warn("Not found kafka client jar from classpath, it is optional for normal build: " + e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 619de90..7dcb73e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -55,7 +55,7 @@ public class HadoopStatusGetter {
private final String mrJobId;
private final String yarnUrl;
- protected static final Logger logger = LoggerFactory.getLogger(HadoopStatusChecker.class);
+ protected static final Logger logger = LoggerFactory.getLogger(HadoopStatusGetter.class);
public HadoopStatusGetter(String yarnUrl, String mrJobId) {
this.yarnUrl = yarnUrl;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
index bb676e6..7a5d94f 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
@@ -107,6 +107,14 @@ public class KafkaSource implements ISource {
totalEndOffset += v;
}
+ if (totalStartOffset > totalEndOffset) {
+ throw new IllegalArgumentException("Illegal offset: start: " + totalStartOffset + ", end: " + totalEndOffset);
+ }
+
+ if (totalStartOffset == totalEndOffset) {
+ throw new IllegalArgumentException("No new message comes, startOffset = endOffset:" + totalStartOffset);
+ }
+
result.setStartOffset(totalStartOffset);
result.setEndOffset(totalEndOffset);