You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/28 10:13:05 UTC
incubator-kylin git commit: fix ci: add more info on
checkAndPutResourceImpl
Repository: incubator-kylin
Updated Branches:
refs/heads/streaming-cubing 867425938 -> 399176bf2
fix ci: add more info on checkAndPutResourceImpl
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/399176bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/399176bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/399176bf
Branch: refs/heads/streaming-cubing
Commit: 399176bf24bfa854946238901a4b6a5784261b39
Parents: 8674259
Author: honma <ho...@ebay.com>
Authored: Thu May 28 16:12:48 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 28 16:12:48 2015 +0800
----------------------------------------------------------------------
.../common/persistence/HBaseResourceStore.java | 35 +++++++++-----------
.../org/apache/kylin/common/util/TimeUtil.java | 13 ++++++--
.../apache/kylin/common/util/TimeUtilTest.java | 25 +++++++-------
.../kylin/job/streaming/StreamingBootstrap.java | 1 -
.../job/ITKafkaBasedIIStreamBuilderTest.java | 32 +++---------------
5 files changed, 42 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/399176bf/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
index 2868368..e665298 100644
--- a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
+++ b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
@@ -18,34 +18,27 @@
package org.apache.kylin.common.persistence;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.kylin.common.util.Bytes;
-
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.HadoopUtil;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
public class HBaseResourceStore extends ResourceStore {
private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
@@ -197,8 +190,10 @@ public class HBaseResourceStore extends ResourceStore {
Put put = buildPut(resPath, newTS, row, content, table);
boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
- if (!ok)
- throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + getResourceTimestamp(resPath));
+ if (!ok) {
+ long real = getResourceTimestamp(resPath);
+ throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
+ }
table.flushCommits();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/399176bf/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
index 26e3e06..0aa58e4 100644
--- a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -3,8 +3,13 @@ package org.apache.kylin.common.util;
/**
*/
public class TimeUtil {
- private static int ONE_MINUTE_TS = 60 * 1000;
- private static int ONE_HOUR_TS = 60 * 60 * 1000;
+ public enum NormalizedTimeUnit {
+ MINUTE, HOUR, DAY
+ }
+
+ private static long ONE_MINUTE_TS = 60 * 1000;
+ private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS;
+ private static long ONE_DAY_TS = 24 * ONE_HOUR_TS;
public static long getMinuteStart(long ts) {
return ts / ONE_MINUTE_TS * ONE_MINUTE_TS;
@@ -13,4 +18,8 @@ public class TimeUtil {
public static long getHourStart(long ts) {
return ts / ONE_HOUR_TS * ONE_HOUR_TS;
}
+
+ public static long getDayStart(long ts) {
+ return ts / ONE_DAY_TS * ONE_DAY_TS;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/399176bf/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
index 90a0c40..cfa11d8 100644
--- a/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/TimeUtilTest.java
@@ -1,24 +1,25 @@
package org.apache.kylin.common.util;
+import org.apache.kylin.common.util.TimeUtil.NormalizedTimeUnit;
+import org.junit.Assert;
+import org.junit.Test;
+
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
-import org.junit.Assert;
-import org.junit.Test;
-
/**
*/
public class TimeUtilTest {
- public static long normalizeTime(long timeMillis, NormalizeUnit unit) {
+ public static long normalizeTime(long timeMillis, NormalizedTimeUnit unit) {
Calendar a = Calendar.getInstance();
Calendar b = Calendar.getInstance();
b.clear();
a.setTimeInMillis(timeMillis);
- if (unit == NormalizeUnit.MINUTE) {
+ if (unit == NormalizedTimeUnit.MINUTE) {
b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), a.get(Calendar.MINUTE));
- } else if (unit == NormalizeUnit.HOUR) {
+ } else if (unit == NormalizedTimeUnit.HOUR) {
b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), 0);
}
return b.getTimeInMillis();
@@ -29,15 +30,13 @@ public class TimeUtilTest {
java.text.DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
long t1 = dateFormat.parse("2012/01/01 00:00:01").getTime();
- Assert.assertEquals(normalizeTime(t1, NormalizeUnit.HOUR), TimeUtil.getHourStart(t1));
- Assert.assertEquals(normalizeTime(t1, NormalizeUnit.MINUTE), TimeUtil.getMinuteStart(t1));
+ Assert.assertEquals(normalizeTime(t1, NormalizedTimeUnit.HOUR), TimeUtil.getHourStart(t1));
+ Assert.assertEquals(normalizeTime(t1, NormalizedTimeUnit.MINUTE), TimeUtil.getMinuteStart(t1));
long t2 = dateFormat.parse("2012/12/31 11:02:01").getTime();
- Assert.assertEquals(normalizeTime(t2, NormalizeUnit.HOUR), TimeUtil.getHourStart(t2));
- Assert.assertEquals(normalizeTime(t2, NormalizeUnit.MINUTE), TimeUtil.getMinuteStart(t2));
+ Assert.assertEquals(normalizeTime(t2, NormalizedTimeUnit.HOUR), TimeUtil.getHourStart(t2));
+ Assert.assertEquals(normalizeTime(t2, NormalizedTimeUnit.MINUTE), TimeUtil.getMinuteStart(t2));
}
- public enum NormalizeUnit {
- MINUTE, HOUR
- }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/399176bf/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 7854fd5..534a261 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -45,7 +45,6 @@ import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.streaming.*;
import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/399176bf/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
index 2432220..71c9644 100644
--- a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
+++ b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
@@ -34,19 +34,19 @@
package org.apache.kylin.job;
-import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
import org.apache.kylin.job.streaming.StreamingBootstrap;
-import org.junit.*;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
/**
*/
@@ -63,30 +63,6 @@ public class ITKafkaBasedIIStreamBuilderTest {
System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
- @AfterClass
- public static void afterClass() throws Exception {
-// backup();
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = {"--delete", "true"};
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
@Before
public void before() throws Exception {
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);