You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/22 06:13:24 UTC
[34/47] incubator-kylin git commit: KYLIN-875 rename modules:
core-common, core-cube, core-dictionary, core-cube
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/PartialSorter.java b/core-common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
new file mode 100644
index 0000000..f73f525
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ *
+ * This utility class sorts only the specified part of a list
+ */
+public class PartialSorter {
+ public static <T> void partialSort(List<T> list, List<Integer> items, Comparator<? super T> c) {
+ List<T> temp = Lists.newLinkedList();
+ for (int index : items) {
+ temp.add(list.get(index));
+ }
+ Collections.sort(temp, c);
+ for (int i = 0; i < temp.size(); ++i) {
+ list.set(items.get(i), temp.get(i));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/RandomSampler.java b/core-common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
new file mode 100644
index 0000000..f793fdc
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RandomSampler<T> {
+
+ private Random rdm = new Random();
+
+ public List<T> sample(List<T> data, int sampleNumber) {
+ if (data == null) {
+ throw new IllegalArgumentException("Input list is null");
+ }
+ if (data.size() < sampleNumber) {
+ return data;
+ }
+
+ List<T> result = new ArrayList<T>(sampleNumber);
+ int n = data.size();
+ for (int i = 0; i < n; i++) {
+ if (i < sampleNumber) {
+ result.add(data.get(i));
+ } else {
+ int j = rdm.nextInt(i);
+ if (j < sampleNumber) {
+ result.set(j, data.get(i));
+ }
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
new file mode 100644
index 0000000..05e4d35
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
@@ -0,0 +1,174 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.*;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableMap;
+
+/**
+ */
+public class RangeUtil {
+
+ /**
+ * for NavigableMap sorted by C, given a range of C, return the sub map whose key fails in the range
+ */
+ public static <C extends Comparable<?>, V> NavigableMap<C, V> filter(NavigableMap<C, V> values, Range<C> filterRange) {
+ if (filterRange == null || filterRange.isEmpty()) {
+ return Maps.newTreeMap();
+ } else if (filterRange.equals(Ranges.all())) {
+ return values;
+ }
+
+ if (filterRange.hasUpperBound() && !filterRange.hasLowerBound()) {
+ return values.headMap(filterRange.upperEndpoint(), upperBoundInclusive(filterRange));
+ } else if (filterRange.hasLowerBound() && !filterRange.hasUpperBound()) {
+ return values.tailMap(filterRange.lowerEndpoint(), lowerBoundInclusive(filterRange));
+ } else {
+ return values.subMap(filterRange.lowerEndpoint(), lowerBoundInclusive(filterRange),//
+ filterRange.upperEndpoint(), upperBoundInclusive(filterRange));
+ }
+ }
+
+ public static <C extends Comparable<?>> boolean lowerBoundInclusive(Range<C> range) {
+ if (!range.hasLowerBound()) {
+ throw new IllegalArgumentException(("This range does not have lower bound" + range));
+ }
+ return range.lowerBoundType() == BoundType.CLOSED;
+ }
+
+ public static <C extends Comparable<?>> boolean upperBoundInclusive(Range<C> range) {
+ if (!range.hasUpperBound()) {
+ throw new IllegalArgumentException(("This range does not have upper bound" + range));
+ }
+ return range.upperBoundType() == BoundType.CLOSED;
+ }
+
+ public static <C extends Comparable<?>> Range<C> merge(Range<C> a, Range<C> b) {
+ if (a == null && b == null) {
+ return null;
+ } else if (a == null || b == null) {
+ return a == null ? b : a;
+ } else {
+ return a.span(b);
+ }
+ }
+
+ /**
+ * remove from self the elements that exist in other
+ * @return
+ */
+ public static <C extends Comparable<?>> List<Range<C>> remove(Range<C> self, Range<C> other) {
+ // mimic the following logic in guava 18:
+ // RangeSet<C> rangeSet = TreeRangeSet.create();
+ // rangeSet.add(self);
+ // rangeSet.remove(other);
+ // return Lists.newArrayList(rangeSet.asRanges());
+
+ if (!self.isConnected(other)) {
+ return Collections.singletonList(self);
+ }
+ Range<C> share = self.intersection(other);
+ if (share.isEmpty()) {
+ return Collections.singletonList(self);
+ }
+
+ List<Range<C>> ret = Lists.newArrayList();
+
+ //see left part
+ if (!self.hasLowerBound()) {
+ if (share.hasLowerBound()) {
+ if (share.lowerBoundType() == BoundType.CLOSED) {
+ ret.add(Ranges.lessThan(share.lowerEndpoint()));
+ } else {
+ ret.add(Ranges.atMost(share.lowerEndpoint()));
+ }
+ }
+ } else {
+ if (self.lowerEndpoint() != share.lowerEndpoint()) {
+ if (self.lowerBoundType() == BoundType.CLOSED) {
+ if (share.lowerBoundType() == BoundType.CLOSED) {
+ ret.add(Ranges.closedOpen(self.lowerEndpoint(), share.lowerEndpoint()));
+ } else {
+ ret.add(Ranges.closed(self.lowerEndpoint(), share.lowerEndpoint()));
+ }
+ } else {
+ if (share.lowerBoundType() == BoundType.CLOSED) {
+ ret.add(Ranges.open(self.lowerEndpoint(), share.lowerEndpoint()));
+ } else {
+ ret.add(Ranges.openClosed(self.lowerEndpoint(), share.lowerEndpoint()));
+ }
+ }
+ } else {
+ if (self.lowerBoundType() == BoundType.CLOSED && share.lowerBoundType() == BoundType.OPEN) {
+ ret.add(Ranges.closed(self.lowerEndpoint(), share.lowerEndpoint()));
+ }
+ }
+ }
+
+ //see right part
+ if (!self.hasUpperBound()) {
+ if (share.hasUpperBound()) {
+ if (share.upperBoundType() == BoundType.CLOSED) {
+ ret.add(Ranges.greaterThan(share.upperEndpoint()));
+ } else {
+ ret.add(Ranges.atLeast(share.upperEndpoint()));
+ }
+ }
+ } else {
+ if (self.upperEndpoint() != share.upperEndpoint()) {
+ if (self.upperBoundType() == BoundType.CLOSED) {
+ if (share.upperBoundType() == BoundType.CLOSED) {
+ ret.add(Ranges.openClosed(share.upperEndpoint(), self.upperEndpoint()));
+ } else {
+ ret.add(Ranges.closed(share.upperEndpoint(), self.upperEndpoint()));
+ }
+ } else {
+ if (share.upperBoundType() == BoundType.CLOSED) {
+ ret.add(Ranges.open(share.upperEndpoint(), self.upperEndpoint()));
+ } else {
+ ret.add(Ranges.closedOpen(share.upperEndpoint(), self.upperEndpoint()));
+ }
+ }
+ } else {
+ if (self.upperBoundType() == BoundType.CLOSED && share.upperBoundType() == BoundType.OPEN) {
+ ret.add(Ranges.closed(self.upperEndpoint(), share.upperEndpoint()));
+ }
+ }
+ }
+
+ return ret;
+
+ }
+
+ public static String formatTsRange(Range<Long> tsRange) {
+ if (tsRange == null)
+ return null;
+
+ StringBuilder sb = new StringBuilder();
+ if (tsRange.hasLowerBound()) {
+ if (tsRange.lowerBoundType() == BoundType.CLOSED) {
+ sb.append("[");
+ } else {
+ sb.append("(");
+ }
+ sb.append(DateFormat.formatToTimeStr(tsRange.lowerEndpoint()));
+ } else {
+ sb.append("(-∞");
+ }
+
+ sb.append("~");
+
+ if (tsRange.hasUpperBound()) {
+ sb.append(DateFormat.formatToTimeStr(tsRange.upperEndpoint()));
+ if (tsRange.upperBoundType() == BoundType.CLOSED) {
+ sb.append("]");
+ } else {
+ sb.append(")");
+ }
+ } else {
+ sb.append("+∞)");
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java b/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
new file mode 100644
index 0000000..32eb72a
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.slf4j.LoggerFactory;
+
+import com.jcraft.jsch.Channel;
+import com.jcraft.jsch.ChannelExec;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
+
+public class SSHClient {
+ protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(SSHClient.class);
+
+ private String hostname;
+ private int port;
+ private String username;
+ private String password;
+ private String identityPath;
+
+ public SSHClient(String hostname, int port, String username, String password) {
+ this.hostname = hostname;
+ this.username = username;
+ this.port = port;
+ if (password != null && new File(password).exists()) {
+ this.identityPath = new File(password).getAbsolutePath();
+ this.password = null;
+ } else {
+ this.password = password;
+ this.identityPath = null;
+ }
+ }
+
+ public void scpFileToRemote(String localFile, String remoteTargetDirectory) throws Exception {
+ FileInputStream fis = null;
+ try {
+ System.out.println("SCP file " + localFile + " to " + remoteTargetDirectory);
+
+ Session session = newJSchSession();
+ session.connect();
+
+ boolean ptimestamp = false;
+
+ // exec 'scp -t rfile' remotely
+ String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteTargetDirectory;
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+
+ // get I/O streams for remote scp
+ OutputStream out = channel.getOutputStream();
+ InputStream in = channel.getInputStream();
+
+ channel.connect();
+
+ if (checkAck(in) != 0) {
+ System.exit(0);
+ }
+
+ File _lfile = new File(localFile);
+
+ if (ptimestamp) {
+ command = "T " + (_lfile.lastModified() / 1000) + " 0";
+ // The access time should be sent here,
+ // but it is not accessible with JavaAPI ;-<
+ command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
+ out.write(command.getBytes());
+ out.flush();
+ if (checkAck(in) != 0) {
+ throw new Exception("Error in checkAck()");
+ }
+ }
+
+ // send "C0644 filesize filename", where filename should not include '/'
+ long filesize = _lfile.length();
+ command = "C0644 " + filesize + " ";
+ if (localFile.lastIndexOf("/") > 0) {
+ command += localFile.substring(localFile.lastIndexOf("/") + 1);
+ } else if (localFile.lastIndexOf(File.separator) > 0) {
+ command += localFile.substring(localFile.lastIndexOf(File.separator) + 1);
+ } else {
+ command += localFile;
+ }
+ command += "\n";
+ out.write(command.getBytes());
+ out.flush();
+ if (checkAck(in) != 0) {
+ throw new Exception("Error in checkAck()");
+ }
+
+ // send a content of lfile
+ fis = new FileInputStream(localFile);
+ byte[] buf = new byte[1024];
+ while (true) {
+ int len = fis.read(buf, 0, buf.length);
+ if (len <= 0)
+ break;
+ out.write(buf, 0, len); // out.flush();
+ }
+ fis.close();
+ fis = null;
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+ if (checkAck(in) != 0) {
+ throw new Exception("Error in checkAck()");
+ }
+ out.close();
+
+ channel.disconnect();
+ session.disconnect();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ try {
+ if (fis != null)
+ fis.close();
+ } catch (Exception ee) {
+ }
+ }
+ }
+
+ public void scpFileToLocal(String rfile, String lfile) throws Exception {
+ FileOutputStream fos = null;
+ try {
+ System.out.println("SCP remote file " + rfile + " to local " + lfile);
+
+ String prefix = null;
+ if (new File(lfile).isDirectory()) {
+ prefix = lfile + File.separator;
+ }
+
+ Session session = newJSchSession();
+ session.connect();
+ // exec 'scp -f rfile' remotely
+ String command = "scp -f " + rfile;
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+
+ // get I/O streams for remote scp
+ OutputStream out = channel.getOutputStream();
+ InputStream in = channel.getInputStream();
+
+ channel.connect();
+
+ byte[] buf = new byte[1024];
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+
+ while (true) {
+ int c = checkAck(in);
+ if (c != 'C') {
+ break;
+ }
+
+ // read '0644 '
+ in.read(buf, 0, 5);
+
+ long filesize = 0L;
+ while (true) {
+ if (in.read(buf, 0, 1) < 0) {
+ // error
+ break;
+ }
+ if (buf[0] == ' ')
+ break;
+ filesize = filesize * 10L + (long) (buf[0] - '0');
+ }
+
+ String file = null;
+ for (int i = 0;; i++) {
+ in.read(buf, i, 1);
+ if (buf[i] == (byte) 0x0a) {
+ file = new String(buf, 0, i);
+ break;
+ }
+ }
+
+ //System.out.println("filesize="+filesize+", file="+file);
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+
+ // read a content of lfile
+ fos = new FileOutputStream(prefix == null ? lfile : prefix + file);
+ int foo;
+ while (true) {
+ if (buf.length < filesize)
+ foo = buf.length;
+ else
+ foo = (int) filesize;
+ foo = in.read(buf, 0, foo);
+ if (foo < 0) {
+ // error
+ break;
+ }
+ fos.write(buf, 0, foo);
+ filesize -= foo;
+ if (filesize == 0L)
+ break;
+ }
+ fos.close();
+ fos = null;
+
+ if (checkAck(in) != 0) {
+ System.exit(0);
+ }
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+ }
+
+ session.disconnect();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ try {
+ if (fos != null)
+ fos.close();
+ } catch (Exception ee) {
+ }
+ }
+ }
+
+ public SSHClientOutput execCommand(String command) throws Exception {
+ return execCommand(command, 7200, null);
+ }
+
+ public SSHClientOutput execCommand(String command, int timeoutSeconds, Logger logAppender) throws Exception {
+ try {
+ System.out.println("[" + username + "@" + hostname + "] Execute command: " + command);
+
+ StringBuffer text = new StringBuffer();
+ int exitCode = -1;
+
+ Session session = newJSchSession();
+ session.connect();
+
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+
+ channel.setInputStream(null);
+
+ // channel.setOutputStream(System.out);
+
+ ((ChannelExec) channel).setErrStream(System.err);
+
+ InputStream in = channel.getInputStream();
+ InputStream err = ((ChannelExec) channel).getErrStream();
+
+ channel.connect();
+
+ int timeout = timeoutSeconds;
+ byte[] tmp = new byte[1024];
+ while (true) {
+ timeout--;
+ while (in.available() > 0) {
+ int i = in.read(tmp, 0, 1024);
+ if (i < 0)
+ break;
+
+ String line = new String(tmp, 0, i);
+ text.append(line);
+ if (logAppender != null) {
+ logAppender.log(line);
+ }
+ }
+ while (err.available() > 0) {
+ int i = err.read(tmp, 0, 1024);
+ if (i < 0)
+ break;
+
+ String line = new String(tmp, 0, i);
+ text.append(line);
+ if (logAppender != null) {
+ logAppender.log(line);
+ }
+ }
+ if (channel.isClosed()) {
+ if (in.available() > 0)
+ continue;
+ exitCode = channel.getExitStatus();
+ System.out.println("[" + username + "@" + hostname + "] Command exit-status: " + exitCode);
+
+ break;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (Exception ee) {
+ throw ee;
+ }
+ if (timeout < 0)
+ throw new Exception("Remote command not finished within " + timeoutSeconds + " seconds.");
+ }
+ channel.disconnect();
+ session.disconnect();
+ return new SSHClientOutput(exitCode, text.toString());
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ private Session newJSchSession() throws JSchException {
+ JSch jsch = new JSch();
+ if (identityPath != null) {
+ jsch.addIdentity(identityPath);
+ }
+
+ Session session = jsch.getSession(username, hostname, port);
+ if (password != null) {
+ session.setPassword(password);
+ }
+ session.setConfig("StrictHostKeyChecking", "no");
+ return session;
+ }
+
+ private int checkAck(InputStream in) throws IOException {
+ int b = in.read();
+ // b may be 0 for success,
+ // 1 for error,
+ // 2 for fatal error,
+ // -1
+ if (b == 0)
+ return b;
+ if (b == -1)
+ return b;
+
+ if (b == 1 || b == 2) {
+ StringBuffer sb = new StringBuffer();
+ int c;
+ do {
+ c = in.read();
+ sb.append((char) c);
+ } while (c != '\n');
+ if (b == 1) { // error
+ System.out.print(sb.toString());
+ }
+ if (b == 2) { // fatal error
+ System.out.print(sb.toString());
+ }
+ }
+ return b;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java b/core-common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
new file mode 100644
index 0000000..cf25001
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+
+public class SSHClientOutput {
+ private String text;
+ private int exitCode = -1;
+
+ /**
+ * @param text
+ * @param exitCode
+ */
+ public SSHClientOutput(int exitCode, String text) {
+ this.text = text;
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * @return the text
+ */
+ public String getText() {
+ return text.toString();
+ }
+
+ /**
+ * @return the exitCode
+ */
+ public int getExitCode() {
+ return exitCode;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java
new file mode 100644
index 0000000..f0d7cdb
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SortUtil.java
@@ -0,0 +1,20 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.base.Function;
+import com.google.common.collect.TreeMultimap;
+
+import java.util.Iterator;
+
+/**
+ */
+public class SortUtil {
+ public static <T extends Comparable, E extends Comparable> Iterator<T> extractAndSort(Iterator<T> input, Function<T, E> extractor) {
+ TreeMultimap<E, T> reorgnized = TreeMultimap.create();
+ while (input.hasNext()) {
+ T t = input.next();
+ E e = extractor.apply(t);
+ reorgnized.put(e, t);
+ }
+ return reorgnized.values().iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SoutLogger.java b/core-common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
new file mode 100644
index 0000000..0ed03a7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+/**
+ */
+public class SoutLogger implements Logger {
+
+ @Override
+ public void log(String message) {
+ System.out.println(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
new file mode 100644
index 0000000..8751b78
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class SplittedBytes {
+ public SplittedBytes(int length) {
+ this.value = new byte[length];
+ this.length = 0;
+ }
+
+ public byte[] value;
+ public int length;
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringSplitter.java b/core-common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
new file mode 100644
index 0000000..6df82d4
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class StringSplitter {
+ public static String[] split(String str, String delimiter) {
+ // The optimized split function
+ List<String> list = new ArrayList<String>();
+ int index = 0, offset = 0;
+ int l = delimiter.length();
+ if (str.startsWith(delimiter)) {
+ // in case the first field is empty
+ list.add("");
+ offset = offset + l;
+ }
+ while ((index = str.indexOf(delimiter, index + 1)) != -1) {
+ list.add(str.substring(offset, index));
+ offset = index + l;
+ }
+ // add the last field, or the str doesn't contain delimiter at all
+ list.add(str.substring(offset));
+ return list.toArray(new String[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
new file mode 100644
index 0000000..468a6e1
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ */
+public class StringUtil {
+
+ public static String[] filterSystemArgs(String args[]) {
+ ArrayList<String> whatsLeft = new ArrayList<String>();
+ for (String a : args) {
+ if (a.startsWith("-D")) {
+ String key;
+ String value;
+ int cut = a.indexOf('=');
+ if (cut < 0) {
+ key = a.substring(2);
+ value = "";
+ } else {
+ key = a.substring(2, cut);
+ value = a.substring(cut + 1);
+ }
+ System.setProperty(key, value);
+ } else {
+ whatsLeft.add(a);
+ }
+ }
+ return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]);
+ }
+
+ public static String join(Iterable<String> parts, String separator) {
+ StringBuilder buf = new StringBuilder();
+ for (String p : parts) {
+ if (buf.length() > 0)
+ buf.append(separator);
+ buf.append(p);
+ }
+ return buf.toString();
+ }
+
+ public static void toUpperCaseArray(String[] source, String[] target) {
+ if(source!=null) {
+ for (int i = 0; i < source.length; i++) {
+ if (source[i] != null) {
+ target[i] = source[i].toUpperCase();
+ }
+ }
+ }
+ }
+
+ public static String dropSuffix(String str, String suffix) {
+ if (str.endsWith(suffix))
+ return str.substring(0, str.length() - suffix.length());
+ else
+ return str;
+ }
+
+ public static String min(Collection<String> strs) {
+ String min = null;
+ for (String s : strs) {
+ if (min == null || min.compareTo(s) > 0)
+ min = s;
+ }
+ return min;
+ }
+
+ public static String max(Collection<String> strs) {
+ String max = null;
+ for (String s : strs) {
+ if (max == null || max.compareTo(s) < 0)
+ max = s;
+ }
+ return max;
+ }
+
+ public static String min(String s1, String s2) {
+ if (s1 == null)
+ return s2;
+ else if (s2 == null)
+ return s1;
+ else
+ return s1.compareTo(s2) < 0 ? s1 : s2;
+ }
+
+ public static String max(String s1, String s2) {
+ if (s1 == null)
+ return s2;
+ else if (s2 == null)
+ return s1;
+ else
+ return s1.compareTo(s2) > 0 ? s1 : s2;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
new file mode 100644
index 0000000..6f623d3
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+public class TarGZUtil {
+
+ public static void uncompressTarGZ(File tarFile, File dest) throws IOException {
+ dest.mkdir();
+ TarArchiveInputStream tarIn = null;
+
+ tarIn = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(tarFile))));
+
+ TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
+ // tarIn is a TarArchiveInputStream
+ while (tarEntry != null) {// create a file with the same name as the tarEntry
+ File destPath = new File(dest, tarEntry.getName());
+ System.out.println("working: " + destPath.getCanonicalPath());
+ if (tarEntry.isDirectory()) {
+ destPath.mkdirs();
+ } else {
+ destPath.createNewFile();
+ //byte [] btoRead = new byte[(int)tarEntry.getSize()];
+ byte[] btoRead = new byte[1024];
+ //FileInputStream fin
+ // = new FileInputStream(destPath.getCanonicalPath());
+ BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(destPath));
+ int len = 0;
+
+ while ((len = tarIn.read(btoRead)) != -1) {
+ bout.write(btoRead, 0, len);
+ }
+
+ bout.close();
+ btoRead = null;
+
+ }
+ tarEntry = tarIn.getNextTarEntry();
+ }
+ tarIn.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
new file mode 100644
index 0000000..c493424
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ */
+public class ThreadUtil {
+ @SuppressWarnings("unused")
+ public static void main(String[] args) {
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());//Threads.newDaemonThreadFactory("htable"));
+
+ for (int i = 0; i < Integer.MAX_VALUE; ++i) {
+ System.out.println("index: " + i);
+ Future<?> future = pool.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
new file mode 100644
index 0000000..c79e88b
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -0,0 +1,29 @@
+package org.apache.kylin.common.util;
+
+/**
+ */
+public class TimeUtil {
+ public enum NormalizedTimeUnit {
+ MINUTE, HOUR, DAY
+ }
+
+ private static long ONE_MINUTE_TS = 60 * 1000;
+ private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS;
+ private static long ONE_DAY_TS = 24 * ONE_HOUR_TS;
+
+ public static long getMinuteStart(long ts) {
+ return ts / ONE_MINUTE_TS * ONE_MINUTE_TS;
+ }
+
+ public static long getHourStart(long ts) {
+ return ts / ONE_HOUR_TS * ONE_HOUR_TS;
+ }
+
+ public static long getDayStart(long ts) {
+ return ts / ONE_DAY_TS * ONE_DAY_TS;
+ }
+
+ public static long getNextPeriodStart(long ts, long period) {
+ return ((ts + period - 1) / period) * period;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/resources/kylinlog4j.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylinlog4j.properties b/core-common/src/main/resources/kylinlog4j.properties
new file mode 100644
index 0000000..7c3f26a
--- /dev/null
+++ b/core-common/src/main/resources/kylinlog4j.properties
@@ -0,0 +1,10 @@
+# use this when conflict with hbase, enable this by -Dlog4j.configuration=kylinlog4j.properties
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n
+
+#log4j.logger.org.apache.hadoop=ERROR
+log4j.logger.org.apache.kylin=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/log4j.properties b/core-common/src/main/resources/log4j.properties
new file mode 100644
index 0000000..65f2f68
--- /dev/null
+++ b/core-common/src/main/resources/log4j.properties
@@ -0,0 +1,10 @@
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=L4J [%d{HH:mm:ss,SSS}][%t][%p][%C{1}] - %m%n
+
+#log4j.logger.org.apache.hadoop=ERROR
+log4j.logger.org.apache.kylin=DEBUG
+log4j.logger.org.dbunit.database=ERROR
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
new file mode 100644
index 0000000..6195423
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.persistence;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.junit.Assert.*;
+
+public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testHBaseStore() throws Exception {
+ testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
+ }
+
+ @Test
+ public void testHBaseStoreWithLargeCell() throws Exception {
+ String path = "/cube/_test_large_cell.json";
+ String largeContent = "THIS_IS_A_LARGE_CELL";
+ StringEntity content = new StringEntity(largeContent);
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int origSize = config.getHBaseKeyValueSize();
+ ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+
+ try {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
+
+ store.deleteResource(path);
+
+ store.putResource(path, content, StringEntity.serializer);
+ assertTrue(store.exists(path));
+ StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
+ assertEquals(content, t);
+
+ Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
+ Configuration hconf = HadoopUtil.getCurrentConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+ assertTrue(fileSystem.exists(redirectPath));
+
+ FSDataInputStream in = fileSystem.open(redirectPath);
+ assertEquals(largeContent, in.readUTF());
+ in.close();
+
+ store.deleteResource(path);
+ } finally {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
+ store.deleteResource(path);
+ }
+ }
+
+ void testAStore(ResourceStore store) throws IOException {
+ String dir1 = "/cube";
+ String path1 = "/cube/_test.json";
+ StringEntity content1 = new StringEntity("anything");
+ String dir2 = "/table";
+ String path2 = "/table/_test.json";
+ StringEntity content2 = new StringEntity("something");
+
+ // cleanup legacy if any
+ store.deleteResource(path1);
+ store.deleteResource(path2);
+
+ StringEntity t;
+
+ // put/get
+ store.putResource(path1, content1, StringEntity.serializer);
+ assertTrue(store.exists(path1));
+ t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
+ assertEquals(content1, t);
+
+ store.putResource(path2, content2, StringEntity.serializer);
+ assertTrue(store.exists(path2));
+ t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
+ assertEquals(content2, t);
+
+ // overwrite
+ t.str = "new string";
+ store.putResource(path2, t, StringEntity.serializer);
+
+ // write conflict
+ try {
+ t.setLastModified(t.lastModified - 1);
+ store.putResource(path2, t, StringEntity.serializer);
+ fail("write conflict should trigger IllegalStateException");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ // list
+ ArrayList<String> list;
+
+ list = store.listResources(dir1);
+ assertTrue(list.contains(path1));
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(dir2);
+ assertTrue(list.contains(path2));
+ assertTrue(list.contains(path1) == false);
+
+ list = store.listResources("/");
+ assertTrue(list.contains(dir1));
+ assertTrue(list.contains(dir2));
+ assertTrue(list.contains(path1) == false);
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(path1);
+ assertNull(list);
+ list = store.listResources(path2);
+ assertNull(list);
+
+ // delete/exist
+ store.deleteResource(path1);
+ assertTrue(store.exists(path1) == false);
+ list = store.listResources(dir1);
+ assertTrue(list == null || list.contains(path1) == false);
+
+ store.deleteResource(path2);
+ assertTrue(store.exists(path2) == false);
+ list = store.listResources(dir2);
+ assertTrue(list == null || list.contains(path2) == false);
+ }
+
+ public static class StringEntity extends RootPersistentEntity {
+
+ static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
new file mode 100644
index 0000000..d687465
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.persistence;
+
+import static org.junit.Assert.*;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.commons.lang.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+
+public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testFileStore() throws Exception {
+ testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
+ }
+
+ void testAStore(ResourceStore store) throws IOException {
+ String dir1 = "/cube";
+ String path1 = "/cube/_test.json";
+ StringEntity content1 = new StringEntity("anything");
+ String dir2 = "/table";
+ String path2 = "/table/_test.json";
+ StringEntity content2 = new StringEntity("something");
+
+ // cleanup legacy if any
+ store.deleteResource(path1);
+ store.deleteResource(path2);
+
+ StringEntity t;
+
+ // put/get
+ store.putResource(path1, content1, StringEntity.serializer);
+ assertTrue(store.exists(path1));
+ t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
+ assertEquals(content1, t);
+
+ store.putResource(path2, content2, StringEntity.serializer);
+ assertTrue(store.exists(path2));
+ t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
+ assertEquals(content2, t);
+
+ // overwrite
+ t.str = "new string";
+ store.putResource(path2, t, StringEntity.serializer);
+
+ // write conflict
+ try {
+ t.setLastModified(t.lastModified - 1);
+ store.putResource(path2, t, StringEntity.serializer);
+ fail("write conflict should trigger IllegalStateException");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ // list
+ ArrayList<String> list;
+
+ list = store.listResources(dir1);
+ assertTrue(list.contains(path1));
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(dir2);
+ assertTrue(list.contains(path2));
+ assertTrue(list.contains(path1) == false);
+
+ list = store.listResources("/");
+ assertTrue(list.contains(dir1));
+ assertTrue(list.contains(dir2));
+ assertTrue(list.contains(path1) == false);
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(path1);
+ assertNull(list);
+ list = store.listResources(path2);
+ assertNull(list);
+
+ // delete/exist
+ store.deleteResource(path1);
+ assertTrue(store.exists(path1) == false);
+ list = store.listResources(dir1);
+ assertTrue(list == null || list.contains(path1) == false);
+
+ store.deleteResource(path2);
+ assertTrue(store.exists(path2) == false);
+ list = store.listResources(dir2);
+ assertTrue(list == null || list.contains(path2) == false);
+ }
+
+ public static class StringEntity extends RootPersistentEntity {
+
+ static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java b/core-common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
new file mode 100644
index 0000000..81e21cc
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.restclient;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class RestClientTest {
+
+ @SuppressWarnings("unused")
+ @Test
+ public void basicTests() throws IOException {
+ RestClient a = new RestClient("prod01:80");
+ //a.wipeCache("metadata", "a", "a");
+ //String aa = a.getKylinProperties();
+ //System.out.println(aa);
+ RestClient b = new RestClient("sandbox.hortonworks.com:7070");
+ //b.wipeCache("metadata", "a", "a");
+ //String bb = b.getKylinProperties();
+ //System.out.println(bb);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
new file mode 100644
index 0000000..b47816b
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.apache.kylin.common.KylinConfig;
+
+import java.lang.reflect.Method;
+
+/**
+ * @author ysong1
+ *
+ */
+public abstract class AbstractKylinTestCase {
+
+ public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
+
+ public static final String LOCALMETA_TEST_DATA_V1 = "../examples/test_case_data/localmeta_v1";
+
+ public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
+
+ public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
+
+ public static final String[] SERVICES_WITH_CACHE = {//
+ "org.apache.kylin.cube.CubeManager",//
+ "org.apache.kylin.cube.CubeDescManager", //
+ "org.apache.kylin.cube.cuboid.Cuboid", //
+ "org.apache.kylin.invertedindex.IIDescManager",//
+ "org.apache.kylin.invertedindex.IIManager",//
+ "org.apache.kylin.storage.hybrid.HybridManager",
+ "org.apache.kylin.metadata.realization.RealizationRegistry", //
+ "org.apache.kylin.metadata.project.ProjectManager", //
+ "org.apache.kylin.metadata.MetadataManager" //
+ };
+
+ public abstract void createTestMetadata() throws Exception;
+
+ public abstract void cleanupTestMetadata() throws Exception;
+
+ public static KylinConfig getTestConfig() {
+ return KylinConfig.getInstanceFromEnv();
+ }
+
+ public static void staticCleanupTestMetadata() {
+ cleanupCache();
+ System.clearProperty(KylinConfig.KYLIN_CONF);
+ KylinConfig.destoryInstance();
+
+ }
+
+ private static void cleanupCache() {
+
+ for (String serviceClass : SERVICES_WITH_CACHE) {
+ try {
+ Class<?> cls = Class.forName(serviceClass);
+ Method method = cls.getDeclaredMethod("clearCache");
+ method.invoke(null);
+ } catch (ClassNotFoundException e) {
+ // acceptable because lower module test does have CubeManager etc on classpath
+ } catch (Exception e) {
+ System.err.println("Error clean up cache " + serviceClass);
+ e.printStackTrace();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
new file mode 100644
index 0000000..2f0590b
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.TreeMultiset;
+import org.apache.commons.configuration.ConfigurationException;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.IdentityHashMap;
+
+/**
+* <p/>
+* Keep this test case to test basic java functionality
+* development concept proving use
+*/
+@Ignore("convenient trial tool for dev")
+@SuppressWarnings("unused")
+public class BasicTest {
+ protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(BasicTest.class);
+
+ private void log(ByteBuffer a) {
+ Integer x = 4;
+ foo(x);
+ }
+
+ private void foo(Long a) {
+ System.out.printf("a");
+ }
+
+ private void foo(Integer b) {
+ System.out.printf("b");
+ }
+
+ private enum MetricType {
+ Count, DimensionAsMetric, DistinctCount, Normal
+ }
+
+ @Test
+ public void test0() throws Exception {
+ TreeMultiset<Long> xx = TreeMultiset.create();
+ xx.add(2L);
+ xx.add(1L);
+ xx.add(1L);
+ for(Long hi : xx)
+ {
+ System.out.println(hi);
+ }
+ System.out.println(Long.MAX_VALUE);
+
+ IdentityHashMap<String, Void> a = new IdentityHashMap<>();
+ IdentityHashMap<String, Void> b = new IdentityHashMap<>();
+ String s1 = new String("s1");
+ String s2 = new String("s1");
+ Assert.assertEquals(s1, s2);
+ Assert.assertTrue(s1 != s2);
+ a.put(s1, null);
+ b.put(s2, null);
+ }
+
+ @Test
+ @Ignore("convenient trial tool for dev")
+ public void test1() throws Exception {
+ System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433833611000L));
+ System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433250517000L));
+ System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-06-01 00:00:00"));
+ System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-05-15 17:00:00"));
+
+ String bb = "\\x00\\x00\\x00\\x00\\x01\\x3F\\xD0\\x2D\\58\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/07/12 07:59:37
+ String cc = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/10/16 08:00:00
+ String dd = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x07\\x00\\x18\\x00\\x00\\x00";
+
+ byte[] bytes = BytesUtil.fromReadableText(dd);
+ long ttt = BytesUtil.readLong(bytes, 2, 8);
+ System.out.println(time(ttt));
+
+ System.out.println("\\");
+ System.out.println("n");
+
+ System.out.println("The start key is set to " + null);
+ long current = System.currentTimeMillis();
+ System.out.println(time(current));
+
+ Calendar a = Calendar.getInstance();
+ Calendar b = Calendar.getInstance();
+ Calendar c = Calendar.getInstance();
+ b.clear();
+ c.clear();
+
+ System.out.println(time(b.getTimeInMillis()));
+ System.out.println(time(c.getTimeInMillis()));
+
+ a.setTimeInMillis(current);
+ b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), a.get(Calendar.MINUTE));
+ c.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), 0);
+
+ System.out.println(time(b.getTimeInMillis()));
+ System.out.println(time(c.getTimeInMillis()));
+
+ }
+
+ @Test
+ @Ignore("fix it later")
+ public void test2() throws IOException, ConfigurationException {
+ ArrayList<String> x = Lists.newArrayListWithCapacity(10);
+ x.set(2, "dd");
+ for (String y : x) {
+ System.out.println(y);
+ }
+ }
+
+ private static String time(long t) {
+ DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+ Calendar cal = Calendar.getInstance();
+ cal.setTimeInMillis(t);
+ return dateFormat.format(cal.getTime());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
new file mode 100644
index 0000000..7d4dea9
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * by honma
+ */
+public class BytesUtilTest extends TestCase {
+ @Test
+ public void test() {
+ ByteBuffer buffer = ByteBuffer.allocate(10000);
+ int[] x = new int[] { 1, 2, 3 };
+ BytesUtil.writeIntArray(x, buffer);
+ buffer.flip();
+
+ byte[] buf = new byte[buffer.limit()];
+ System.arraycopy(buffer.array(), 0, buf, 0, buffer.limit());
+
+ ByteBuffer newBuffer = ByteBuffer.wrap(buf);
+ int[] y = BytesUtil.readIntArray(newBuffer);
+ assertEquals(y[2], 3);
+ }
+
+ @Test
+ public void testBooleanArray() {
+ ByteBuffer buffer = ByteBuffer.allocate(10000);
+ boolean[] x = new boolean[] { true, false, true };
+ BytesUtil.writeBooleanArray(x, buffer);
+ buffer.flip();
+ boolean[] y = BytesUtil.readBooleanArray(buffer);
+ assertEquals(y[2], true);
+ assertEquals(y[1], false);
+ }
+
+ @Test
+ public void testWriteReadUnsignedInt() {
+ testWriteReadUnsignedInt(735033, 3);
+ testWriteReadUnsignedInt(73503300, 4);
+ }
+
+
+ public void testWriteReadUnsignedInt(int testInt, int length) {
+ ByteArray ba = new ByteArray(new byte[length]);
+ BytesUtil.writeUnsigned(testInt, length, ba.asBuffer());
+
+ byte[] newBytes = new byte[length];
+ System.arraycopy(ba.array(), 0, newBytes, 0, length);
+ int value = BytesUtil.readUnsigned(new ByteArray(newBytes).asBuffer(), length);
+
+ assertEquals(value, testInt);
+
+ byte[] anOtherNewBytes = new byte[length];
+ BytesUtil.writeUnsigned(testInt, anOtherNewBytes, 0, length);
+
+ assertTrue(Arrays.equals(anOtherNewBytes, ba.array()));
+ }
+
+
+
+ public void testReadable()
+ {
+ String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
+ byte[] bytes = BytesUtil.fromReadableText(x);
+ String y = BytesUtil.toHex(bytes);
+ assertEquals(x,y);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
new file mode 100644
index 0000000..6473b3d
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.apache.kylin.common.KylinConfig;
+
+import java.io.File;
+
+/**
+ * @author ysong1
+ */
+public class HBaseMetadataTestCase extends AbstractKylinTestCase {
+
+ static {
+ if (useSandbox()) {
+ try {
+ ClassUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public void createTestMetadata() throws Exception {
+ staticCreateTestMetadata();
+ }
+
+ @Override
+ public void cleanupTestMetadata() {
+ staticCleanupTestMetadata();
+ }
+
+ public static void staticCreateTestMetadata() throws Exception {
+ if (useSandbox()) {
+ staticCreateTestMetadata(SANDBOX_TEST_DATA);
+ } else {
+ staticCreateTestMetadata(MINICLUSTER_TEST_DATA);
+ HBaseMiniclusterHelper.startupMinicluster();
+ }
+
+ }
+ public static void staticCreateTestMetadata(String kylinConfigFolder) {
+
+ KylinConfig.destoryInstance();
+
+ if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
+ System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder);
+
+ }
+
+ public static boolean useSandbox() {
+ String useSandbox = System.getProperty("useSandbox");
+ return Boolean.parseBoolean(useSandbox);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/core-common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java b/core-common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
new file mode 100644
index 0000000..fb0d313
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.HBaseResourceStore;
+
+/**
+ * a helper class to start and shutdown hbase mini cluster
+ *
+ * @author shaoshi
+ */
+public class HBaseMiniclusterHelper {
+
+ public static final String SHARED_STORAGE_PREFIX = "KYLIN_";
+ public static final String CUBE_STORAGE_PREFIX = "KYLIN_";
+ public static final String II_STORAGE_PREFIX = "KYLIN_II_";
+ public static final String TEST_METADATA_TABLE = "kylin_metadata";
+
+ private static final String hbaseTarLocation = "../examples/test_case_data/minicluster/hbase-export.tar.gz";
+ private static final String iiEndpointClassName = "org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
+
+ public static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static volatile boolean clusterStarted = false;
+ private static String hbaseconnectionUrl = "";
+
+ private static final Log logger = LogFactory.getLog(HBaseMiniclusterHelper.class);
+
+ static {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ shutdownMiniCluster();
+ }
+ });
+ }
+
+ /**
+ * Start the minicluster; Sub-classes should invoke this in BeforeClass method.
+ *
+ * @throws Exception
+ */
+ public static void startupMinicluster() throws Exception {
+
+ if (!clusterStarted) {
+ synchronized (HBaseMiniclusterHelper.class) {
+ if (!clusterStarted) {
+ startupMiniClusterAndImportData();
+ clusterStarted = true;
+ }
+ }
+ } else {
+ updateKylinConfigWithMinicluster();
+ }
+ }
+
+ private static void updateKylinConfigWithMinicluster() {
+
+ KylinConfig.getInstanceFromEnv().setMetadataUrl(TEST_METADATA_TABLE + "@" + hbaseconnectionUrl);
+ KylinConfig.getInstanceFromEnv().setStorageUrl(hbaseconnectionUrl);
+ }
+
+ private static void startupMiniClusterAndImportData() throws Exception {
+
+ logger.info("Going to start mini cluster.");
+
+ if (existInClassPath(iiEndpointClassName)) {
+ HBaseMiniclusterHelper.UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, iiEndpointClassName);
+ }
+
+ //https://issues.apache.org/jira/browse/HBASE-11711
+ UTIL.getConfiguration().setInt("hbase.master.info.port", -1);//avoid port clobbering
+
+ MiniHBaseCluster hbaseCluster = UTIL.startMiniCluster();
+
+ Configuration config = hbaseCluster.getConf();
+ String host = config.get(HConstants.ZOOKEEPER_QUORUM);
+ String port = config.get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ String parent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
+
+ // see in: https://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired
+ config.set("zookeeper.session.timeout", "1200000");
+ config.set("hbase.zookeeper.property.tickTime", "6000");
+ // reduce rpc retry
+ config.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
+ config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1");
+ config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
+
+ hbaseconnectionUrl = "hbase:" + host + ":" + port + ":" + parent;
+ updateKylinConfigWithMinicluster();
+
+ UTIL.startMiniMapReduceCluster();
+
+ // create the metadata htables;
+ @SuppressWarnings("unused")
+ HBaseResourceStore store = new HBaseResourceStore(KylinConfig.getInstanceFromEnv());
+
+ // import the table content
+ HbaseImporter.importHBaseData(hbaseTarLocation, UTIL.getConfiguration());
+
+ }
+
+ private static boolean existInClassPath(String className) {
+ try {
+ Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Shutdown the minicluster;
+ */
+ public static void shutdownMiniCluster() {
+
+ logger.info("Going to shutdown mini cluster.");
+
+ try {
+ UTIL.shutdownMiniMapReduceCluster();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ try {
+ UTIL.shutdownMiniCluster();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void main(String[] args) {
+ HBaseMiniclusterHelper t = new HBaseMiniclusterHelper();
+ logger.info(t);
+ try {
+ HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.MINICLUSTER_TEST_DATA);
+ HBaseMiniclusterHelper.startupMinicluster();
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ HBaseMiniclusterHelper.shutdownMiniCluster();
+ }
+ }
+}