You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/22 06:13:31 UTC
[41/47] incubator-kylin git commit: KYLIN-875 rename modules:
core-common, core-cube, core-dictionary, core-cube
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java b/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
deleted file mode 100644
index f73f525..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/PartialSorter.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import com.google.common.collect.Lists;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-
-/**
- *
- * This utility class sorts only the specified part of a list
- */
-public class PartialSorter {
- public static <T> void partialSort(List<T> list, List<Integer> items, Comparator<? super T> c) {
- List<T> temp = Lists.newLinkedList();
- for (int index : items) {
- temp.add(list.get(index));
- }
- Collections.sort(temp, c);
- for (int i = 0; i < temp.size(); ++i) {
- list.set(items.get(i), temp.get(i));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java b/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
deleted file mode 100644
index f793fdc..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/RandomSampler.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-/**
- * @author ysong1
- *
- */
-public class RandomSampler<T> {
-
- private Random rdm = new Random();
-
- public List<T> sample(List<T> data, int sampleNumber) {
- if (data == null) {
- throw new IllegalArgumentException("Input list is null");
- }
- if (data.size() < sampleNumber) {
- return data;
- }
-
- List<T> result = new ArrayList<T>(sampleNumber);
- int n = data.size();
- for (int i = 0; i < n; i++) {
- if (i < sampleNumber) {
- result.add(data.get(i));
- } else {
- int j = rdm.nextInt(i);
- if (j < sampleNumber) {
- result.set(j, data.get(i));
- }
- }
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java b/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
deleted file mode 100644
index 05e4d35..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package org.apache.kylin.common.util;
-
-import com.google.common.collect.*;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.NavigableMap;
-
-/**
- */
-public class RangeUtil {
-
- /**
- * for NavigableMap sorted by C, given a range of C, return the sub map whose key fails in the range
- */
- public static <C extends Comparable<?>, V> NavigableMap<C, V> filter(NavigableMap<C, V> values, Range<C> filterRange) {
- if (filterRange == null || filterRange.isEmpty()) {
- return Maps.newTreeMap();
- } else if (filterRange.equals(Ranges.all())) {
- return values;
- }
-
- if (filterRange.hasUpperBound() && !filterRange.hasLowerBound()) {
- return values.headMap(filterRange.upperEndpoint(), upperBoundInclusive(filterRange));
- } else if (filterRange.hasLowerBound() && !filterRange.hasUpperBound()) {
- return values.tailMap(filterRange.lowerEndpoint(), lowerBoundInclusive(filterRange));
- } else {
- return values.subMap(filterRange.lowerEndpoint(), lowerBoundInclusive(filterRange),//
- filterRange.upperEndpoint(), upperBoundInclusive(filterRange));
- }
- }
-
- public static <C extends Comparable<?>> boolean lowerBoundInclusive(Range<C> range) {
- if (!range.hasLowerBound()) {
- throw new IllegalArgumentException(("This range does not have lower bound" + range));
- }
- return range.lowerBoundType() == BoundType.CLOSED;
- }
-
- public static <C extends Comparable<?>> boolean upperBoundInclusive(Range<C> range) {
- if (!range.hasUpperBound()) {
- throw new IllegalArgumentException(("This range does not have upper bound" + range));
- }
- return range.upperBoundType() == BoundType.CLOSED;
- }
-
- public static <C extends Comparable<?>> Range<C> merge(Range<C> a, Range<C> b) {
- if (a == null && b == null) {
- return null;
- } else if (a == null || b == null) {
- return a == null ? b : a;
- } else {
- return a.span(b);
- }
- }
-
- /**
- * remove from self the elements that exist in other
- * @return
- */
- public static <C extends Comparable<?>> List<Range<C>> remove(Range<C> self, Range<C> other) {
- // mimic the following logic in guava 18:
- // RangeSet<C> rangeSet = TreeRangeSet.create();
- // rangeSet.add(self);
- // rangeSet.remove(other);
- // return Lists.newArrayList(rangeSet.asRanges());
-
- if (!self.isConnected(other)) {
- return Collections.singletonList(self);
- }
- Range<C> share = self.intersection(other);
- if (share.isEmpty()) {
- return Collections.singletonList(self);
- }
-
- List<Range<C>> ret = Lists.newArrayList();
-
- //see left part
- if (!self.hasLowerBound()) {
- if (share.hasLowerBound()) {
- if (share.lowerBoundType() == BoundType.CLOSED) {
- ret.add(Ranges.lessThan(share.lowerEndpoint()));
- } else {
- ret.add(Ranges.atMost(share.lowerEndpoint()));
- }
- }
- } else {
- if (self.lowerEndpoint() != share.lowerEndpoint()) {
- if (self.lowerBoundType() == BoundType.CLOSED) {
- if (share.lowerBoundType() == BoundType.CLOSED) {
- ret.add(Ranges.closedOpen(self.lowerEndpoint(), share.lowerEndpoint()));
- } else {
- ret.add(Ranges.closed(self.lowerEndpoint(), share.lowerEndpoint()));
- }
- } else {
- if (share.lowerBoundType() == BoundType.CLOSED) {
- ret.add(Ranges.open(self.lowerEndpoint(), share.lowerEndpoint()));
- } else {
- ret.add(Ranges.openClosed(self.lowerEndpoint(), share.lowerEndpoint()));
- }
- }
- } else {
- if (self.lowerBoundType() == BoundType.CLOSED && share.lowerBoundType() == BoundType.OPEN) {
- ret.add(Ranges.closed(self.lowerEndpoint(), share.lowerEndpoint()));
- }
- }
- }
-
- //see right part
- if (!self.hasUpperBound()) {
- if (share.hasUpperBound()) {
- if (share.upperBoundType() == BoundType.CLOSED) {
- ret.add(Ranges.greaterThan(share.upperEndpoint()));
- } else {
- ret.add(Ranges.atLeast(share.upperEndpoint()));
- }
- }
- } else {
- if (self.upperEndpoint() != share.upperEndpoint()) {
- if (self.upperBoundType() == BoundType.CLOSED) {
- if (share.upperBoundType() == BoundType.CLOSED) {
- ret.add(Ranges.openClosed(share.upperEndpoint(), self.upperEndpoint()));
- } else {
- ret.add(Ranges.closed(share.upperEndpoint(), self.upperEndpoint()));
- }
- } else {
- if (share.upperBoundType() == BoundType.CLOSED) {
- ret.add(Ranges.open(share.upperEndpoint(), self.upperEndpoint()));
- } else {
- ret.add(Ranges.closedOpen(share.upperEndpoint(), self.upperEndpoint()));
- }
- }
- } else {
- if (self.upperBoundType() == BoundType.CLOSED && share.upperBoundType() == BoundType.OPEN) {
- ret.add(Ranges.closed(self.upperEndpoint(), share.upperEndpoint()));
- }
- }
- }
-
- return ret;
-
- }
-
- public static String formatTsRange(Range<Long> tsRange) {
- if (tsRange == null)
- return null;
-
- StringBuilder sb = new StringBuilder();
- if (tsRange.hasLowerBound()) {
- if (tsRange.lowerBoundType() == BoundType.CLOSED) {
- sb.append("[");
- } else {
- sb.append("(");
- }
- sb.append(DateFormat.formatToTimeStr(tsRange.lowerEndpoint()));
- } else {
- sb.append("(-∞");
- }
-
- sb.append("~");
-
- if (tsRange.hasUpperBound()) {
- sb.append(DateFormat.formatToTimeStr(tsRange.upperEndpoint()));
- if (tsRange.upperBoundType() == BoundType.CLOSED) {
- sb.append("]");
- } else {
- sb.append(")");
- }
- } else {
- sb.append("+∞)");
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java b/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
deleted file mode 100644
index 32eb72a..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.slf4j.LoggerFactory;
-
-import com.jcraft.jsch.Channel;
-import com.jcraft.jsch.ChannelExec;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.Session;
-
-public class SSHClient {
- protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(SSHClient.class);
-
- private String hostname;
- private int port;
- private String username;
- private String password;
- private String identityPath;
-
- public SSHClient(String hostname, int port, String username, String password) {
- this.hostname = hostname;
- this.username = username;
- this.port = port;
- if (password != null && new File(password).exists()) {
- this.identityPath = new File(password).getAbsolutePath();
- this.password = null;
- } else {
- this.password = password;
- this.identityPath = null;
- }
- }
-
- public void scpFileToRemote(String localFile, String remoteTargetDirectory) throws Exception {
- FileInputStream fis = null;
- try {
- System.out.println("SCP file " + localFile + " to " + remoteTargetDirectory);
-
- Session session = newJSchSession();
- session.connect();
-
- boolean ptimestamp = false;
-
- // exec 'scp -t rfile' remotely
- String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteTargetDirectory;
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- // get I/O streams for remote scp
- OutputStream out = channel.getOutputStream();
- InputStream in = channel.getInputStream();
-
- channel.connect();
-
- if (checkAck(in) != 0) {
- System.exit(0);
- }
-
- File _lfile = new File(localFile);
-
- if (ptimestamp) {
- command = "T " + (_lfile.lastModified() / 1000) + " 0";
- // The access time should be sent here,
- // but it is not accessible with JavaAPI ;-<
- command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- }
-
- // send "C0644 filesize filename", where filename should not include '/'
- long filesize = _lfile.length();
- command = "C0644 " + filesize + " ";
- if (localFile.lastIndexOf("/") > 0) {
- command += localFile.substring(localFile.lastIndexOf("/") + 1);
- } else if (localFile.lastIndexOf(File.separator) > 0) {
- command += localFile.substring(localFile.lastIndexOf(File.separator) + 1);
- } else {
- command += localFile;
- }
- command += "\n";
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
-
- // send a content of lfile
- fis = new FileInputStream(localFile);
- byte[] buf = new byte[1024];
- while (true) {
- int len = fis.read(buf, 0, buf.length);
- if (len <= 0)
- break;
- out.write(buf, 0, len); // out.flush();
- }
- fis.close();
- fis = null;
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- out.close();
-
- channel.disconnect();
- session.disconnect();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- if (fis != null)
- fis.close();
- } catch (Exception ee) {
- }
- }
- }
-
- public void scpFileToLocal(String rfile, String lfile) throws Exception {
- FileOutputStream fos = null;
- try {
- System.out.println("SCP remote file " + rfile + " to local " + lfile);
-
- String prefix = null;
- if (new File(lfile).isDirectory()) {
- prefix = lfile + File.separator;
- }
-
- Session session = newJSchSession();
- session.connect();
- // exec 'scp -f rfile' remotely
- String command = "scp -f " + rfile;
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- // get I/O streams for remote scp
- OutputStream out = channel.getOutputStream();
- InputStream in = channel.getInputStream();
-
- channel.connect();
-
- byte[] buf = new byte[1024];
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
-
- while (true) {
- int c = checkAck(in);
- if (c != 'C') {
- break;
- }
-
- // read '0644 '
- in.read(buf, 0, 5);
-
- long filesize = 0L;
- while (true) {
- if (in.read(buf, 0, 1) < 0) {
- // error
- break;
- }
- if (buf[0] == ' ')
- break;
- filesize = filesize * 10L + (long) (buf[0] - '0');
- }
-
- String file = null;
- for (int i = 0;; i++) {
- in.read(buf, i, 1);
- if (buf[i] == (byte) 0x0a) {
- file = new String(buf, 0, i);
- break;
- }
- }
-
- //System.out.println("filesize="+filesize+", file="+file);
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
-
- // read a content of lfile
- fos = new FileOutputStream(prefix == null ? lfile : prefix + file);
- int foo;
- while (true) {
- if (buf.length < filesize)
- foo = buf.length;
- else
- foo = (int) filesize;
- foo = in.read(buf, 0, foo);
- if (foo < 0) {
- // error
- break;
- }
- fos.write(buf, 0, foo);
- filesize -= foo;
- if (filesize == 0L)
- break;
- }
- fos.close();
- fos = null;
-
- if (checkAck(in) != 0) {
- System.exit(0);
- }
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
- }
-
- session.disconnect();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- if (fos != null)
- fos.close();
- } catch (Exception ee) {
- }
- }
- }
-
- public SSHClientOutput execCommand(String command) throws Exception {
- return execCommand(command, 7200, null);
- }
-
- public SSHClientOutput execCommand(String command, int timeoutSeconds, Logger logAppender) throws Exception {
- try {
- System.out.println("[" + username + "@" + hostname + "] Execute command: " + command);
-
- StringBuffer text = new StringBuffer();
- int exitCode = -1;
-
- Session session = newJSchSession();
- session.connect();
-
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- channel.setInputStream(null);
-
- // channel.setOutputStream(System.out);
-
- ((ChannelExec) channel).setErrStream(System.err);
-
- InputStream in = channel.getInputStream();
- InputStream err = ((ChannelExec) channel).getErrStream();
-
- channel.connect();
-
- int timeout = timeoutSeconds;
- byte[] tmp = new byte[1024];
- while (true) {
- timeout--;
- while (in.available() > 0) {
- int i = in.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
- while (err.available() > 0) {
- int i = err.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
- if (channel.isClosed()) {
- if (in.available() > 0)
- continue;
- exitCode = channel.getExitStatus();
- System.out.println("[" + username + "@" + hostname + "] Command exit-status: " + exitCode);
-
- break;
- }
- try {
- Thread.sleep(1000);
- } catch (Exception ee) {
- throw ee;
- }
- if (timeout < 0)
- throw new Exception("Remote command not finished within " + timeoutSeconds + " seconds.");
- }
- channel.disconnect();
- session.disconnect();
- return new SSHClientOutput(exitCode, text.toString());
- } catch (Exception e) {
- throw e;
- }
- }
-
- private Session newJSchSession() throws JSchException {
- JSch jsch = new JSch();
- if (identityPath != null) {
- jsch.addIdentity(identityPath);
- }
-
- Session session = jsch.getSession(username, hostname, port);
- if (password != null) {
- session.setPassword(password);
- }
- session.setConfig("StrictHostKeyChecking", "no");
- return session;
- }
-
- private int checkAck(InputStream in) throws IOException {
- int b = in.read();
- // b may be 0 for success,
- // 1 for error,
- // 2 for fatal error,
- // -1
- if (b == 0)
- return b;
- if (b == -1)
- return b;
-
- if (b == 1 || b == 2) {
- StringBuffer sb = new StringBuffer();
- int c;
- do {
- c = in.read();
- sb.append((char) c);
- } while (c != '\n');
- if (b == 1) { // error
- System.out.print(sb.toString());
- }
- if (b == 2) { // fatal error
- System.out.print(sb.toString());
- }
- }
- return b;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java b/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
deleted file mode 100644
index cf25001..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-public class SSHClientOutput {
- private String text;
- private int exitCode = -1;
-
- /**
- * @param text
- * @param exitCode
- */
- public SSHClientOutput(int exitCode, String text) {
- this.text = text;
- this.exitCode = exitCode;
- }
-
- /**
- * @return the text
- */
- public String getText() {
- return text.toString();
- }
-
- /**
- * @return the exitCode
- */
- public int getExitCode() {
- return exitCode;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SortUtil.java b/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
deleted file mode 100644
index f0d7cdb..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SortUtil.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.apache.kylin.common.util;
-
-import com.google.common.base.Function;
-import com.google.common.collect.TreeMultimap;
-
-import java.util.Iterator;
-
-/**
- */
-public class SortUtil {
- public static <T extends Comparable, E extends Comparable> Iterator<T> extractAndSort(Iterator<T> input, Function<T, E> extractor) {
- TreeMultimap<E, T> reorgnized = TreeMultimap.create();
- while (input.hasNext()) {
- T t = input.next();
- E e = extractor.apply(t);
- reorgnized.put(e, t);
- }
- return reorgnized.values().iterator();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java b/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
deleted file mode 100644
index 0ed03a7..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.common.util;
-
-/**
- */
-public class SoutLogger implements Logger {
-
- @Override
- public void log(String message) {
- System.out.println(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
deleted file mode 100644
index 8751b78..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class SplittedBytes {
- public SplittedBytes(int length) {
- this.value = new byte[length];
- this.length = 0;
- }
-
- public byte[] value;
- public int length;
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java b/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
deleted file mode 100644
index 6df82d4..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class StringSplitter {
- public static String[] split(String str, String delimiter) {
- // The optimized split function
- List<String> list = new ArrayList<String>();
- int index = 0, offset = 0;
- int l = delimiter.length();
- if (str.startsWith(delimiter)) {
- // in case the first field is empty
- list.add("");
- offset = offset + l;
- }
- while ((index = str.indexOf(delimiter, index + 1)) != -1) {
- list.add(str.substring(offset, index));
- offset = index + l;
- }
- // add the last field, or the str doesn't contain delimiter at all
- list.add(str.substring(offset));
- return list.toArray(new String[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
deleted file mode 100644
index 468a6e1..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-/**
- */
-public class StringUtil {
-
- public static String[] filterSystemArgs(String args[]) {
- ArrayList<String> whatsLeft = new ArrayList<String>();
- for (String a : args) {
- if (a.startsWith("-D")) {
- String key;
- String value;
- int cut = a.indexOf('=');
- if (cut < 0) {
- key = a.substring(2);
- value = "";
- } else {
- key = a.substring(2, cut);
- value = a.substring(cut + 1);
- }
- System.setProperty(key, value);
- } else {
- whatsLeft.add(a);
- }
- }
- return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]);
- }
-
- public static String join(Iterable<String> parts, String separator) {
- StringBuilder buf = new StringBuilder();
- for (String p : parts) {
- if (buf.length() > 0)
- buf.append(separator);
- buf.append(p);
- }
- return buf.toString();
- }
-
- public static void toUpperCaseArray(String[] source, String[] target) {
- if(source!=null) {
- for (int i = 0; i < source.length; i++) {
- if (source[i] != null) {
- target[i] = source[i].toUpperCase();
- }
- }
- }
- }
-
- public static String dropSuffix(String str, String suffix) {
- if (str.endsWith(suffix))
- return str.substring(0, str.length() - suffix.length());
- else
- return str;
- }
-
- public static String min(Collection<String> strs) {
- String min = null;
- for (String s : strs) {
- if (min == null || min.compareTo(s) > 0)
- min = s;
- }
- return min;
- }
-
- public static String max(Collection<String> strs) {
- String max = null;
- for (String s : strs) {
- if (max == null || max.compareTo(s) < 0)
- max = s;
- }
- return max;
- }
-
- public static String min(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) < 0 ? s1 : s2;
- }
-
- public static String max(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) > 0 ? s1 : s2;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java b/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
deleted file mode 100644
index 6f623d3..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-
-public class TarGZUtil {
-
- public static void uncompressTarGZ(File tarFile, File dest) throws IOException {
- dest.mkdir();
- TarArchiveInputStream tarIn = null;
-
- tarIn = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(tarFile))));
-
- TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
- // tarIn is a TarArchiveInputStream
- while (tarEntry != null) {// create a file with the same name as the tarEntry
- File destPath = new File(dest, tarEntry.getName());
- System.out.println("working: " + destPath.getCanonicalPath());
- if (tarEntry.isDirectory()) {
- destPath.mkdirs();
- } else {
- destPath.createNewFile();
- //byte [] btoRead = new byte[(int)tarEntry.getSize()];
- byte[] btoRead = new byte[1024];
- //FileInputStream fin
- // = new FileInputStream(destPath.getCanonicalPath());
- BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(destPath));
- int len = 0;
-
- while ((len = tarIn.read(btoRead)) != -1) {
- bout.write(btoRead, 0, len);
- }
-
- bout.close();
- btoRead = null;
-
- }
- tarEntry = tarIn.getNextTarEntry();
- }
- tarIn.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
deleted file mode 100644
index c493424..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- */
-public class ThreadUtil {
- @SuppressWarnings("unused")
- public static void main(String[] args) {
- ThreadPoolExecutor pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());//Threads.newDaemonThreadFactory("htable"));
-
- for (int i = 0; i < Integer.MAX_VALUE; ++i) {
- System.out.println("index: " + i);
- Future<?> future = pool.submit(new Runnable() {
- @Override
- public void run() {
- try {
- Thread.sleep(10000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
deleted file mode 100644
index c79e88b..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.apache.kylin.common.util;
-
-/**
- */
-public class TimeUtil {
- public enum NormalizedTimeUnit {
- MINUTE, HOUR, DAY
- }
-
- private static long ONE_MINUTE_TS = 60 * 1000;
- private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS;
- private static long ONE_DAY_TS = 24 * ONE_HOUR_TS;
-
- public static long getMinuteStart(long ts) {
- return ts / ONE_MINUTE_TS * ONE_MINUTE_TS;
- }
-
- public static long getHourStart(long ts) {
- return ts / ONE_HOUR_TS * ONE_HOUR_TS;
- }
-
- public static long getDayStart(long ts) {
- return ts / ONE_DAY_TS * ONE_DAY_TS;
- }
-
- public static long getNextPeriodStart(long ts, long period) {
- return ((ts + period - 1) / period) * period;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/resources/kylinlog4j.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/kylinlog4j.properties b/common/src/main/resources/kylinlog4j.properties
deleted file mode 100644
index 7c3f26a..0000000
--- a/common/src/main/resources/kylinlog4j.properties
+++ /dev/null
@@ -1,10 +0,0 @@
-# use this when conflict with hbase, enable this by -Dlog4j.configuration=kylinlog4j.properties
-
-log4j.rootLogger=INFO,stdout
-
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n
-
-#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/log4j.properties b/common/src/main/resources/log4j.properties
deleted file mode 100644
index 65f2f68..0000000
--- a/common/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,10 +0,0 @@
-
-log4j.rootLogger=INFO,stdout
-
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=L4J [%d{HH:mm:ss,SSS}][%t][%p][%C{1}] - %m%n
-
-#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
-log4j.logger.org.dbunit.database=ERROR
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
deleted file mode 100644
index 6195423..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.persistence;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import static org.junit.Assert.*;
-
-public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testHBaseStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- @Test
- public void testHBaseStoreWithLargeCell() throws Exception {
- String path = "/cube/_test_large_cell.json";
- String largeContent = "THIS_IS_A_LARGE_CELL";
- StringEntity content = new StringEntity(largeContent);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int origSize = config.getHBaseKeyValueSize();
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- try {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
-
- store.deleteResource(path);
-
- store.putResource(path, content, StringEntity.serializer);
- assertTrue(store.exists(path));
- StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
- assertEquals(content, t);
-
- Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
- assertTrue(fileSystem.exists(redirectPath));
-
- FSDataInputStream in = fileSystem.open(redirectPath);
- assertEquals(largeContent, in.readUTF());
- in.close();
-
- store.deleteResource(path);
- } finally {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
- store.deleteResource(path);
- }
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
deleted file mode 100644
index d687465..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.*;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-
-public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testFileStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java b/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
deleted file mode 100644
index 81e21cc..0000000
--- a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.restclient;
-
-import org.junit.Test;
-
-import java.io.IOException;
-
-public class RestClientTest {
-
- @SuppressWarnings("unused")
- @Test
- public void basicTests() throws IOException {
- RestClient a = new RestClient("prod01:80");
- //a.wipeCache("metadata", "a", "a");
- //String aa = a.getKylinProperties();
- //System.out.println(aa);
- RestClient b = new RestClient("sandbox.hortonworks.com:7070");
- //b.wipeCache("metadata", "a", "a");
- //String bb = b.getKylinProperties();
- //System.out.println(bb);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
deleted file mode 100644
index b47816b..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import org.apache.kylin.common.KylinConfig;
-
-import java.lang.reflect.Method;
-
-/**
- * @author ysong1
- *
- */
-public abstract class AbstractKylinTestCase {
-
- public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
-
- public static final String LOCALMETA_TEST_DATA_V1 = "../examples/test_case_data/localmeta_v1";
-
- public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
-
- public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
-
- public static final String[] SERVICES_WITH_CACHE = {//
- "org.apache.kylin.cube.CubeManager",//
- "org.apache.kylin.cube.CubeDescManager", //
- "org.apache.kylin.cube.cuboid.Cuboid", //
- "org.apache.kylin.invertedindex.IIDescManager",//
- "org.apache.kylin.invertedindex.IIManager",//
- "org.apache.kylin.storage.hybrid.HybridManager",
- "org.apache.kylin.metadata.realization.RealizationRegistry", //
- "org.apache.kylin.metadata.project.ProjectManager", //
- "org.apache.kylin.metadata.MetadataManager" //
- };
-
- public abstract void createTestMetadata() throws Exception;
-
- public abstract void cleanupTestMetadata() throws Exception;
-
- public static KylinConfig getTestConfig() {
- return KylinConfig.getInstanceFromEnv();
- }
-
- public static void staticCleanupTestMetadata() {
- cleanupCache();
- System.clearProperty(KylinConfig.KYLIN_CONF);
- KylinConfig.destoryInstance();
-
- }
-
- private static void cleanupCache() {
-
- for (String serviceClass : SERVICES_WITH_CACHE) {
- try {
- Class<?> cls = Class.forName(serviceClass);
- Method method = cls.getDeclaredMethod("clearCache");
- method.invoke(null);
- } catch (ClassNotFoundException e) {
- // acceptable because lower module test does have CubeManager etc on classpath
- } catch (Exception e) {
- System.err.println("Error clean up cache " + serviceClass);
- e.printStackTrace();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
deleted file mode 100644
index 2f0590b..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.TreeMultiset;
-import org.apache.commons.configuration.ConfigurationException;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.IdentityHashMap;
-
-/**
-* <p/>
-* Keep this test case to test basic java functionality
-* development concept proving use
-*/
-@Ignore("convenient trial tool for dev")
-@SuppressWarnings("unused")
-public class BasicTest {
- protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(BasicTest.class);
-
- private void log(ByteBuffer a) {
- Integer x = 4;
- foo(x);
- }
-
- private void foo(Long a) {
- System.out.printf("a");
- }
-
- private void foo(Integer b) {
- System.out.printf("b");
- }
-
- private enum MetricType {
- Count, DimensionAsMetric, DistinctCount, Normal
- }
-
- @Test
- public void test0() throws Exception {
- TreeMultiset<Long> xx = TreeMultiset.create();
- xx.add(2L);
- xx.add(1L);
- xx.add(1L);
- for(Long hi : xx)
- {
- System.out.println(hi);
- }
- System.out.println(Long.MAX_VALUE);
-
- IdentityHashMap<String, Void> a = new IdentityHashMap<>();
- IdentityHashMap<String, Void> b = new IdentityHashMap<>();
- String s1 = new String("s1");
- String s2 = new String("s1");
- Assert.assertEquals(s1, s2);
- Assert.assertTrue(s1 != s2);
- a.put(s1, null);
- b.put(s2, null);
- }
-
- @Test
- @Ignore("convenient trial tool for dev")
- public void test1() throws Exception {
- System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433833611000L));
- System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433250517000L));
- System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-06-01 00:00:00"));
- System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-05-15 17:00:00"));
-
- String bb = "\\x00\\x00\\x00\\x00\\x01\\x3F\\xD0\\x2D\\58\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/07/12 07:59:37
- String cc = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/10/16 08:00:00
- String dd = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x07\\x00\\x18\\x00\\x00\\x00";
-
- byte[] bytes = BytesUtil.fromReadableText(dd);
- long ttt = BytesUtil.readLong(bytes, 2, 8);
- System.out.println(time(ttt));
-
- System.out.println("\\");
- System.out.println("n");
-
- System.out.println("The start key is set to " + null);
- long current = System.currentTimeMillis();
- System.out.println(time(current));
-
- Calendar a = Calendar.getInstance();
- Calendar b = Calendar.getInstance();
- Calendar c = Calendar.getInstance();
- b.clear();
- c.clear();
-
- System.out.println(time(b.getTimeInMillis()));
- System.out.println(time(c.getTimeInMillis()));
-
- a.setTimeInMillis(current);
- b.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), a.get(Calendar.MINUTE));
- c.set(a.get(Calendar.YEAR), a.get(Calendar.MONTH), a.get(Calendar.DAY_OF_MONTH), a.get(Calendar.HOUR_OF_DAY), 0);
-
- System.out.println(time(b.getTimeInMillis()));
- System.out.println(time(c.getTimeInMillis()));
-
- }
-
- @Test
- @Ignore("fix it later")
- public void test2() throws IOException, ConfigurationException {
- ArrayList<String> x = Lists.newArrayListWithCapacity(10);
- x.set(2, "dd");
- for (String y : x) {
- System.out.println(y);
- }
- }
-
- private static String time(long t) {
- DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
- Calendar cal = Calendar.getInstance();
- cal.setTimeInMillis(t);
- return dateFormat.format(cal.getTime());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
deleted file mode 100644
index 7d4dea9..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import junit.framework.TestCase;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-/**
- * by honma
- */
-public class BytesUtilTest extends TestCase {
- @Test
- public void test() {
- ByteBuffer buffer = ByteBuffer.allocate(10000);
- int[] x = new int[] { 1, 2, 3 };
- BytesUtil.writeIntArray(x, buffer);
- buffer.flip();
-
- byte[] buf = new byte[buffer.limit()];
- System.arraycopy(buffer.array(), 0, buf, 0, buffer.limit());
-
- ByteBuffer newBuffer = ByteBuffer.wrap(buf);
- int[] y = BytesUtil.readIntArray(newBuffer);
- assertEquals(y[2], 3);
- }
-
- @Test
- public void testBooleanArray() {
- ByteBuffer buffer = ByteBuffer.allocate(10000);
- boolean[] x = new boolean[] { true, false, true };
- BytesUtil.writeBooleanArray(x, buffer);
- buffer.flip();
- boolean[] y = BytesUtil.readBooleanArray(buffer);
- assertEquals(y[2], true);
- assertEquals(y[1], false);
- }
-
- @Test
- public void testWriteReadUnsignedInt() {
- testWriteReadUnsignedInt(735033, 3);
- testWriteReadUnsignedInt(73503300, 4);
- }
-
-
- public void testWriteReadUnsignedInt(int testInt, int length) {
- ByteArray ba = new ByteArray(new byte[length]);
- BytesUtil.writeUnsigned(testInt, length, ba.asBuffer());
-
- byte[] newBytes = new byte[length];
- System.arraycopy(ba.array(), 0, newBytes, 0, length);
- int value = BytesUtil.readUnsigned(new ByteArray(newBytes).asBuffer(), length);
-
- assertEquals(value, testInt);
-
- byte[] anOtherNewBytes = new byte[length];
- BytesUtil.writeUnsigned(testInt, anOtherNewBytes, 0, length);
-
- assertTrue(Arrays.equals(anOtherNewBytes, ba.array()));
- }
-
-
-
- public void testReadable()
- {
- String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
- byte[] bytes = BytesUtil.fromReadableText(x);
- String y = BytesUtil.toHex(bytes);
- assertEquals(x,y);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
deleted file mode 100644
index 6473b3d..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import org.apache.kylin.common.KylinConfig;
-
-import java.io.File;
-
-/**
- * @author ysong1
- */
-public class HBaseMetadataTestCase extends AbstractKylinTestCase {
-
- static {
- if (useSandbox()) {
- try {
- ClassUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath());
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
-
- @Override
- public void createTestMetadata() throws Exception {
- staticCreateTestMetadata();
- }
-
- @Override
- public void cleanupTestMetadata() {
- staticCleanupTestMetadata();
- }
-
- public static void staticCreateTestMetadata() throws Exception {
- if (useSandbox()) {
- staticCreateTestMetadata(SANDBOX_TEST_DATA);
- } else {
- staticCreateTestMetadata(MINICLUSTER_TEST_DATA);
- HBaseMiniclusterHelper.startupMinicluster();
- }
-
- }
- public static void staticCreateTestMetadata(String kylinConfigFolder) {
-
- KylinConfig.destoryInstance();
-
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder);
-
- }
-
- public static boolean useSandbox() {
- String useSandbox = System.getProperty("useSandbox");
- return Boolean.parseBoolean(useSandbox);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
deleted file mode 100644
index fb0d313..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseResourceStore;
-
-/**
- * a helper class to start and shutdown hbase mini cluster
- *
- * @author shaoshi
- */
-public class HBaseMiniclusterHelper {
-
- public static final String SHARED_STORAGE_PREFIX = "KYLIN_";
- public static final String CUBE_STORAGE_PREFIX = "KYLIN_";
- public static final String II_STORAGE_PREFIX = "KYLIN_II_";
- public static final String TEST_METADATA_TABLE = "kylin_metadata";
-
- private static final String hbaseTarLocation = "../examples/test_case_data/minicluster/hbase-export.tar.gz";
- private static final String iiEndpointClassName = "org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
-
- public static HBaseTestingUtility UTIL = new HBaseTestingUtility();
- private static volatile boolean clusterStarted = false;
- private static String hbaseconnectionUrl = "";
-
- private static final Log logger = LogFactory.getLog(HBaseMiniclusterHelper.class);
-
- static {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- shutdownMiniCluster();
- }
- });
- }
-
- /**
- * Start the minicluster; Sub-classes should invoke this in BeforeClass method.
- *
- * @throws Exception
- */
- public static void startupMinicluster() throws Exception {
-
- if (!clusterStarted) {
- synchronized (HBaseMiniclusterHelper.class) {
- if (!clusterStarted) {
- startupMiniClusterAndImportData();
- clusterStarted = true;
- }
- }
- } else {
- updateKylinConfigWithMinicluster();
- }
- }
-
- private static void updateKylinConfigWithMinicluster() {
-
- KylinConfig.getInstanceFromEnv().setMetadataUrl(TEST_METADATA_TABLE + "@" + hbaseconnectionUrl);
- KylinConfig.getInstanceFromEnv().setStorageUrl(hbaseconnectionUrl);
- }
-
- private static void startupMiniClusterAndImportData() throws Exception {
-
- logger.info("Going to start mini cluster.");
-
- if (existInClassPath(iiEndpointClassName)) {
- HBaseMiniclusterHelper.UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, iiEndpointClassName);
- }
-
- //https://issues.apache.org/jira/browse/HBASE-11711
- UTIL.getConfiguration().setInt("hbase.master.info.port", -1);//avoid port clobbering
-
- MiniHBaseCluster hbaseCluster = UTIL.startMiniCluster();
-
- Configuration config = hbaseCluster.getConf();
- String host = config.get(HConstants.ZOOKEEPER_QUORUM);
- String port = config.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- String parent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
-
- // see in: https://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired
- config.set("zookeeper.session.timeout", "1200000");
- config.set("hbase.zookeeper.property.tickTime", "6000");
- // reduce rpc retry
- config.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1");
- config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
-
- hbaseconnectionUrl = "hbase:" + host + ":" + port + ":" + parent;
- updateKylinConfigWithMinicluster();
-
- UTIL.startMiniMapReduceCluster();
-
- // create the metadata htables;
- @SuppressWarnings("unused")
- HBaseResourceStore store = new HBaseResourceStore(KylinConfig.getInstanceFromEnv());
-
- // import the table content
- HbaseImporter.importHBaseData(hbaseTarLocation, UTIL.getConfiguration());
-
- }
-
- private static boolean existInClassPath(String className) {
- try {
- Class.forName(className);
- } catch (ClassNotFoundException e) {
- return false;
- }
- return true;
- }
-
- /**
- * Shutdown the minicluster;
- */
- public static void shutdownMiniCluster() {
-
- logger.info("Going to shutdown mini cluster.");
-
- try {
- UTIL.shutdownMiniMapReduceCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- try {
- UTIL.shutdownMiniCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- public static void main(String[] args) {
- HBaseMiniclusterHelper t = new HBaseMiniclusterHelper();
- logger.info(t);
- try {
- HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.MINICLUSTER_TEST_DATA);
- HBaseMiniclusterHelper.startupMinicluster();
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- HBaseMiniclusterHelper.shutdownMiniCluster();
- }
- }
-}