You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/11 13:50:21 UTC
[42/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java b/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
deleted file mode 100644
index 027045a..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SSHClient.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.slf4j.LoggerFactory;
-
-import com.jcraft.jsch.Channel;
-import com.jcraft.jsch.ChannelExec;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.Session;
-
-public class SSHClient {
- protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(SSHClient.class);
-
- private String hostname;
- private String username;
- private String password;
- private String identityPath;
-
- public SSHClient(String hostname, String username, String password) {
- this.hostname = hostname;
- this.username = username;
- if (password != null && new File(password).exists()) {
- this.identityPath = new File(password).getAbsolutePath();
- this.password = null;
- } else {
- this.password = password;
- this.identityPath = null;
- }
- }
-
- public void scpFileToRemote(String localFile, String remoteTargetDirectory) throws Exception {
- FileInputStream fis = null;
- try {
- System.out.println("SCP file " + localFile + " to " + remoteTargetDirectory);
-
- Session session = newJSchSession();
- session.connect();
-
- boolean ptimestamp = false;
-
- // exec 'scp -t rfile' remotely
- String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteTargetDirectory;
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- // get I/O streams for remote scp
- OutputStream out = channel.getOutputStream();
- InputStream in = channel.getInputStream();
-
- channel.connect();
-
- if (checkAck(in) != 0) {
- System.exit(0);
- }
-
- File _lfile = new File(localFile);
-
- if (ptimestamp) {
- command = "T " + (_lfile.lastModified() / 1000) + " 0";
- // The access time should be sent here,
- // but it is not accessible with JavaAPI ;-<
- command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- }
-
- // send "C0644 filesize filename", where filename should not include '/'
- long filesize = _lfile.length();
- command = "C0644 " + filesize + " ";
- if (localFile.lastIndexOf("/") > 0) {
- command += localFile.substring(localFile.lastIndexOf("/") + 1);
- } else if (localFile.lastIndexOf(File.separator) > 0) {
- command += localFile.substring(localFile.lastIndexOf(File.separator) + 1);
- } else {
- command += localFile;
- }
- command += "\n";
- out.write(command.getBytes());
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
-
- // send a content of lfile
- fis = new FileInputStream(localFile);
- byte[] buf = new byte[1024];
- while (true) {
- int len = fis.read(buf, 0, buf.length);
- if (len <= 0)
- break;
- out.write(buf, 0, len); // out.flush();
- }
- fis.close();
- fis = null;
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
- if (checkAck(in) != 0) {
- throw new Exception("Error in checkAck()");
- }
- out.close();
-
- channel.disconnect();
- session.disconnect();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- if (fis != null)
- fis.close();
- } catch (Exception ee) {
- }
- }
- }
-
- public void scpFileToLocal(String rfile, String lfile) throws Exception {
- FileOutputStream fos = null;
- try {
- System.out.println("SCP remote file " + rfile + " to local " + lfile);
-
- String prefix = null;
- if (new File(lfile).isDirectory()) {
- prefix = lfile + File.separator;
- }
-
- Session session = newJSchSession();
- session.connect();
- // exec 'scp -f rfile' remotely
- String command = "scp -f " + rfile;
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- // get I/O streams for remote scp
- OutputStream out = channel.getOutputStream();
- InputStream in = channel.getInputStream();
-
- channel.connect();
-
- byte[] buf = new byte[1024];
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
-
- while (true) {
- int c = checkAck(in);
- if (c != 'C') {
- break;
- }
-
- // read '0644 '
- in.read(buf, 0, 5);
-
- long filesize = 0L;
- while (true) {
- if (in.read(buf, 0, 1) < 0) {
- // error
- break;
- }
- if (buf[0] == ' ')
- break;
- filesize = filesize * 10L + (long) (buf[0] - '0');
- }
-
- String file = null;
- for (int i = 0;; i++) {
- in.read(buf, i, 1);
- if (buf[i] == (byte) 0x0a) {
- file = new String(buf, 0, i);
- break;
- }
- }
-
- //System.out.println("filesize="+filesize+", file="+file);
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
-
- // read a content of lfile
- fos = new FileOutputStream(prefix == null ? lfile : prefix + file);
- int foo;
- while (true) {
- if (buf.length < filesize)
- foo = buf.length;
- else
- foo = (int) filesize;
- foo = in.read(buf, 0, foo);
- if (foo < 0) {
- // error
- break;
- }
- fos.write(buf, 0, foo);
- filesize -= foo;
- if (filesize == 0L)
- break;
- }
- fos.close();
- fos = null;
-
- if (checkAck(in) != 0) {
- System.exit(0);
- }
-
- // send '\0'
- buf[0] = 0;
- out.write(buf, 0, 1);
- out.flush();
- }
-
- session.disconnect();
- } catch (Exception e) {
- throw e;
- } finally {
- try {
- if (fos != null)
- fos.close();
- } catch (Exception ee) {
- }
- }
- }
-
- public SSHClientOutput execCommand(String command) throws Exception {
- return execCommand(command, 7200, null);
- }
-
- public SSHClientOutput execCommand(String command, int timeoutSeconds, Logger logAppender) throws Exception {
- try {
- System.out.println("[" + username + "@" + hostname + "] Execute command: " + command);
-
- StringBuffer text = new StringBuffer();
- int exitCode = -1;
-
- Session session = newJSchSession();
- session.connect();
-
- Channel channel = session.openChannel("exec");
- ((ChannelExec) channel).setCommand(command);
-
- channel.setInputStream(null);
-
- // channel.setOutputStream(System.out);
-
- ((ChannelExec) channel).setErrStream(System.err);
-
- InputStream in = channel.getInputStream();
- InputStream err = ((ChannelExec) channel).getErrStream();
-
- channel.connect();
-
- int timeout = timeoutSeconds;
- byte[] tmp = new byte[1024];
- while (true) {
- timeout--;
- while (in.available() > 0) {
- int i = in.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
- while (err.available() > 0) {
- int i = err.read(tmp, 0, 1024);
- if (i < 0)
- break;
-
- String line = new String(tmp, 0, i);
- text.append(line);
- if (logAppender != null) {
- logAppender.log(line);
- }
- }
- if (channel.isClosed()) {
- if (in.available() > 0)
- continue;
- exitCode = channel.getExitStatus();
- System.out.println("[" + username + "@" + hostname + "] Command exit-status: " + exitCode);
-
- break;
- }
- try {
- Thread.sleep(1000);
- } catch (Exception ee) {
- throw ee;
- }
- if (timeout < 0)
- throw new Exception("Remote commmand not finished within " + timeoutSeconds + " seconds.");
- }
- channel.disconnect();
- session.disconnect();
- return new SSHClientOutput(exitCode, text.toString());
- } catch (Exception e) {
- throw e;
- }
- }
-
- private Session newJSchSession() throws JSchException {
- JSch jsch = new JSch();
- if (identityPath != null) {
- jsch.addIdentity(identityPath);
- }
-
- Session session = jsch.getSession(username, hostname, 22);
- if (password != null) {
- session.setPassword(password);
- }
- session.setConfig("StrictHostKeyChecking", "no");
- return session;
- }
-
- private int checkAck(InputStream in) throws IOException {
- int b = in.read();
- // b may be 0 for success,
- // 1 for error,
- // 2 for fatal error,
- // -1
- if (b == 0)
- return b;
- if (b == -1)
- return b;
-
- if (b == 1 || b == 2) {
- StringBuffer sb = new StringBuffer();
- int c;
- do {
- c = in.read();
- sb.append((char) c);
- } while (c != '\n');
- if (b == 1) { // error
- System.out.print(sb.toString());
- }
- if (b == 2) { // fatal error
- System.out.print(sb.toString());
- }
- }
- return b;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java b/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
deleted file mode 100644
index a7e9e12..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SSHClientOutput.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-
-public class SSHClientOutput {
- private String text;
- private int exitCode = -1;
-
- /**
- * @param text
- * @param exitCode
- */
- public SSHClientOutput(int exitCode, String text) {
- this.text = text;
- this.exitCode = exitCode;
- }
-
- /**
- * @return the text
- */
- public String getText() {
- return text.toString();
- }
-
- /**
- * @return the exitCode
- */
- public int getExitCode() {
- return exitCode;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java b/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
deleted file mode 100644
index f82c8d6..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SoutLogger.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.util;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
- */
-public class SoutLogger implements Logger {
-
- @Override
- public void log(String message) {
- System.out.println(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
deleted file mode 100644
index 8deeb5f..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class SplittedBytes {
- public SplittedBytes(int length) {
- this.value = new byte[length];
- this.length = 0;
- }
-
- public byte[] value;
- public int length;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java b/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
deleted file mode 100644
index 4b7ba7e..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/StringSplitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class StringSplitter {
- public static String[] split(String str, String delimiter) {
- // The optimized split function
- List<String> list = new ArrayList<String>();
- int index = 0, offset = 0;
- int l = delimiter.length();
- if (str.startsWith(delimiter)) {
- // in case the first field is empty
- list.add("");
- offset = offset + l;
- }
- while ((index = str.indexOf(delimiter, index + 1)) != -1) {
- list.add(str.substring(offset, index));
- offset = index + l;
- }
- // add the last field, or the str doesn't contain delimiter at all
- list.add(str.substring(offset));
- return list.toArray(new String[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
deleted file mode 100644
index dbf3380..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.commons.lang.StringUtils;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 12/2/13 Time: 11:43 AM To
- * change this template use File | Settings | File Templates.
- */
-public class StringUtil {
-
- public static String[] filterSystemArgs(String args[]) {
- ArrayList<String> whatsLeft = new ArrayList<String>();
- for (String a : args) {
- if (a.startsWith("-D")) {
- String key;
- String value;
- int cut = a.indexOf('=');
- if (cut < 0) {
- key = a.substring(2);
- value = "";
- } else {
- key = a.substring(2, cut);
- value = a.substring(cut + 1);
- }
- System.setProperty(key, value);
- } else {
- whatsLeft.add(a);
- }
- }
- return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]);
- }
-
- public static void toUpperCaseArray(String[] source, String[] target) {
- for (int i = 0; i < source.length; i++) {
- if (source[i] != null) {
- target[i] = source[i].toUpperCase();
- }
- }
- }
-
- public static String noBlank(String str, String dft) {
- return StringUtils.isBlank(str) ? dft : str;
- }
-
- public static String dropSuffix(String str, String suffix) {
- if (str.endsWith(suffix))
- return str.substring(0, str.length() - suffix.length());
- else
- return str;
- }
-
- public static String min(Collection<String> strs) {
- String min = null;
- for (String s : strs) {
- if (min == null || min.compareTo(s) > 0)
- min = s;
- }
- return min;
- }
-
- public static String max(Collection<String> strs) {
- String max = null;
- for (String s : strs) {
- if (max == null || max.compareTo(s) < 0)
- max = s;
- }
- return max;
- }
-
- public static String min(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) < 0 ? s1 : s2;
- }
-
- public static String max(String s1, String s2) {
- if (s1 == null)
- return s2;
- else if (s2 == null)
- return s1;
- else
- return s1.compareTo(s2) > 0 ? s1 : s2;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java b/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
deleted file mode 100644
index 78751dc..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/TarGZUtil.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-
-public class TarGZUtil {
-
- public static void uncompressTarGZ(File tarFile, File dest) throws IOException {
- dest.mkdir();
- TarArchiveInputStream tarIn = null;
-
- tarIn = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(tarFile))));
-
- TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
- // tarIn is a TarArchiveInputStream
- while (tarEntry != null) {// create a file with the same name as the tarEntry
- File destPath = new File(dest, tarEntry.getName());
- System.out.println("working: " + destPath.getCanonicalPath());
- if (tarEntry.isDirectory()) {
- destPath.mkdirs();
- } else {
- destPath.createNewFile();
- //byte [] btoRead = new byte[(int)tarEntry.getSize()];
- byte[] btoRead = new byte[1024];
- //FileInputStream fin
- // = new FileInputStream(destPath.getCanonicalPath());
- BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(destPath));
- int len = 0;
-
- while ((len = tarIn.read(btoRead)) != -1) {
- bout.write(btoRead, 0, len);
- }
-
- bout.close();
- btoRead = null;
-
- }
- tarEntry = tarIn.getNextTarEntry();
- }
- tarIn.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
deleted file mode 100644
index 37c939c..0000000
--- a/common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 12/31/14.
- */
-public class ThreadUtil {
- @SuppressWarnings("unused")
- public static void main(String[] args) {
- ThreadPoolExecutor pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());//Threads.newDaemonThreadFactory("htable"));
-
- for (int i = 0; i < Integer.MAX_VALUE; ++i) {
- System.out.println("index: " + i);
- Future<?> future = pool.submit(new Runnable() {
- @Override
- public void run() {
- try {
- Thread.sleep(10000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/resources/kylin-log4j.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/kylin-log4j.properties b/common/src/main/resources/kylin-log4j.properties
deleted file mode 100644
index 3772972..0000000
--- a/common/src/main/resources/kylin-log4j.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# enable this by -Dlog4j.configuration=kylin-log4j.properties
-
-log4j.rootLogger=INFO,stdout
-
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n
-
-#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
-log4j.logger.org.springframework=WARN
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
deleted file mode 100644
index 75625fb..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class HBaseResourceStoreTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testHBaseStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- @Test
- public void testHBaseStoreWithLargeCell() throws Exception {
- String path = "/cube/_test_large_cell.json";
- String largeContent = "THIS_IS_A_LARGE_CELL";
- StringEntity content = new StringEntity(largeContent);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int origSize = config.getHBaseKeyValueSize();
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- try {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
-
- store.deleteResource(path);
-
- store.putResource(path, content, StringEntity.serializer);
- assertTrue(store.exists(path));
- StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
- assertEquals(content, t);
-
- Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
- Configuration hconf = HadoopUtil.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
- assertTrue(fileSystem.exists(redirectPath));
-
- FSDataInputStream in = fileSystem.open(redirectPath);
- assertEquals(largeContent, in.readUTF());
- in.close();
-
- store.deleteResource(path);
- } finally {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
- store.deleteResource(path);
- }
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
deleted file mode 100644
index 4a9daa6..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testFileStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java b/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
deleted file mode 100644
index 14538c6..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by honma on 9/18/14.
- */
-@Ignore("convenient trial tool for dev")
-public class ResourceToolTest {
- @Before
- public void setup() throws Exception {
- ClassUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath());
- }
-
- @Test
- public void test() throws IOException {
- ResourceTool.copy(KylinConfig.createInstanceFromUri("../examples/test_case_data"), KylinConfig.createInstanceFromUri("../examples/test_case_data/kylin.properties"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java b/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
deleted file mode 100644
index f4d3410..0000000
--- a/common/src/test/java/org/apache/kylin/common/restclient/RestClientTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.restclient;
-
-import java.io.IOException;
-
-import org.junit.Test;
-
-public class RestClientTest {
-
- @SuppressWarnings("unused")
- @Test
- public void basicTests() throws IOException {
- RestClient a = new RestClient("prod01:80");
- //a.wipeCache("metadata", "a", "a");
- //String aa = a.getKylinProperties();
- //System.out.println(aa);
- RestClient b = new RestClient("sandbox.hortonworks.com:7070");
- //b.wipeCache("metadata", "a", "a");
- //String bb = b.getKylinProperties();
- //System.out.println(bb);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
deleted file mode 100644
index 58d3ac1..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.lang.reflect.Method;
-
-import org.apache.kylin.common.KylinConfig;
-
-/**
- * @author ysong1
- *
- */
-public abstract class AbstractKylinTestCase {
-
- public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
-
- public static final String LOCALMETA_TEST_DATA_V1 = "../examples/test_case_data/localmeta_v1";
-
- public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
-
- public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
-
- public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.metadata.project.ProjectManager" };
-
- public abstract void createTestMetadata() throws Exception;
-
- public abstract void cleanupTestMetadata() throws Exception;
-
- public static KylinConfig getTestConfig() {
- return KylinConfig.getInstanceFromEnv();
- }
-
- public static void staticCreateTestMetadata(String kylinConfigFolder) {
-
- KylinConfig.destoryInstance();
-
- HadoopUtil.setCurrentConfiguration(null);
- HadoopUtil.setCurrentHBaseConfiguration(null);
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder);
-
- }
-
- public static void staticCleanupTestMetadata() {
- cleanupCache();
- System.clearProperty(KylinConfig.KYLIN_CONF);
- KylinConfig.destoryInstance();
-
- HadoopUtil.setCurrentConfiguration(null);
- HadoopUtil.setCurrentHBaseConfiguration(null);
-
- }
-
- private static void cleanupCache() {
-
- for (String serviceClass : SERVICES_WITH_CACHE) {
- try {
- Class<?> cls = Class.forName(serviceClass);
- Method method = cls.getDeclaredMethod("clearCache");
- method.invoke(null);
- } catch (ClassNotFoundException e) {
- // acceptable because lower module test does have CubeManager etc on classpath
- } catch (Exception e) {
- System.err.println("Error clean up cache " + serviceClass);
- e.printStackTrace();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java
deleted file mode 100644
index 6d2762c..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BasicHadoopTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Created by honma on 11/11/14.
- *
- * development concept proving use
- */
-@Ignore("convenient trial tool for dev")
-public class BasicHadoopTest {
-
- @BeforeClass
- public static void setup() throws Exception {
- ClassUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath());
- }
-
- @Test
- public void testCreateHtable() throws IOException {
- HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf("testhbase"));
- tableDesc.setValue("KYLIN_HOST", "dev01");
-
- HColumnDescriptor cf = new HColumnDescriptor("f");
- cf.setMaxVersions(1);
-
- cf.setInMemory(true);
- cf.setBlocksize(4 * 1024 * 1024); // set to 4MB
- tableDesc.addFamily(cf);
-
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin admin = new HBaseAdmin(conf);
- admin.createTable(tableDesc);
- admin.close();
- }
-
- @Test
- public void testRetriveHtableHost() throws IOException {
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
- HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables();
- for (HTableDescriptor table : tableDescriptors) {
- String value = table.getValue("KYLIN_HOST");
- if (value != null) {
- System.out.println(table.getTableName());
- System.out.println("host is " + value);
- hbaseAdmin.disableTable(table.getTableName());
- table.setValue("KYLIN_HOST_ANOTHER", "dev02");
- hbaseAdmin.modifyTable(table.getTableName(), table);
- hbaseAdmin.enableTable(table.getTableName());
- }
- }
- hbaseAdmin.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
deleted file mode 100644
index f2778d2..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
-
-/**
-* Created by honma on 10/17/14.
-* <p/>
-* Keep this test case to test basic java functionality
-* development concept proving use
-*/
-@Ignore("convenient trial tool for dev")
-@SuppressWarnings("unused")
-public class BasicTest {
- protected static final org.slf4j.Logger log = LoggerFactory.getLogger(BasicTest.class);
-
- private void log(ByteBuffer a) {
- Integer x = 4;
- foo(x);
- }
-
- private void foo(Long a) {
- System.out.printf("a");
-
- }
-
- private void foo(Integer b) {
- System.out.printf("b");
- }
-
- private enum MetricType {
- Count, DimensionAsMetric, DistinctCount, Normal
- }
-
- @Test
- @Ignore("convenient trial tool for dev")
- public void test1() throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(10);
- ListenableFuture futureTask = MoreExecutors.listeningDecorator(executorService).submit(new Runnable() {
- @Override
- public void run() {
-
- }
- });
- futureTask.addListener(new Runnable() {
- @Override
- public void run() {
-
- }
- }, executorService);
-
- }
-
- @Test
- @Ignore("fix it later")
- public void test2() throws IOException, ConfigurationException {
- int m = 1 << 15;
- System.out.println(m);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
deleted file mode 100644
index 178e568..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.nio.ByteBuffer;
-
-import junit.framework.TestCase;
-
-import org.junit.Test;
-
-/**
- * by honma
- */
-public class BytesUtilTest extends TestCase {
- @Test
- public void test() {
- ByteBuffer buffer = ByteBuffer.allocate(10000);
- int[] x = new int[] { 1, 2, 3 };
- BytesUtil.writeIntArray(x, buffer);
- buffer.flip();
-
- byte[] buf = new byte[buffer.limit()];
- System.arraycopy(buffer.array(), 0, buf, 0, buffer.limit());
-
- ByteBuffer newBuffer = ByteBuffer.wrap(buf);
- int[] y = BytesUtil.readIntArray(newBuffer);
- assertEquals(y[2], 3);
- }
-
- @Test
- public void testBooleanArray() {
- ByteBuffer buffer = ByteBuffer.allocate(10000);
- boolean[] x = new boolean[] { true, false, true };
- BytesUtil.writeBooleanArray(x, buffer);
- buffer.flip();
- boolean[] y = BytesUtil.readBooleanArray(buffer);
- assertEquals(y[2], true);
- assertEquals(y[1], false);
- }
-
- @Test
- public void testReadable() {
- String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
- byte[] bytes = BytesUtil.fromReadableText(x);
- String y = BytesUtil.toHex(bytes);
- assertEquals(x, y);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
deleted file mode 100644
index d2e3238..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- * @author ysong1
- */
-public class HBaseMetadataTestCase extends AbstractKylinTestCase {
-
- static {
- if (useSandbox()) {
- try {
- File sandboxFolder = new File("../examples/test_case_data/sandbox/");
- if (sandboxFolder.exists() == false) {
- throw new IOException("The sandbox folder doesn't exist: " + sandboxFolder.getAbsolutePath());
- }
- ClassUtil.addClasspath(sandboxFolder.getAbsolutePath());
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
-
- @Override
- public void createTestMetadata() throws Exception {
- staticCreateTestMetadata();
- }
-
- @Override
- public void cleanupTestMetadata() {
- staticCleanupTestMetadata();
- }
-
- public static void staticCreateTestMetadata() throws Exception {
- if (useSandbox()) {
- staticCreateTestMetadata(SANDBOX_TEST_DATA);
- } else {
- staticCreateTestMetadata(MINICLUSTER_TEST_DATA);
- HBaseMiniclusterHelper.startupMinicluster();
- }
-
- }
-
- public static boolean useSandbox() {
- String useSandbox = System.getProperty("useSandbox");
- if (StringUtils.isEmpty(useSandbox)) {
- return true;
- }
-
- return Boolean.parseBoolean(useSandbox);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
deleted file mode 100644
index 6f31776..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseResourceStore;
-
-/**
- * a helper class to start and shutdown hbase mini cluster
- *
- * @author shaoshi
- */
-public class HBaseMiniclusterHelper {
-
- public static final String SHARED_STORAGE_PREFIX = "KYLIN_";
- public static final String CUBE_STORAGE_PREFIX = "KYLIN_";
- public static final String II_STORAGE_PREFIX = "KYLIN_II";
- public static final String TEST_METADATA_TABLE = "kylin_metadata";
-
- private static final String hbaseTarLocation = "../examples/test_case_data/minicluster/hbase-export.tar.gz";
- private static final String iiEndpointClassName = "org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
-
- public static HBaseTestingUtility UTIL = new HBaseTestingUtility();
- private static volatile boolean clusterStarted = false;
- private static String zkHost, zkPort, zkParent;
-
- private static final Log logger = LogFactory.getLog(HBaseMiniclusterHelper.class);
-
- static {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- shutdownMiniCluster();
- }
- });
- }
-
- /**
- * Start the minicluster; Sub-classes should invoke this in BeforeClass method.
- *
- * @throws Exception
- */
- public static void startupMinicluster() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.MINICLUSTER_TEST_DATA);
-
- if (!clusterStarted) {
- synchronized (HBaseMiniclusterHelper.class) {
- if (!clusterStarted) {
- startupMiniClusterAndImportData();
- clusterStarted = true;
- }
- }
- } else {
- updateKylinConfigWithMinicluster();
- }
- }
-
- private static void updateKylinConfigWithMinicluster() {
-
- Configuration hbaseConfiguration = HadoopUtil.getCurrentHBaseConfiguration();
- hbaseConfiguration.set(HConstants.ZOOKEEPER_QUORUM, zkHost);
- hbaseConfiguration.set(HConstants.ZOOKEEPER_CLIENT_PORT, zkPort);
- hbaseConfiguration.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkParent);
- }
-
- private static void startupMiniClusterAndImportData() throws Exception {
-
- logger.info("Going to start mini cluster.");
-
- if (existInClassPath(iiEndpointClassName)) {
- HBaseMiniclusterHelper.UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, iiEndpointClassName);
- }
-
- //https://issues.apache.org/jira/browse/HBASE-11711
- UTIL.getConfiguration().setInt("hbase.master.info.port", -1);//avoid port clobbering
-
- MiniHBaseCluster hbaseCluster = UTIL.startMiniCluster();
- UTIL.startMiniMapReduceCluster();
- Configuration config = hbaseCluster.getConf();
- zkHost = config.get(HConstants.ZOOKEEPER_QUORUM);
- zkPort = config.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- zkParent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
-
- // see in: https://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired
- config.set("zookeeper.session.timeout", "1200000");
- config.set("hbase.zookeeper.property.tickTime", "6000");
- // reduce rpc retry
- config.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1");
- config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
-
- updateKylinConfigWithMinicluster();
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- // create the metadata htables;
- @SuppressWarnings("unused")
- HBaseResourceStore store = new HBaseResourceStore(kylinConfig);
-
- // import the table content
- HbaseImporter.importHBaseData(hbaseTarLocation, UTIL.getConfiguration());
-
- }
-
- private static boolean existInClassPath(String className) {
- try {
- Class.forName(className);
- } catch (ClassNotFoundException e) {
- return false;
- }
- return true;
- }
-
- /**
- * Shutdown the minicluster;
- */
- public static void shutdownMiniCluster() {
-
- logger.info("Going to shutdown mini cluster.");
-
- try {
- UTIL.shutdownMiniMapReduceCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- try {
- UTIL.shutdownMiniCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- public static void main(String[] args) {
- HBaseMiniclusterHelper t = new HBaseMiniclusterHelper();
- logger.info(t);
- try {
- HBaseMiniclusterHelper.startupMinicluster();
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- HBaseMiniclusterHelper.shutdownMiniCluster();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java
deleted file mode 100644
index 8587683..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HadoopUtilTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-
-import static org.junit.Assert.*;
-
-/**
- * Created by sunyerui on 15/8/26.
- * Tests for HadoopUtil
- */
-public class HadoopUtilTest {
-
- @BeforeClass
- public static void beforeClass() {
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- }
-
- @After
- public void after() {
- HadoopUtil.setCurrentConfiguration(null);
- HadoopUtil.setCurrentHBaseConfiguration(null);
- }
-
- @Test
- public void testGetCurrentHBaseConfiguration() throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- config.setProperty(KylinConfig.KYLIN_HBASE_CLUSTER_FS, "hdfs://hbase-cluster/");
-
- Configuration conf = HadoopUtil.getCurrentHBaseConfiguration();
- assertEquals("hdfs://hbase-cluster/", conf.get(FileSystem.FS_DEFAULT_NAME_KEY));
- }
-
- @Test
- public void testMakeQualifiedPathInHBaseCluster() throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- config.setProperty(KylinConfig.KYLIN_HBASE_CLUSTER_FS, "file:/");
-
- String path = HadoopUtil.makeQualifiedPathInHBaseCluster("/path/to/test/hbase");
- assertEquals("file:/path/to/test/hbase", path);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
deleted file mode 100644
index 5d8d6d6..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.mapreduce.Import;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-
-import com.google.common.base.Preconditions;
-
-public class HbaseImporter {
-
- private static final Log logger = LogFactory.getLog(HbaseImporter.class);
-
- public static void importHBaseData(String hbaseTarLocation, Configuration conf) throws IOException, ClassNotFoundException, InterruptedException {
-
- if (System.getenv("JAVA_HOME") == null) {
- logger.error("Didn't find $JAVA_HOME, this will cause HBase data import failed. Please set $JAVA_HOME.");
- logger.error("Skipping table import...");
- return;
- }
-
- File exportFile = new File(hbaseTarLocation);
- if (!exportFile.exists()) {
- logger.error("Didn't find the export achieve file on " + exportFile.getAbsolutePath());
- return;
- }
-
- File folder = File.createTempFile("hbase-import", "tmp");
- if (folder.exists()) {
- FileUtils.forceDelete(folder);
- }
- folder.mkdirs();
- FileUtils.forceDeleteOnExit(folder);
-
- //TarGZUtil.uncompressTarGZ(exportFile, folder);
- FileUtil.unTar(exportFile, folder);
- String[] child = folder.list();
- Preconditions.checkState(child.length == 1);
- String backupFolderName = child[0];
- File backupFolder = new File(folder, backupFolderName);
- String[] tableNames = backupFolder.list();
-
- for (String table : tableNames) {
-
- if (!(table.equalsIgnoreCase(HBaseMiniclusterHelper.TEST_METADATA_TABLE) || table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))) {
- continue;
- }
-
- // create the htable; otherwise the import will fail.
- if (table.startsWith(HBaseMiniclusterHelper.II_STORAGE_PREFIX)) {
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "f");
- } else if (table.startsWith(HBaseMiniclusterHelper.CUBE_STORAGE_PREFIX)) {
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2");
- }
-
- // directly import from local fs, no need to copy to hdfs
- String importLocation = "file://" + backupFolder.getAbsolutePath() + "/" + table;
- String[] args = new String[] { table, importLocation };
- boolean result = runImport(args, conf);
- logger.info("importing table '" + table + "' with result:" + result);
-
- if (!result)
- break;
- }
-
- }
-
- private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException {
- // need to make a copy of the configuration because to make sure different temp dirs are used.
- GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args);
- Configuration newConf = opts.getConfiguration();
- args = opts.getRemainingArgs();
- Job job = Import.createSubmittableJob(newConf, args);
- job.waitForCompletion(false);
- return job.isSuccessful();
- }
-
- public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
- if (args.length != 1) {
- logger.error("Usage: HbaseImporter hbase_tar_lcoation");
- System.exit(-1);
- }
-
- logger.info("The KylinConfig being used:");
- logger.info("=================================================");
- KylinConfig.getInstanceFromEnv().printProperties();
- logger.info("=================================================");
-
- importHBaseData(args[0], HBaseConfiguration.create());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
deleted file mode 100644
index ff42713..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class HyperLogLogCounterTest {
-
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- Random rand1 = new Random(1);
- Random rand2 = new Random(2);
- Random rand3 = new Random(3);
- int errorCount1 = 0;
- int errorCount2 = 0;
- int errorCount3 = 0;
-
- private Set<String> generateTestData(int n) {
- Set<String> testData = new HashSet<String>();
- for (int i = 0; i < n; i++) {
- String[] samples = generateSampleData();
- for (String sample : samples) {
- testData.add(sample);
- }
- }
- return testData;
- }
-
- // simulate the visit (=visitor+id)
- private String[] generateSampleData() {
-
- StringBuilder buf = new StringBuilder();
- for (int i = 0; i < 19; i++) {
- buf.append(Math.abs(rand1.nextInt()) % 10);
- }
- String header = buf.toString();
-
- int size = Math.abs(rand3.nextInt()) % 9 + 1;
- String[] samples = new String[size];
- for (int k = 0; k < size; k++) {
- buf = new StringBuilder(header);
- buf.append("-");
- for (int i = 0; i < 10; i++) {
- buf.append(Math.abs(rand3.nextInt()) % 10);
- }
- samples[k] = buf.toString();
- }
-
- return samples;
- }
-
- @Test
- public void countTest() throws IOException {
- int n = 10;
- for (int i = 0; i < 5; i++) {
- count(n);
- n *= 10;
- }
- }
-
- private void count(int n) throws IOException {
- Set<String> testSet = generateTestData(n);
-
- HyperLogLogPlusCounter hllc = newHLLC();
- for (String testData : testSet) {
- hllc.add(Bytes.toBytes(testData));
- }
- long estimate = hllc.getCountEstimate();
- double errorRate = hllc.getErrorRate();
- double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
- System.out.println(estimate);
- System.out.println(testSet.size());
- System.out.println(errorRate);
- System.out.println("=" + actualError);
- Assert.assertTrue(actualError < errorRate * 3.0);
-
- checkSerialize(hllc);
- }
-
- private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException {
- long estimate = hllc.getCountEstimate();
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- hllc.readRegisters(buf);
- Assert.assertEquals(estimate, hllc.getCountEstimate());
- }
-
- @Test
- public void mergeTest() throws IOException {
- double error = 0;
- double absError = 0;
- int n = 100;
- for (int i = 0; i < n; i++) {
- double e = merge();
- error += e;
- absError += Math.abs(e);
- }
- System.out.println("Total average error is " + error / n + " and absolute error is " + absError / n);
-
- System.out.println(" errorRateCount1 is " + errorCount1 + "!");
- System.out.println(" errorRateCount2 is " + errorCount2 + "!");
- System.out.println(" errorRateCount3 is " + errorCount3 + "!");
-
- Assert.assertTrue(errorCount1 <= n * 0.40);
- Assert.assertTrue(errorCount2 <= n * 0.08);
- Assert.assertTrue(errorCount3 <= n * 0.02);
- }
-
- private double merge() throws IOException {
-
- int ln = 50;
- int dn = 300;
- Set<String> testSet = new HashSet<String>();
- HyperLogLogPlusCounter[] hllcs = new HyperLogLogPlusCounter[ln];
- for (int i = 0; i < ln; i++) {
- hllcs[i] = newHLLC();
- for (int k = 0; k < dn; k++) {
- String[] samples = generateSampleData();
- for (String data : samples) {
- testSet.add(data);
- hllcs[i].add(Bytes.toBytes(data));
- }
- }
- }
- HyperLogLogPlusCounter mergeHllc = newHLLC();
- for (HyperLogLogPlusCounter hllc : hllcs) {
- mergeHllc.merge(hllc);
- checkSerialize(mergeHllc);
- }
-
- double errorRate = mergeHllc.getErrorRate();
- long estimate = mergeHllc.getCountEstimate();
- double actualError = (double) (testSet.size() - estimate) / testSet.size();
-
- System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
-
- if (Math.abs(actualError) > errorRate) {
- errorCount1++;
- }
- if (Math.abs(actualError) > 2 * errorRate) {
- errorCount2++;
- }
- if (Math.abs(actualError) > 3 * errorRate) {
- errorCount3++;
- }
-
- return actualError;
- }
-
- @Test
- public void testPerformance() throws IOException {
- int N = 3; // reduce N HLLC into one
- int M = 1000; // for M times, use 100000 for real perf test
-
- HyperLogLogPlusCounter samples[] = new HyperLogLogPlusCounter[N];
- for (int i = 0; i < N; i++) {
- samples[i] = newHLLC();
- for (String str : generateTestData(10000))
- samples[i].add(str);
- }
-
- System.out.println("Perf test running ... ");
- long start = System.currentTimeMillis();
- HyperLogLogPlusCounter sum = newHLLC();
- for (int i = 0; i < M; i++) {
- sum.clear();
- for (int j = 0; j < N; j++) {
- sum.merge(samples[j]);
- checkSerialize(sum);
- }
- }
- long duration = System.currentTimeMillis() - start;
- System.out.println("Perf test result: " + duration / 1000 + " seconds");
- }
-
- @Test
- public void testEquivalence() {
- byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
- byte[] b = new byte[] { 3, 4, 42 };
- HyperLogLogPlusCounter ha = new HyperLogLogPlusCounter();
- HyperLogLogPlusCounter hb = new HyperLogLogPlusCounter();
- ha.add(a, 1, 3);
- hb.add(b);
-
- Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
- }
-
- private HyperLogLogPlusCounter newHLLC() {
- return new HyperLogLogPlusCounter(16);
- }
-}