You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/07 15:47:05 UTC
[44/51] [partial] incubator-kylin git commit: migrate repo from
github.com to apache git
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseRegionSizeCalculator.java b/common/src/main/java/com/kylinolap/common/util/HBaseRegionSizeCalculator.java
new file mode 100644
index 0000000..36ffc29
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/HBaseRegionSizeCalculator.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** This class will come with HBase 2.0 in package org.apache.hadoop.hbase.util **/
+package com.kylinolap.common.util;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.RegionLoad;
+import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HBaseRegionSizeCalculator {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseRegionSizeCalculator.class);
+
+ /**
+ * Maps each region to its size in bytes.
+ **/
+ private final Map<byte[], Long> sizeMap = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
+
+ static final String ENABLE_REGIONSIZECALCULATOR = "hbase.regionsizecalculator.enable";
+
+ /**
+ * Computes size of each region for table and given column families.
+ * */
+ public HBaseRegionSizeCalculator(HTable table) throws IOException {
+ this(table, new HBaseAdmin(table.getConfiguration()));
+ }
+
+ /** Constructor for unit testing */
+ HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+
+ try {
+ if (!enabled(table.getConfiguration())) {
+ logger.info("Region size calculation disabled.");
+ return;
+ }
+
+ logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+
+ // Get regions for table.
+ Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+
+ for (HRegionInfo regionInfo : tableRegionInfos) {
+ tableRegions.add(regionInfo.getRegionName());
+ }
+
+ ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ Collection<ServerName> servers = clusterStatus.getServers();
+ final long megaByte = 1024L * 1024L;
+
+ // Iterate all cluster regions, filter regions from our table and
+ // compute their size.
+ for (ServerName serverName : servers) {
+ ServerLoad serverLoad = clusterStatus.getLoad(serverName);
+
+ for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
+ byte[] regionId = regionLoad.getName();
+
+ if (tableRegions.contains(regionId)) {
+
+ long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
+ sizeMap.put(regionId, regionSizeBytes);
+
+ // logger.info("Region " + regionLoad.getNameAsString()
+ // + " has size " + regionSizeBytes);
+ }
+ }
+ }
+ } finally {
+ hBaseAdmin.close();
+ }
+
+ }
+
+ boolean enabled(Configuration configuration) {
+ return configuration.getBoolean(ENABLE_REGIONSIZECALCULATOR, true);
+ }
+
+ /**
+ * Returns size of given region in bytes. Returns 0 if region was not found.
+ **/
+ public long getRegionSize(byte[] regionId) {
+ Long size = sizeMap.get(regionId);
+ if (size == null) {
+ logger.info("Unknown region:" + Arrays.toString(regionId));
+ return 0;
+ } else {
+ return size;
+ }
+ }
+
+ public Map<byte[], Long> getRegionSizeMap() {
+ return Collections.unmodifiableMap(sizeMap);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java b/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
new file mode 100644
index 0000000..77767fc
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HadoopUtil {
+ private static final Logger logger = LoggerFactory.getLogger(HadoopUtil.class);
+
+ private static Configuration hadoopConfig;
+
+ public static Configuration getDefaultConfiguration() {
+ if (hadoopConfig == null) {
+ hadoopConfig = new Configuration();
+ }
+ return hadoopConfig;
+ }
+
+ public static FileSystem getFileSystem(String path) throws IOException {
+ return FileSystem.get(makeURI(path), getDefaultConfiguration());
+ }
+
+ public static URI makeURI(String filePath) {
+ try {
+ return new URI(filePath);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException("Cannot create FileSystem from URI: " + filePath, e);
+ }
+ }
+
+ /**
+ * e.g. "hbase:kylin-local.corp.ebay.com:2181:/hbase-unsecure"
+ */
+ public static Configuration newHBaseConfiguration(String url) {
+ Configuration conf = HBaseConfiguration.create();
+ if (StringUtils.isEmpty(url))
+ return conf;
+
+ // chop off "hbase:"
+ if (url.startsWith("hbase:") == false)
+ throw new IllegalArgumentException("hbase url must start with 'hbase:' -- " + url);
+
+ url = StringUtils.substringAfter(url, "hbase:");
+ if (StringUtils.isEmpty(url))
+ return conf;
+
+ // case of "hbase:domain.com:2181:/hbase-unsecure"
+ Pattern urlPattern = Pattern.compile("([\\w\\d\\-.]+)[:](\\d+)(?:[:](.*))?");
+ Matcher m = urlPattern.matcher(url);
+ if (m.matches() == false)
+ throw new IllegalArgumentException("HBase URL '" + url + "' is invalid, expected url is like '" + "hbase:domain.com:2181:/hbase-unsecure" + "'");
+
+ logger.debug("Creating hbase conf by parsing -- " + url);
+
+ String quorum = m.group(1);
+ try {
+ InetAddress.getByName(quorum);
+ } catch (UnknownHostException e) {
+ throw new IllegalArgumentException("Zookeeper quorum is invalid: " + quorum + "; urlString=" + url, e);
+ }
+ conf.set(HConstants.ZOOKEEPER_QUORUM, quorum);
+
+ String port = m.group(2);
+ conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, port);
+
+ String znodePath = m.group(3) == null ? "" : m.group(3);
+ if (StringUtils.isEmpty(znodePath) == false)
+ conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodePath);
+
+ // reduce rpc retry
+ conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
+ conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5");
+ conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
+ // conf.set(ScannerCallable.LOG_SCANNER_ACTIVITY, "true");
+
+ return conf;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/JsonUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/JsonUtil.java b/common/src/main/java/com/kylinolap/common/util/JsonUtil.java
new file mode 100644
index 0000000..1ca8a90
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/JsonUtil.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Reader;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+public class JsonUtil {
+
+ // reuse the object mapper to save memory footprint
+ private static final ObjectMapper mapper = new ObjectMapper();
+ private static final ObjectMapper indentMapper = new ObjectMapper();
+
+ static {
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true);
+ }
+
+ public static <T> T readValue(File src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
+ return mapper.readValue(src, valueType);
+ }
+
+ public static <T> T readValue(String content, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
+ return mapper.readValue(content, valueType);
+ }
+
+ public static <T> T readValue(Reader src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
+ return mapper.readValue(src, valueType);
+ }
+
+ public static <T> T readValue(InputStream src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
+ return mapper.readValue(src, valueType);
+ }
+
+ public static <T> T readValue(byte[] src, Class<T> valueType) throws IOException, JsonParseException, JsonMappingException {
+ return mapper.readValue(src, valueType);
+ }
+
+ public static void writeValueIndent(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException {
+ indentMapper.writeValue(out, value);
+ }
+
+ public static void writeValue(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException {
+ mapper.writeValue(out, value);
+ }
+
+ public static String writeValueAsString(Object value) throws JsonProcessingException {
+ return mapper.writeValueAsString(value);
+ }
+
+ public static byte[] writeValueAsBytes(Object value) throws JsonProcessingException {
+ return mapper.writeValueAsBytes(value);
+ }
+
+ public static String writeValueAsIndentString(Object value) throws JsonProcessingException {
+ return indentMapper.writeValueAsString(value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/LocalFileMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/LocalFileMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/LocalFileMetadataTestCase.java
new file mode 100644
index 0000000..f176c71
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/LocalFileMetadataTestCase.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.persistence.ResourceStore;
+
+/**
+ * @author ysong1
+ */
+public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
+ private String tempTestMetadataUrl = null;
+
+ @Override
+ public void createTestMetadata() {
+ KylinConfig.destoryInstance();
+
+ this.tempTestMetadataUrl = "../examples/test_metadata";
+ try {
+ FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
+ FileUtils.copyDirectory(new File(LOCALMETA_TEST_DATA), new File(tempTestMetadataUrl));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
+ System.setProperty(KylinConfig.KYLIN_CONF, tempTestMetadataUrl);
+
+ KylinConfig.getInstanceFromEnv().setMetadataUrl(tempTestMetadataUrl);
+
+ }
+
+ @Override
+ public void cleanupTestMetadata() {
+ try {
+ FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
+ } catch (IOException e) {
+ throw new IllegalStateException("Can't delete directory " + tempTestMetadataUrl, e);
+ }
+ System.clearProperty(KylinConfig.KYLIN_CONF);
+ KylinConfig.destoryInstance();
+ this.tempTestMetadataUrl = null;
+ }
+
+ protected ResourceStore getStore() {
+ return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/LongAsFloat.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/LongAsFloat.java b/common/src/main/java/com/kylinolap/common/util/LongAsFloat.java
new file mode 100644
index 0000000..3d38ec2
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/LongAsFloat.java
@@ -0,0 +1,5 @@
+package com.kylinolap.common.util;
+
+public class LongAsFloat {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/MailService.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/MailService.java b/common/src/main/java/com/kylinolap/common/util/MailService.java
new file mode 100644
index 0000000..608806c
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/MailService.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.mail.Email;
+import org.apache.commons.mail.EmailException;
+import org.apache.commons.mail.HtmlEmail;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * @author xduo
+ *
+ */
+public class MailService {
+
+ private Boolean enabled = Boolean.TRUE;
+ private String host;
+ private String username;
+ private String password;
+ private String sender;
+
+ private static final Log logger = LogFactory.getLog(MailService.class);
+
+ public MailService() {
+ this(KylinConfig.getInstanceFromEnv());
+ }
+
+ public MailService(KylinConfig config) {
+ enabled = "true".equalsIgnoreCase(config.getProperty(KylinConfig.MAIL_ENABLED, "true"));
+ host = config.getProperty(KylinConfig.MAIL_HOST, "");
+ username = config.getProperty(KylinConfig.MAIL_USERNAME, "");
+ password = config.getProperty(KylinConfig.MAIL_PASSWORD, "");
+ sender = config.getProperty(KylinConfig.MAIL_SENDER, "");
+
+ if (enabled) {
+ assert !host.isEmpty();
+ }
+ }
+
+ /**
+ *
+ * @param receivers
+ * @param subject
+ * @param content
+ * @return true or false indicating whether the email was delivered successfully
+ * @throws IOException
+ */
+ public boolean sendMail(List<String> receivers, String subject, String content) throws IOException {
+
+ if (!enabled) {
+ logger.info("Email service is disabled; this mail will not be delivered: " + subject);
+ logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties");
+ return false;
+ }
+
+ Email email = new HtmlEmail();
+ email.setHostName(host);
+ if (username != null && username.trim().length() > 0) {
+ email.setAuthentication(username, password);
+ }
+
+ //email.setDebug(true);
+ try {
+ for (String receiver : receivers) {
+ email.addTo(receiver);
+ }
+
+ email.setFrom(sender);
+ email.setSubject(subject);
+ email.setCharset("UTF-8");
+ ((HtmlEmail) email).setHtmlMsg(content);
+ email.send();
+ email.getMailSession();
+
+ } catch (EmailException e) {
+ logger.error(e.getLocalizedMessage(),e);
+ return false;
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/MyLogFormatter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/MyLogFormatter.java b/common/src/main/java/com/kylinolap/common/util/MyLogFormatter.java
new file mode 100644
index 0000000..b7e6180
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/MyLogFormatter.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.sql.Timestamp;
+import java.util.Date;
+import java.util.logging.Formatter;
+import java.util.logging.LogRecord;
+
+public class MyLogFormatter extends Formatter {
+
+ Date dat = new Date();
+
+ // Line separator string. This is the value of the line.separator
+ // property at the moment that the SimpleFormatter was created.
+ private String lineSeparator = "\n";
+
+ /**
+ * Format the given LogRecord.
+ *
+ * @param record
+ * the log record to be formatted.
+ * @return a formatted log record
+ */
+ public synchronized String format(LogRecord record) {
+ StringBuffer sb = new StringBuffer();
+ // Minimize memory allocations here.
+ Timestamp ts = new Timestamp(record.getMillis());
+ String text = ts.toString();
+ sb.append("JUL ");
+ sb.append(text);
+ sb.append(" ");
+ if (record.getSourceClassName() != null) {
+ sb.append(record.getSourceClassName());
+ } else {
+ sb.append(record.getLoggerName());
+ }
+ if (record.getSourceMethodName() != null) {
+ sb.append(" ");
+ sb.append(record.getSourceMethodName());
+ }
+ sb.append(lineSeparator);
+ String message = formatMessage(record);
+ sb.append(record.getLevel().getLocalizedName());
+ sb.append(": ");
+ sb.append(message);
+ sb.append(lineSeparator);
+ if (record.getThrown() != null) {
+ try {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ record.getThrown().printStackTrace(pw);
+ pw.close();
+ sb.append(sw.toString());
+ } catch (Exception ex) {
+ }
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/RandomSampler.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/RandomSampler.java b/common/src/main/java/com/kylinolap/common/util/RandomSampler.java
new file mode 100644
index 0000000..214d187
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/RandomSampler.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RandomSampler<T> {
+
+ private Random rdm = new Random();
+
+ public List<T> sample(List<T> data, int sampleNumber) {
+ if (data == null) {
+ throw new IllegalArgumentException("Input list is null");
+ }
+ if (data.size() < sampleNumber) {
+ return data;
+ }
+
+ List<T> result = new ArrayList<T>(sampleNumber);
+ int n = data.size();
+ for (int i = 0; i < n; i++) {
+ if (i < sampleNumber) {
+ result.add(data.get(i));
+ } else {
+ int j = rdm.nextInt(i);
+ if (j < sampleNumber) {
+ result.set(j, data.get(i));
+ }
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/SSHClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/SSHClient.java b/common/src/main/java/com/kylinolap/common/util/SSHClient.java
new file mode 100644
index 0000000..aadef76
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/SSHClient.java
@@ -0,0 +1,268 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.jcraft.jsch.Channel;
+import com.jcraft.jsch.ChannelExec;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
+
+public class SSHClient {
+ protected static final Logger logger = LoggerFactory.getLogger(SSHClient.class);
+
+ private String hostname;
+ private String username;
+ private String password;
+ private String identityPath;
+
+ private SSHLogger sshLogger;
+
+ public SSHClient(String hostname, String username, String password, SSHLogger sshLogger) {
+ this.hostname = hostname;
+ this.username = username;
+ if (password != null && new File(password).exists()) {
+ this.identityPath = new File(password).getAbsolutePath();
+ this.password = null;
+ } else {
+ this.password = password;
+ this.identityPath = null;
+ }
+ this.sshLogger = sshLogger;
+ }
+
+ public void scpFileToRemote(String localFile, String remoteTargetDirectory) throws Exception {
+ FileInputStream fis = null;
+ try {
+ System.out.println("SCP file " + localFile + " to " + remoteTargetDirectory);
+
+ Session session = newJSchSession();
+ session.connect();
+
+ boolean ptimestamp = false;
+
+ // exec 'scp -t rfile' remotely
+ String command = "scp " + (ptimestamp ? "-p" : "") + " -t " + remoteTargetDirectory;
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+
+ // get I/O streams for remote scp
+ OutputStream out = channel.getOutputStream();
+ InputStream in = channel.getInputStream();
+
+ channel.connect();
+
+ if (checkAck(in) != 0) {
+ System.exit(0);
+ }
+
+ File _lfile = new File(localFile);
+
+ if (ptimestamp) {
+ command = "T " + (_lfile.lastModified() / 1000) + " 0";
+ // The access time should be sent here,
+ // but it is not accessible with JavaAPI ;-<
+ command += (" " + (_lfile.lastModified() / 1000) + " 0\n");
+ out.write(command.getBytes());
+ out.flush();
+ if (checkAck(in) != 0) {
+ throw new Exception("Error in checkAck()");
+ }
+ }
+
+ // send "C0644 filesize filename", where filename should not include '/'
+ long filesize = _lfile.length();
+ command = "C0644 " + filesize + " ";
+ if (localFile.lastIndexOf("/") > 0) {
+ command += localFile.substring(localFile.lastIndexOf("/") + 1);
+ } else if (localFile.lastIndexOf(File.separator) > 0) {
+ command += localFile.substring(localFile.lastIndexOf(File.separator) + 1);
+ } else {
+ command += localFile;
+ }
+ command += "\n";
+ out.write(command.getBytes());
+ out.flush();
+ if (checkAck(in) != 0) {
+ throw new Exception("Error in checkAck()");
+ }
+
+ // send a content of lfile
+ fis = new FileInputStream(localFile);
+ byte[] buf = new byte[1024];
+ while (true) {
+ int len = fis.read(buf, 0, buf.length);
+ if (len <= 0)
+ break;
+ out.write(buf, 0, len); // out.flush();
+ }
+ fis.close();
+ fis = null;
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+ if (checkAck(in) != 0) {
+ throw new Exception("Error in checkAck()");
+ }
+ out.close();
+
+ channel.disconnect();
+ session.disconnect();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ try {
+ if (fis != null)
+ fis.close();
+ } catch (Exception ee) {
+ }
+ }
+ }
+
+ public SSHClientOutput execCommand(String command) throws Exception {
+ return execCommand(command, 7200);
+ }
+
+ public SSHClientOutput execCommand(String command, int timeoutSeconds) throws Exception {
+ try {
+ System.out.println("[" + username + "@" + hostname + "] Execute command: " + command);
+
+ StringBuffer text = new StringBuffer();
+ int exitCode = -1;
+
+ Session session = newJSchSession();
+ session.connect();
+
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+
+ channel.setInputStream(null);
+
+ // channel.setOutputStream(System.out);
+
+ ((ChannelExec) channel).setErrStream(System.err);
+
+ InputStream in = channel.getInputStream();
+ InputStream err = ((ChannelExec) channel).getErrStream();
+
+ channel.connect();
+
+ int timeout = timeoutSeconds;
+ byte[] tmp = new byte[1024];
+ while (true) {
+ timeout--;
+ while (in.available() > 0) {
+ int i = in.read(tmp, 0, 1024);
+ if (i < 0)
+ break;
+
+ String line = new String(tmp, 0, i);
+ text.append(line);
+ if (this.sshLogger != null) {
+ this.sshLogger.log(line);
+ }
+ }
+ while (err.available() > 0) {
+ int i = err.read(tmp, 0, 1024);
+ if (i < 0)
+ break;
+
+ String line = new String(tmp, 0, i);
+ text.append(line);
+ if (this.sshLogger != null) {
+ this.sshLogger.log(line);
+ }
+ }
+ if (channel.isClosed()) {
+ if (in.available() > 0)
+ continue;
+ exitCode = channel.getExitStatus();
+ System.out.println("[" + username + "@" + hostname + "] Command exit-status: " + exitCode);
+
+ break;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (Exception ee) {
+ throw ee;
+ }
+ if (timeout < 0)
+ throw new Exception("Remote commmand not finished within " + timeoutSeconds + " seconds.");
+ }
+ channel.disconnect();
+ session.disconnect();
+ return new SSHClientOutput(exitCode, text.toString());
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ private Session newJSchSession() throws JSchException {
+ JSch jsch = new JSch();
+ if (identityPath != null) {
+ jsch.addIdentity(identityPath);
+ }
+
+ Session session = jsch.getSession(username, hostname, 22);
+ if (password != null) {
+ session.setPassword(password);
+ }
+ session.setConfig("StrictHostKeyChecking", "no");
+ return session;
+ }
+
+ private int checkAck(InputStream in) throws IOException {
+ int b = in.read();
+ // b may be 0 for success,
+ // 1 for error,
+ // 2 for fatal error,
+ // -1
+ if (b == 0)
+ return b;
+ if (b == -1)
+ return b;
+
+ if (b == 1 || b == 2) {
+ StringBuffer sb = new StringBuffer();
+ int c;
+ do {
+ c = in.read();
+ sb.append((char) c);
+ } while (c != '\n');
+ if (b == 1) { // error
+ System.out.print(sb.toString());
+ }
+ if (b == 2) { // fatal error
+ System.out.print(sb.toString());
+ }
+ }
+ return b;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/SSHClientOutput.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/SSHClientOutput.java b/common/src/main/java/com/kylinolap/common/util/SSHClientOutput.java
new file mode 100644
index 0000000..40f2d2d
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/SSHClientOutput.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+
+public class SSHClientOutput {
+ private String text;
+ private int exitCode = -1;
+
+ /**
+ * @param text
+ * @param exitCode
+ */
+ public SSHClientOutput(int exitCode, String text) {
+ this.text = text;
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * @return the text
+ */
+ public String getText() {
+ return text.toString();
+ }
+
+ /**
+ * @return the exitCode
+ */
+ public int getExitCode() {
+ return exitCode;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/SSHLogger.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/SSHLogger.java b/common/src/main/java/com/kylinolap/common/util/SSHLogger.java
new file mode 100644
index 0000000..ebf025c
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/SSHLogger.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+/**
+ * @author ysong1
+ *
+ */
+public interface SSHLogger {
+ public void log(String message);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/StringSplitter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/StringSplitter.java b/common/src/main/java/com/kylinolap/common/util/StringSplitter.java
new file mode 100644
index 0000000..fc7b6ea
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/StringSplitter.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class StringSplitter {
+ public static String[] split(String str, String delimiter) {
+ // The optimized split function
+ List<String> list = new ArrayList<String>();
+ int index = 0, offset = 0;
+ int l = delimiter.length();
+ if (str.startsWith(delimiter)) {
+ // in case the first field is empty
+ list.add("");
+ offset = offset + l;
+ }
+ while ((index = str.indexOf(delimiter, index + 1)) != -1) {
+ list.add(str.substring(offset, index));
+ offset = index + l;
+ }
+ // add the last field, or the str doesn't contain delimiter at all
+ list.add(str.substring(offset));
+ return list.toArray(new String[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/StringUtil.java b/common/src/main/java/com/kylinolap/common/util/StringUtil.java
new file mode 100644
index 0000000..6a3eaa3
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/StringUtil.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * Created with IntelliJ IDEA. User: lukhan Date: 12/2/13 Time: 11:43 AM To
+ * change this template use File | Settings | File Templates.
+ */
+public class StringUtil {
+
+ public static String[] filterSystemArgs(String args[]) {
+ ArrayList<String> whatsLeft = new ArrayList<String>();
+ for (String a : args) {
+ if (a.startsWith("-D")) {
+ String key;
+ String value;
+ int cut = a.indexOf('=');
+ if (cut < 0) {
+ key = a.substring(2);
+ value = "";
+ } else {
+ key = a.substring(2, cut);
+ value = a.substring(cut + 1);
+ }
+ System.setProperty(key, value);
+ } else {
+ whatsLeft.add(a);
+ }
+ }
+ return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]);
+ }
+
+ public static void toUpperCaseArray(String[] source, String[] target) {
+ for (int i = 0; i < source.length; i++) {
+ if (source[i] != null) {
+ target[i] = source[i].toUpperCase();
+ }
+ }
+ }
+
+ public static String dropSuffix(String str, String suffix) {
+ if (str.endsWith(suffix))
+ return str.substring(0, str.length() - suffix.length());
+ else
+ return str;
+ }
+
+ public static String min(Collection<String> strs) {
+ String min = null;
+ for (String s : strs) {
+ if (min == null || min.compareTo(s) > 0)
+ min = s;
+ }
+ return min;
+ }
+
+ public static String max(Collection<String> strs) {
+ String max = null;
+ for (String s : strs) {
+ if (max == null || max.compareTo(s) < 0)
+ max = s;
+ }
+ return max;
+ }
+
+ public static String min(String s1, String s2) {
+ if (s1 == null)
+ return s2;
+ else if (s2 == null)
+ return s1;
+ else
+ return s1.compareTo(s2) < 0 ? s1 : s2;
+ }
+
+ public static String max(String s1, String s2) {
+ if (s1 == null)
+ return s2;
+ else if (s2 == null)
+ return s1;
+ else
+ return s1.compareTo(s2) > 0 ? s1 : s2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/resources/kylinlog4j.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/kylinlog4j.properties b/common/src/main/resources/kylinlog4j.properties
new file mode 100644
index 0000000..49fb5cc
--- /dev/null
+++ b/common/src/main/resources/kylinlog4j.properties
@@ -0,0 +1,10 @@
+# use this when conflict with hbase, enable this by -Dlog4j.configuration=kylinlog4j.properties
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n
+
+#log4j.logger.org.apache.hadoop=ERROR
+log4j.logger.com.kylinolap=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/log4j.properties b/common/src/main/resources/log4j.properties
new file mode 100644
index 0000000..b00f355
--- /dev/null
+++ b/common/src/main/resources/log4j.properties
@@ -0,0 +1,9 @@
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=L4J [%d{yyyy-MM-dd HH:mm:ss,SSS}][%p][%c] - %m%n
+
+#log4j.logger.org.apache.hadoop=ERROR
+log4j.logger.com.kylinolap=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/persistence/HBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/persistence/HBaseResourceStoreTest.java b/common/src/test/java/com/kylinolap/common/persistence/HBaseResourceStoreTest.java
new file mode 100644
index 0000000..86e11d0
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/persistence/HBaseResourceStoreTest.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import static org.junit.Assert.*;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.HBaseMetadataTestCase;
+import com.kylinolap.common.util.HadoopUtil;
+
+public class HBaseResourceStoreTest extends HBaseMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testHBaseStore() throws Exception {
+ testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
+ }
+
+ @Test
+ public void testHBaseStoreWithLargeCell() throws Exception {
+ String path = "/cube/_test_large_cell.json";
+ String largeContent = "THIS_IS_A_LARGE_CELL";
+ StringEntity content = new StringEntity(largeContent);
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int origSize = config.getHBaseKeyValueSize();
+ ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+
+ try {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
+
+ store.deleteResource(path);
+
+ store.putResource(path, content, StringEntity.serializer);
+ assertTrue(store.exists(path));
+ StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
+ assertEquals(content, t);
+
+ Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
+ Configuration hconf = HadoopUtil.getDefaultConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+ assertTrue(fileSystem.exists(redirectPath));
+
+ FSDataInputStream in = fileSystem.open(redirectPath);
+ assertEquals(largeContent, in.readUTF());
+ in.close();
+
+ store.deleteResource(path);
+ } finally {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
+ store.deleteResource(path);
+ }
+ }
+
+ void testAStore(ResourceStore store) throws IOException {
+ String dir1 = "/cube";
+ String path1 = "/cube/_test.json";
+ StringEntity content1 = new StringEntity("anything");
+ String dir2 = "/table";
+ String path2 = "/table/_test.json";
+ StringEntity content2 = new StringEntity("something");
+
+ // cleanup legacy if any
+ store.deleteResource(path1);
+ store.deleteResource(path2);
+
+ StringEntity t;
+
+ // put/get
+ store.putResource(path1, content1, StringEntity.serializer);
+ assertTrue(store.exists(path1));
+ t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
+ assertEquals(content1, t);
+
+ store.putResource(path2, content2, StringEntity.serializer);
+ assertTrue(store.exists(path2));
+ t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
+ assertEquals(content2, t);
+
+ // overwrite
+ t.str = "new string";
+ store.putResource(path2, t, StringEntity.serializer);
+
+ // write conflict
+ try {
+ t.setLastModified(t.lastModified - 1);
+ store.putResource(path2, t, StringEntity.serializer);
+ fail("write conflict should trigger IllegalStateException");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ // list
+ ArrayList<String> list;
+
+ list = store.listResources(dir1);
+ assertTrue(list.contains(path1));
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(dir2);
+ assertTrue(list.contains(path2));
+ assertTrue(list.contains(path1) == false);
+
+ list = store.listResources("/");
+ assertTrue(list.contains(dir1));
+ assertTrue(list.contains(dir2));
+ assertTrue(list.contains(path1) == false);
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(path1);
+ assertNull(list);
+ list = store.listResources(path2);
+ assertNull(list);
+
+ // delete/exist
+ store.deleteResource(path1);
+ assertTrue(store.exists(path1) == false);
+ list = store.listResources(dir1);
+ assertTrue(list == null || list.contains(path1) == false);
+
+ store.deleteResource(path2);
+ assertTrue(store.exists(path2) == false);
+ list = store.listResources(dir2);
+ assertTrue(list == null || list.contains(path2) == false);
+ }
+
+ public static class StringEntity extends RootPersistentEntity {
+
+ static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/persistence/LocalFileResourceStoreTest.java b/common/src/test/java/com/kylinolap/common/persistence/LocalFileResourceStoreTest.java
new file mode 100644
index 0000000..03247e9
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/persistence/LocalFileResourceStoreTest.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import static org.junit.Assert.*;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.commons.lang.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+
+public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testFileStore() throws Exception {
+ testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
+ }
+
+ void testAStore(ResourceStore store) throws IOException {
+ String dir1 = "/cube";
+ String path1 = "/cube/_test.json";
+ StringEntity content1 = new StringEntity("anything");
+ String dir2 = "/table";
+ String path2 = "/table/_test.json";
+ StringEntity content2 = new StringEntity("something");
+
+ // cleanup legacy if any
+ store.deleteResource(path1);
+ store.deleteResource(path2);
+
+ StringEntity t;
+
+ // put/get
+ store.putResource(path1, content1, StringEntity.serializer);
+ assertTrue(store.exists(path1));
+ t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
+ assertEquals(content1, t);
+
+ store.putResource(path2, content2, StringEntity.serializer);
+ assertTrue(store.exists(path2));
+ t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
+ assertEquals(content2, t);
+
+ // overwrite
+ t.str = "new string";
+ store.putResource(path2, t, StringEntity.serializer);
+
+ // write conflict
+ try {
+ t.setLastModified(t.lastModified - 1);
+ store.putResource(path2, t, StringEntity.serializer);
+ fail("write conflict should trigger IllegalStateException");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ // list
+ ArrayList<String> list;
+
+ list = store.listResources(dir1);
+ assertTrue(list.contains(path1));
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(dir2);
+ assertTrue(list.contains(path2));
+ assertTrue(list.contains(path1) == false);
+
+ list = store.listResources("/");
+ assertTrue(list.contains(dir1));
+ assertTrue(list.contains(dir2));
+ assertTrue(list.contains(path1) == false);
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(path1);
+ assertNull(list);
+ list = store.listResources(path2);
+ assertNull(list);
+
+ // delete/exist
+ store.deleteResource(path1);
+ assertTrue(store.exists(path1) == false);
+ list = store.listResources(dir1);
+ assertTrue(list == null || list.contains(path1) == false);
+
+ store.deleteResource(path2);
+ assertTrue(store.exists(path2) == false);
+ list = store.listResources(dir2);
+ assertTrue(list == null || list.contains(path2) == false);
+ }
+
+ public static class StringEntity extends RootPersistentEntity {
+
+ static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/persistence/ResourceToolTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/persistence/ResourceToolTest.java b/common/src/test/java/com/kylinolap/common/persistence/ResourceToolTest.java
new file mode 100644
index 0000000..5a1fca3
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/persistence/ResourceToolTest.java
@@ -0,0 +1,28 @@
+package com.kylinolap.common.persistence;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.ClasspathUtil;
+
+/**
+ * Created by honma on 9/18/14.
+ */
+@Ignore
+public class ResourceToolTest {
+ @Before
+ public void setup() throws Exception {
+ ClasspathUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath());
+ }
+
+ @Test
+ public void test() throws IOException {
+ ResourceTool.copy(KylinConfig.createInstanceFromUri("../examples/test_case_data"), KylinConfig.createInstanceFromUri("../examples/test_case_data/kylin.properties"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java b/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java
new file mode 100644
index 0000000..25968b3
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java
@@ -0,0 +1,25 @@
+package com.kylinolap.common.restclient;
+
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class RestClientTest {
+
+ @SuppressWarnings("unused")
+ @Test
+ public void basicTests() throws IOException {
+ RestClient a = new RestClient("prod01:80");
+ //a.wipeCache("metadata", "a", "a");
+ //String aa = a.getKylinProperties();
+ //System.out.println(aa);
+ RestClient b = new RestClient("sandbox.hortonworks.com:7070");
+ //b.wipeCache("metadata", "a", "a");
+ //String bb = b.getKylinProperties();
+ //System.out.println(bb);
+
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/util/BasicHadoopTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/util/BasicHadoopTest.java b/common/src/test/java/com/kylinolap/common/util/BasicHadoopTest.java
new file mode 100644
index 0000000..b09173e
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/util/BasicHadoopTest.java
@@ -0,0 +1,65 @@
+package com.kylinolap.common.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Created by honma on 11/11/14.
+ *
+ * development concept proving use
+ */
+@Ignore
+public class BasicHadoopTest {
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ ClasspathUtil.addClasspath(new File("../examples/test_case_data/hadoop-site").getAbsolutePath());
+ }
+
+ @Test
+ public void testCreateHtable() throws IOException {
+ HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf("testhbase"));
+ tableDesc.setValue("KYLIN_HOST", "dev01");
+
+ HColumnDescriptor cf = new HColumnDescriptor("f");
+ cf.setMaxVersions(1);
+
+ cf.setInMemory(true);
+ cf.setBlocksize(4 * 1024 * 1024); // set to 4MB
+ tableDesc.addFamily(cf);
+
+ Configuration conf = HBaseConfiguration.create();
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ admin.createTable(tableDesc);
+ admin.close();
+ }
+
+ @Test
+ public void testRetriveHtableHost() throws IOException {
+ Configuration conf = HBaseConfiguration.create();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables();
+ for (HTableDescriptor table : tableDescriptors) {
+ String value = table.getValue("KYLIN_HOST");
+ if (value != null) {
+ System.out.println(table.getTableName());
+ System.out.println("host is " + value);
+ hbaseAdmin.disableTable(table.getTableName());
+ table.setValue("KYLIN_HOST_ANOTHER", "dev02");
+ hbaseAdmin.modifyTable(table.getTableName(), table);
+ hbaseAdmin.enableTable(table.getTableName());
+ }
+ }
+ hbaseAdmin.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/util/BasicTest.java b/common/src/test/java/com/kylinolap/common/util/BasicTest.java
new file mode 100644
index 0000000..347f951
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/util/BasicTest.java
@@ -0,0 +1,21 @@
+package com.kylinolap.common.util;
+
+import java.io.IOException;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Created by honma on 10/17/14.
+ *
+ * Keep this test case to test basic java functionality
+ * development concept proving use
+ */
+@Ignore
+public class BasicTest {
+ @Test
+ public void test() throws IOException {
+ double i2 = 3234.4324234324234;
+ System.out.println(String.format("%.2f", i2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/util/HyperLogLogCounterTest.java b/common/src/test/java/com/kylinolap/common/util/HyperLogLogCounterTest.java
new file mode 100644
index 0000000..c41683b
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/util/HyperLogLogCounterTest.java
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.kylinolap.common.hll.HyperLogLogPlusCounter;
+
+/**
+ * @author yangli9
+ *
+ */
+public class HyperLogLogCounterTest {
+
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ Random rand1 = new Random(1);
+ Random rand2 = new Random(2);
+ Random rand3 = new Random(3);
+ int errorCount1 = 0;
+ int errorCount2 = 0;
+ int errorCount3 = 0;
+
+ private Set<String> generateTestData(int n) {
+ Set<String> testData = new HashSet<String>();
+ for (int i = 0; i < n; i++) {
+ String[] samples = generateSampleData();
+ for (String sample : samples) {
+ testData.add(sample);
+ }
+ }
+ return testData;
+ }
+
+ // simulate the visit (=visitor+id)
+ private String[] generateSampleData() {
+
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < 19; i++) {
+ buf.append(Math.abs(rand1.nextInt()) % 10);
+ }
+ String header = buf.toString();
+
+ int size = Math.abs(rand3.nextInt()) % 9 + 1;
+ String[] samples = new String[size];
+ for (int k = 0; k < size; k++) {
+ buf = new StringBuilder(header);
+ buf.append("-");
+ for (int i = 0; i < 10; i++) {
+ buf.append(Math.abs(rand3.nextInt()) % 10);
+ }
+ samples[k] = buf.toString();
+ }
+
+ return samples;
+ }
+
+ @Test
+ public void countTest() throws IOException {
+ int n = 10;
+ for (int i = 0; i < 5; i++) {
+ count(n);
+ n *= 10;
+ }
+ }
+
+ private void count(int n) throws IOException {
+ Set<String> testSet = generateTestData(n);
+
+ HyperLogLogPlusCounter hllc = newHLLC();
+ for (String testData : testSet) {
+ hllc.add(Bytes.toBytes(testData));
+ }
+ long estimate = hllc.getCountEstimate();
+ double errorRate = hllc.getErrorRate();
+ double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
+ System.out.println(estimate);
+ System.out.println(testSet.size());
+ System.out.println(errorRate);
+ System.out.println("=" + actualError);
+ Assert.assertTrue(actualError < errorRate * 3.0);
+
+ checkSerialize(hllc);
+ }
+
+ private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException {
+ long estimate = hllc.getCountEstimate();
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ hllc.readRegisters(buf);
+ Assert.assertEquals(estimate, hllc.getCountEstimate());
+ }
+
+ @Test
+ public void mergeTest() throws IOException {
+ double error = 0;
+ double absError = 0;
+ int n = 100;
+ for (int i = 0; i < n; i++) {
+ System.out.println("============" + i);
+ double e = merge();
+ error += e;
+ absError += Math.abs(e);
+ }
+ System.out.println("Total average error is " + error / n + " and absolute error is " + absError / n);
+
+ System.out.println("errorCount1 is " + errorCount1 + "!");
+ System.out.println("errorCount2 is " + errorCount2 + "!");
+ System.out.println("errorCount3 is " + errorCount3 + "!");
+
+ Assert.assertTrue(errorCount1 <= n * 0.40);
+ Assert.assertTrue(errorCount2 <= n * 0.08);
+ Assert.assertTrue(errorCount3 <= n * 0.02);
+ }
+
+ private double merge() throws IOException {
+
+ int ln = 50;
+ int dn = 300;
+ Set<String> testSet = new HashSet<String>();
+ HyperLogLogPlusCounter[] hllcs = new HyperLogLogPlusCounter[ln];
+ for (int i = 0; i < ln; i++) {
+ hllcs[i] = newHLLC();
+ for (int k = 0; k < dn; k++) {
+ String[] samples = generateSampleData();
+ for (String data : samples) {
+ testSet.add(data);
+ hllcs[i].add(Bytes.toBytes(data));
+ }
+ }
+ }
+ HyperLogLogPlusCounter mergeHllc = newHLLC();
+ for (HyperLogLogPlusCounter hllc : hllcs) {
+ mergeHllc.merge(hllc);
+ checkSerialize(mergeHllc);
+ }
+
+ double errorRate = mergeHllc.getErrorRate();
+ long estimate = mergeHllc.getCountEstimate();
+ double actualError = (double) (testSet.size() - estimate) / testSet.size();
+
+ System.out.println(testSet.size() + "-" + estimate);
+
+ System.out.println("=" + actualError);
+ if (Math.abs(actualError) > errorRate) {
+ errorCount1++;
+ }
+ if (Math.abs(actualError) > 2 * errorRate) {
+ errorCount2++;
+ }
+ if (Math.abs(actualError) > 3 * errorRate) {
+ errorCount3++;
+ }
+
+ return actualError;
+ }
+
+ @Test
+ public void testPerformance() throws IOException {
+ int N = 3; // reduce N HLLC into one
+ int M = 1000; // for M times, use 100000 for real perf test
+
+ HyperLogLogPlusCounter samples[] = new HyperLogLogPlusCounter[N];
+ for (int i = 0; i < N; i++) {
+ samples[i] = newHLLC();
+ for (String str : generateTestData(10000))
+ samples[i].add(str);
+ }
+
+ System.out.println("Perf test running ... ");
+ long start = System.currentTimeMillis();
+ HyperLogLogPlusCounter sum = newHLLC();
+ for (int i = 0; i < M; i++) {
+ sum.clear();
+ for (int j = 0; j < N; j++) {
+ sum.merge(samples[j]);
+ checkSerialize(sum);
+ }
+ }
+ long duration = System.currentTimeMillis() - start;
+ System.out.println("Perf test result: " + duration / 1000 + " seconds");
+ }
+
+ private HyperLogLogPlusCounter newHLLC() {
+ return new HyperLogLogPlusCounter(16);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/util/InstallJarIntoMavenTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/util/InstallJarIntoMavenTest.java b/common/src/test/java/com/kylinolap/common/util/InstallJarIntoMavenTest.java
new file mode 100644
index 0000000..0bc77d5
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/util/InstallJarIntoMavenTest.java
@@ -0,0 +1,45 @@
+package com.kylinolap.common.util;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.junit.Ignore;
+
+/**
+ * Created by honma on 6/6/14.
+ */
+public class InstallJarIntoMavenTest {
+
+ @Ignore
+ public void testInstall() throws IOException {
+ File folder = new File("/export/home/b_kylin/tmp");
+ File out = new File("/export/home/b_kylin/tmp/out.sh");
+ out.createNewFile();
+ FileWriter fw = new FileWriter(out);
+
+ for (File file : folder.listFiles()) {
+ String name = file.getName();
+
+ if (!name.endsWith(".jar"))
+ continue;
+
+ int firstSlash = name.indexOf('-');
+ int lastDot = name.lastIndexOf('.');
+ String groupId = name.substring(0, firstSlash);
+
+ Pattern pattern = Pattern.compile("-\\d");
+ Matcher match = pattern.matcher(name);
+ match.find();
+ String artifactId = name.substring(0, match.start());
+ String version = name.substring(match.start() + 1, lastDot);
+
+ fw.write(String.format("mvn install:install-file -Dfile=%s -DgroupId=%s -DartifactId=%s -Dversion=%s -Dpackaging=jar", name, "org.apache." + groupId, artifactId, version));
+ fw.write("\n");
+ }
+ fw.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java b/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java
new file mode 100644
index 0000000..044d134
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kylinolap.common.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.kylinolap.common.KylinConfig;
+
+public class MailServiceTest {
+
+ @Test
+ public void testSendEmail() throws IOException {
+
+ @SuppressWarnings("deprecation")
+ KylinConfig config = KylinConfig.getInstanceForTest(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ MailService mailservice = new MailService(config);
+ boolean sent = sendTestEmail(mailservice);
+ assert sent;
+
+ // set mail.enabled=false, and run again, this time should be no mail delviered
+ config.setProperty(KylinConfig.MAIL_ENABLED, "false");
+ mailservice = new MailService(config);
+ sent = sendTestEmail(mailservice);
+ assert !sent;
+
+ }
+
+ private boolean sendTestEmail(MailService mailservice) {
+
+ List<String> receivers = new ArrayList<String>(1);
+ receivers.add("shaoshi@ebay.com");
+ try {
+ return mailservice.sendMail(receivers, "A test email from Kylin", "Hello!");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/util/RandomSamplerTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/util/RandomSamplerTest.java b/common/src/test/java/com/kylinolap/common/util/RandomSamplerTest.java
new file mode 100644
index 0000000..73e8e4a
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/util/RandomSamplerTest.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RandomSamplerTest {
+
+ @Test
+ public void test() {
+ RandomSampler<Text> s = new RandomSampler<Text>();
+ List<Text> data = new ArrayList<Text>();
+ for (int i = 0; i < 1000; i++) {
+ data.add(new Text(String.valueOf(i)));
+ }
+
+ List<Text> result = s.sample(data, 50);
+ System.out.println(result);
+ assertEquals(50, result.size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/test/java/com/kylinolap/common/util/SSHClientTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/kylinolap/common/util/SSHClientTest.java b/common/src/test/java/com/kylinolap/common/util/SSHClientTest.java
new file mode 100644
index 0000000..4dd465b
--- /dev/null
+++ b/common/src/test/java/com/kylinolap/common/util/SSHClientTest.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * @author ysong1
+ *
+ */
+public class SSHClientTest extends LocalFileMetadataTestCase {
+
+ private boolean isRemote;
+ private String hostname;
+ private String username;
+ private String password;
+
+ private void loadPropertiesFile() throws IOException {
+
+ KylinConfig cfg = KylinConfig.getInstanceFromEnv();
+
+ this.isRemote = cfg.getRunAsRemoteCommand();
+ this.hostname = cfg.getRemoteHadoopCliHostname();
+ this.username = cfg.getRemoteHadoopCliUsername();
+ this.password = cfg.getRemoteHadoopCliPassword();
+ }
+
+ @Before
+ public void before() throws Exception {
+ this.createTestMetadata();
+ loadPropertiesFile();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCmd() throws Exception {
+ if (isRemote == false)
+ return;
+
+ SSHClient ssh = new SSHClient(this.hostname, this.username, this.password, null);
+ SSHClientOutput output = ssh.execCommand("echo hello");
+ assertEquals(0, output.getExitCode());
+ assertEquals("hello\n", output.getText());
+ }
+
+ @Test
+ public void testScp() throws Exception {
+ if (isRemote == false)
+ return;
+
+ SSHClient ssh = new SSHClient(this.hostname, this.username, this.password, null);
+ File tmpFile = FileUtil.createLocalTempFile(new File("/tmp/test_scp"), "temp_", false);
+ ssh.scpFileToRemote(tmpFile.getAbsolutePath(), "/tmp");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/cube/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/cube/.settings/org.eclipse.core.resources.prefs b/cube/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..04cfa2c
--- /dev/null
+++ b/cube/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,6 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding//src/test/resources=UTF-8
+encoding/<project>=UTF-8