You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/07 15:47:06 UTC
[45/51] [partial] incubator-kylin git commit: migrate repo from
github.com to apache git
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
new file mode 100644
index 0000000..3e101af
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
@@ -0,0 +1,307 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.common.util.HadoopUtil;
+
+public class HBaseResourceStore extends ResourceStore {
+
+ private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
+ private static final String FAMILY = "f";
+ private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
+ private static final String COLUMN = "c";
+ private static final byte[] B_COLUMN = Bytes.toBytes(COLUMN);
+ private static final String COLUMN_TS = "t";
+ private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS);
+
+ private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>();
+
+ static {
+ TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube");
+ TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict");
+ TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex");
+ TABLE_SUFFIX_MAP.put(JOB_PATH_ROOT + "/", "_job");
+ TABLE_SUFFIX_MAP.put(JOB_OUTPUT_PATH_ROOT + "/", "_job_output");
+ TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj");
+ TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot");
+ TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE
+ }
+
+ final String tableNameBase;
+ final String hbaseUrl;
+
+ final Map<String, String> tableNameMap; // path prefix ==> HBase table name
+
+ private HConnection getConnection() throws IOException {
+ return HBaseConnection.get(hbaseUrl);
+ }
+
+ public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
+ super(kylinConfig);
+
+ String metadataUrl = kylinConfig.getMetadataUrl();
+ // split TABLE@HBASE_URL
+ int cut = metadataUrl.indexOf('@');
+ tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
+ hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+
+ tableNameMap = new LinkedHashMap<String, String>();
+ for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) {
+ String pathPrefix = entry.getKey();
+ String tableName = tableNameBase + entry.getValue();
+ tableNameMap.put(pathPrefix, tableName);
+ createHTableIfNeeded(tableName);
+ }
+
+ }
+
+ private void createHTableIfNeeded(String tableName) throws IOException {
+ HBaseConnection.createHTableIfNeeded(getConnection(), tableName, FAMILY);
+ }
+
+ private String getTableName(String path) {
+ for (Entry<String, String> entry : tableNameMap.entrySet()) {
+ String pathPrefix = entry.getKey();
+ if (path.startsWith(pathPrefix))
+ return entry.getValue();
+ }
+ throw new IllegalStateException("failed to find HBase table for path -- " + path);
+ }
+
+ @Override
+ protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
+ assert resPath.startsWith("/");
+ String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
+ byte[] startRow = Bytes.toBytes(lookForPrefix);
+ byte[] endRow = Bytes.toBytes(lookForPrefix);
+ endRow[endRow.length - 1]++;
+
+ ArrayList<String> result = new ArrayList<String>();
+
+ for (Entry<String, String> entry : tableNameMap.entrySet()) {
+ String pathPrefix = entry.getKey();
+ String tableName = entry.getValue();
+
+ if ((pathPrefix.startsWith(lookForPrefix) || lookForPrefix.startsWith(pathPrefix)) == false)
+ continue;
+
+ HTableInterface table = getConnection().getTable(tableName);
+
+ Scan scan = new Scan(startRow, endRow);
+ scan.setFilter(new KeyOnlyFilter());
+ try {
+ ResultScanner scanner = table.getScanner(scan);
+ for (Result r : scanner) {
+ String path = Bytes.toString(r.getRow());
+ assert path.startsWith(lookForPrefix);
+ int cut = path.indexOf('/', lookForPrefix.length());
+ String child = cut < 0 ? path : path.substring(0, cut);
+ if (result.contains(child) == false)
+ result.add(child);
+ }
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+ // return null to indicate not a folder
+ return result.isEmpty() ? null : result;
+ }
+
+ @Override
+ protected boolean existsImpl(String resPath) throws IOException {
+ Result r = getByScan(resPath, null, null);
+ return r != null;
+ }
+
+ @Override
+ protected InputStream getResourceImpl(String resPath) throws IOException {
+ Result r = getByScan(resPath, B_FAMILY, B_COLUMN);
+ if (r == null)
+ return null;
+
+ byte[] value = r.getValue(B_FAMILY, B_COLUMN);
+ if (value.length == 0) {
+ Path redirectPath = bigCellHDFSPath(resPath);
+ Configuration hconf = HadoopUtil.getDefaultConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+
+ return fileSystem.open(redirectPath);
+ } else {
+ return new ByteArrayInputStream(value);
+ }
+ }
+
+ @Override
+ protected long getResourceTimestampImpl(String resPath) throws IOException {
+ Result r = getByScan(resPath, B_FAMILY, B_COLUMN_TS);
+ if (r == null)
+ return 0;
+ else
+ return Bytes.toLong(r.getValue(B_FAMILY, B_COLUMN_TS));
+ }
+
+ @Override
+ protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ IOUtils.copy(content, bout);
+ bout.close();
+
+ HTableInterface table = getConnection().getTable(getTableName(resPath));
+ try {
+ byte[] row = Bytes.toBytes(resPath);
+ Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
+
+ table.put(put);
+ table.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ @Override
+ protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
+ HTableInterface table = getConnection().getTable(getTableName(resPath));
+ try {
+ byte[] row = Bytes.toBytes(resPath);
+ byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
+ Put put = buildPut(resPath, newTS, row, content, table);
+
+ boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
+ if (!ok)
+ throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + getResourceTimestamp(resPath));
+
+ table.flushCommits();
+
+ return newTS;
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ @Override
+ protected void deleteResourceImpl(String resPath) throws IOException {
+ HTableInterface table = getConnection().getTable(getTableName(resPath));
+ try {
+ Delete del = new Delete(Bytes.toBytes(resPath));
+ table.delete(del);
+ table.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ @Override
+ protected String getReadableResourcePathImpl(String resPath) {
+ return tableNameBase + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl();
+ }
+
+ private Result getByScan(String path, byte[] family, byte[] column) throws IOException {
+ byte[] startRow = Bytes.toBytes(path);
+ byte[] endRow = plusZero(startRow);
+
+ Scan scan = new Scan(startRow, endRow);
+ if (family == null || column == null) {
+ scan.setFilter(new KeyOnlyFilter());
+ } else {
+ scan.addColumn(family, column);
+ }
+
+ HTableInterface table = getConnection().getTable(getTableName(path));
+ try {
+ ResultScanner scanner = table.getScanner(scan);
+ Result result = null;
+ for (Result r : scanner) {
+ result = r;
+ }
+ return result == null || result.isEmpty() ? null : result;
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ private byte[] plusZero(byte[] startRow) {
+ byte[] endRow = Arrays.copyOf(startRow, startRow.length + 1);
+ endRow[endRow.length - 1] = 0;
+ return endRow;
+ }
+
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ Path redirectPath = bigCellHDFSPath(resPath);
+ Configuration hconf = HadoopUtil.getDefaultConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+
+ if (fileSystem.exists(redirectPath)) {
+ fileSystem.delete(redirectPath, true);
+ }
+
+ FSDataOutputStream out = fileSystem.create(redirectPath);
+
+ try {
+ out.write(largeColumn);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+
+ return redirectPath;
+ }
+
+ public Path bigCellHDFSPath(String resPath) {
+ String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory();
+ Path redirectPath = new Path(hdfsWorkingDirectory, "resources" + resPath);
+ return redirectPath;
+ }
+
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
+ if (content.length > kvSizeLimit) {
+ writeLargeCellToHdfs(resPath, content, table);
+ content = BytesUtil.EMPTY_BYTE_ARRAY;
+ }
+
+ Put put = new Put(row);
+ put.add(B_FAMILY, B_COLUMN, content);
+ put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+
+ return put;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java b/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
new file mode 100644
index 0000000..1471d09
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.persistence;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import com.kylinolap.common.util.JsonUtil;
+
+/**
+ * @author yangli9
+ */
+public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> {
+
+ Class<T> clz;
+
+ public JsonSerializer(Class<T> clz) {
+ this.clz = clz;
+ }
+
+ @Override
+ public T deserialize(DataInputStream in) throws IOException {
+ return JsonUtil.readValue(in, clz);
+ }
+
+ @Override
+ public void serialize(T obj, DataOutputStream out) throws IOException {
+ JsonUtil.writeValueIndent(out, obj);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
new file mode 100644
index 0000000..36abfdd
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.kylinolap.common.KylinConfig;
+
+abstract public class ResourceStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class);
+
+ public static final String CUBE_RESOURCE_ROOT = "/cube";
+ public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc";
+ public static final String DICT_RESOURCE_ROOT = "/dict";
+ public static final String IIDESC_RESOURCE_ROOT = "/invertedindex_desc";
+ public static final String JOB_PATH_ROOT = "/job";
+ public static final String JOB_OUTPUT_PATH_ROOT = "/job_output";
+ public static final String PROJECT_RESOURCE_ROOT = "/project";
+ public static final String SNAPSHOT_RESOURCE_ROOT = "/table_snapshot";
+ public static final String TABLE_EXD_RESOURCE_ROOT = "/table_exd";
+ public static final String TABLE_RESOURCE_ROOT = "/table";
+
+ private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
+
+ public static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
+
+ static {
+ knownImpl.add(HBaseResourceStore.class);
+ knownImpl.add(FileResourceStore.class);
+ }
+
+ public static ResourceStore getStore(KylinConfig kylinConfig) {
+ ResourceStore r = CACHE.get(kylinConfig);
+ List<Throwable> es = new ArrayList<Throwable>();
+ if (r == null) {
+ logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
+ for (Class<? extends ResourceStore> cls : knownImpl) {
+
+ try {
+ r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
+ } catch (Exception e) {
+ es.add(e);
+ } catch (NoClassDefFoundError er) {
+ // may throw NoClassDefFoundError
+ es.add(er);
+ }
+ if (r != null) {
+ break;
+ }
+ }
+ if (r == null) {
+ for (Throwable exceptionOrError : es) {
+ logger.error("Create new store instance failed ", exceptionOrError);
+ }
+ throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
+ }
+
+ CACHE.put(kylinConfig, r);
+ }
+ return r;
+ }
+
+ // ============================================================================
+
+ KylinConfig kylinConfig;
+
+ ResourceStore(KylinConfig kylinConfig) {
+ this.kylinConfig = kylinConfig;
+ }
+
+ /**
+ * return a list of child resources & folders under given path, return null
+ * if given path is not a folder
+ */
+ final public ArrayList<String> listResources(String resPath) throws IOException {
+ resPath = norm(resPath);
+ return listResourcesImpl(resPath);
+ }
+
+ abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException;
+
+ /**
+ * return true if a resource exists, return false in case of folder or
+ * non-exist
+ */
+ final public boolean exists(String resPath) throws IOException {
+ return existsImpl(norm(resPath));
+ }
+
+ abstract protected boolean existsImpl(String resPath) throws IOException;
+
+ /**
+ * read a resource, return null in case of not found
+ */
+ final public <T extends RootPersistentEntity> T getResource(String resPath, Class<T> clz, Serializer<T> serializer) throws IOException {
+ resPath = norm(resPath);
+ InputStream in = getResourceImpl(resPath);
+ if (in == null)
+ return null;
+
+ DataInputStream din = new DataInputStream(in);
+ try {
+ T r = serializer.deserialize(din);
+ r.setLastModified(getResourceTimestamp(resPath));
+ return r;
+ } finally {
+ IOUtils.closeQuietly(din);
+ IOUtils.closeQuietly(in);
+ }
+ }
+
+ final public InputStream getResource(String resPath) throws IOException {
+ return getResourceImpl(norm(resPath));
+ }
+
+ abstract protected InputStream getResourceImpl(String resPath) throws IOException;
+
+ final public long getResourceTimestamp(String resPath) throws IOException {
+ return getResourceTimestampImpl(norm(resPath));
+ }
+
+ abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
+
+ /**
+ * overwrite a resource without write conflict check
+ */
+ final public void putResource(String resPath, InputStream content, long ts) throws IOException {
+ resPath = norm(resPath);
+ logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+ putResourceImpl(resPath, content, ts);
+ }
+
+ abstract protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException;
+
+ /**
+ * check & set, overwrite a resource
+ */
+ final public <T extends RootPersistentEntity> void putResource(String resPath, T obj, Serializer<T> serializer) throws IOException {
+ resPath = norm(resPath);
+ logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+
+ long oldTS = obj.getLastModified();
+ long newTS = System.currentTimeMillis();
+ obj.setLastModified(newTS);
+
+ try {
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ DataOutputStream dout = new DataOutputStream(buf);
+ serializer.serialize(obj, dout);
+ dout.close();
+ buf.close();
+
+ newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS, newTS);
+ obj.setLastModified(newTS); // update again the confirmed TS
+
+ } catch (IOException e) {
+ obj.setLastModified(oldTS); // roll back TS when write fail
+ throw e;
+ } catch (RuntimeException e) {
+ obj.setLastModified(oldTS); // roll back TS when write fail
+ throw e;
+ }
+ }
+
+ /**
+ * checks old timestamp when overwriting existing
+ */
+ abstract protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException;
+
+ /**
+ * delete a resource, does nothing on a folder
+ */
+ final public void deleteResource(String resPath) throws IOException {
+ logger.debug("Deleting resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+ deleteResourceImpl(norm(resPath));
+ }
+
+ abstract protected void deleteResourceImpl(String resPath) throws IOException;
+
+ /**
+ * get a readable string of a resource path
+ */
+ final public String getReadableResourcePath(String resPath) {
+ return getReadableResourcePathImpl(norm(resPath));
+ }
+
+ abstract protected String getReadableResourcePathImpl(String resPath);
+
+ private String norm(String resPath) {
+ resPath = resPath.trim();
+ while (resPath.startsWith("//"))
+ resPath = resPath.substring(1);
+ while (resPath.endsWith("/"))
+ resPath = resPath.substring(0, resPath.length() - 1);
+ if (resPath.startsWith("/") == false)
+ resPath = "/" + resPath;
+ return resPath;
+ }
+
+ // ============================================================================
+
+ public static interface Visitor {
+ void visit(String path) throws IOException;
+ }
+
+ public void scanRecursively(String path, Visitor visitor) throws IOException {
+ ArrayList<String> children = listResources(path);
+ if (children != null) {
+ for (String child : children)
+ scanRecursively(child, visitor);
+ return;
+ }
+
+ if (exists(path))
+ visitor.visit(path);
+ }
+
+ public List<String> collectResourceRecursively(String root, final String suffix) throws IOException {
+ final ArrayList<String> collector = Lists.newArrayList();
+ scanRecursively(root, new Visitor() {
+ @Override
+ public void visit(String path) {
+ if (path.endsWith(suffix))
+ collector.add(path);
+ }
+ });
+ return collector;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
new file mode 100644
index 0000000..66783ca
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.StringUtil;
+
+public class ResourceTool {
+
+ private static String[] excludes = null;
+
+ public static void main(String[] args) throws IOException {
+ args = StringUtil.filterSystemArgs(args);
+
+ if (args.length == 0) {
+ System.out.println("Usage: MetadataTool reset METADATA_URI");
+ System.out.println("Usage: MetadataTool copy METADATA_URI_SRC METADATA_URI_DST");
+ System.out.println("Usage: MetadataTool list METADATA_URI PATH");
+ return;
+ }
+
+ String exclude = System.getProperty("exclude");
+ if (exclude != null) {
+ excludes = exclude.split("\\s*,\\s*");
+ }
+
+ String cmd = args[0];
+ if (cmd.equals("reset"))
+ reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1]));
+ else if (cmd.equals("copy"))
+ copy(args[1], args[2]);
+ else if (cmd.equals("list"))
+ list(args[1], args[2]);
+ else if (cmd.equals("download"))
+ copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
+ else if (cmd.equals("upload"))
+ copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
+ else if (cmd.equals("remove"))
+ remove(KylinConfig.getInstanceFromEnv(), args[1]);
+ else
+ System.out.println("Unknown cmd: " + cmd);
+ }
+
+ public static void list(KylinConfig config, String path) throws IOException {
+ ResourceStore store = ResourceStore.getStore(config);
+ ArrayList<String> result = store.listResources(path);
+ System.out.println("" + result);
+ }
+
+ private static void list(String metadataUri, String path) throws IOException {
+ KylinConfig config = KylinConfig.createInstanceFromUri(metadataUri);
+ list(config, path);
+ }
+
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
+
+ ResourceStore src = ResourceStore.getStore(srcConfig);
+ ResourceStore dst = ResourceStore.getStore(dstConfig);
+ copyR(src, dst, "/");
+ }
+
+ private static void copy(String srcUri, String dstUri) throws IOException {
+
+ System.out.println("Copy from " + srcUri + " to " + dstUri);
+
+ KylinConfig srcConfig = KylinConfig.createInstanceFromUri(srcUri);
+ KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dstUri);
+ copy(srcConfig, dstConfig);
+
+ }
+
+ private static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
+ ArrayList<String> children = src.listResources(path);
+
+ // case of resource (not a folder)
+ if (children == null) {
+ if (matchExclude(path) == false) {
+ InputStream content = src.getResource(path);
+ long ts = src.getResourceTimestamp(path);
+ if (content != null)
+ dst.putResource(path, content, ts);
+ else
+ System.out.println("Null inputstream for " + path);
+ }
+ }
+ // case of folder
+ else {
+ for (String child : children)
+ copyR(src, dst, child);
+ }
+ }
+
+ private static boolean matchExclude(String path) {
+ if (excludes == null)
+ return false;
+ for (String exclude : excludes) {
+ if (path.startsWith(exclude))
+ return true;
+ }
+ return false;
+ }
+
+ public static void reset(KylinConfig config) throws IOException {
+ ResourceStore store = ResourceStore.getStore(config);
+ resetR(store, "/");
+ }
+
+ private static void resetR(ResourceStore store, String path) throws IOException {
+ ArrayList<String> children = store.listResources(path);
+ if (children == null) { // path is a resource (not a folder)
+ if (matchExclude(path) == false) {
+ store.deleteResource(path);
+ }
+ } else {
+ for (String child : children)
+ resetR(store, child);
+ }
+ }
+
+ private static void remove(KylinConfig config, String path) throws IOException {
+ ResourceStore store = ResourceStore.getStore(config);
+ resetR(store, path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
new file mode 100644
index 0000000..d19d583
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.util.UUID;
+
+import org.apache.commons.lang.time.FastDateFormat;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Marks the root entity of JSON persistence. Unit of read, write, cache, and
+ * refresh.
+ *
+ * - CubeInstance - CubeDesc - SourceTable - JobMeta - Dictionary (not JSON but
+ * also top level persistence)
+ *
+ * @author yangli9
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+abstract public class RootPersistentEntity implements AclEntity {
+
+ static FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss z");
+
+ public static String formatTime(long millis) {
+ return format.format(millis);
+ }
+
+ // ============================================================================
+
+ @JsonProperty("uuid")
+ protected String uuid;
+
+ @JsonProperty("last_modified")
+ protected long lastModified;
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public void setUuid(String uuid) {
+ this.uuid = uuid;
+ }
+
+ public String getId() {
+ return uuid;
+ }
+
+ public long getLastModified() {
+ return lastModified;
+ }
+
+ public void setLastModified(long lastModified) {
+ this.lastModified = lastModified;
+ }
+
+ public void updateRandomUuid() {
+ setUuid(UUID.randomUUID().toString());
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (lastModified ^ (lastModified >>> 32));
+ result = prime * result + ((uuid == null) ? 0 : uuid.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ RootPersistentEntity other = (RootPersistentEntity) obj;
+ if (lastModified != other.lastModified)
+ return false;
+ if (uuid == null) {
+ if (other.uuid != null)
+ return false;
+ } else if (!uuid.equals(other.uuid))
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/Serializer.java b/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
new file mode 100644
index 0000000..d063745
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.persistence;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+/**
+ * @author yangli9
+ *
+ */
+public interface Serializer<T extends RootPersistentEntity> {
+
+ public void serialize(T obj, DataOutputStream out) throws IOException;
+
+ public T deserialize(DataInputStream in) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/StorageException.java b/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
new file mode 100644
index 0000000..62f12cd
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class StorageException extends RuntimeException {
+
+ private static final long serialVersionUID = -3748712888242406257L;
+
+ public StorageException(String msg, Throwable t) {
+ super(msg, t);
+ }
+
+ public StorageException(String msg) {
+ super(msg);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
new file mode 100644
index 0000000..8bc2468
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+/**
+ * @author xjiang
+ *
+ */
+public abstract class AbstractRestCache<K, V> {
+
+ protected final Broadcaster.TYPE syncType;
+
+ protected AbstractRestCache(Broadcaster.TYPE syncType) {
+ this.syncType = syncType;
+ }
+
+ protected final void syncRemote(Object key, Broadcaster.EVENT syncAction) {
+ String syncKey = (syncType == Broadcaster.TYPE.METADATA) ? "metadata" : key.toString();
+ Broadcaster.queue(syncType.getType(), syncAction.getType(), syncKey);
+ }
+
+ public abstract void put(K key, V value);
+
+ public abstract void putLocal(K key, V value);
+
+ public abstract void remove(K key);
+
+ public abstract void removeLocal(K key);
+
+ public abstract void clear();
+
+ public abstract int size();
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
new file mode 100644
index 0000000..60f0963
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * Broadcast kylin event out
+ *
+ * @author jianliu
+ *
+ */
+public class Broadcaster {
+
+ private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
+
+ private static List<BroadcastEvent> broadcaseEvents = new ArrayList<BroadcastEvent>();
+
+ static class BroadcasterHolder {
+ static final Broadcaster INSTANCE = new Broadcaster();
+ }
+
+ private Broadcaster() {
+ Timer timer = new Timer();
+ TimerTask task = new TimerTask() {
+ public void run() {
+ Broadcaster.flush();
+ }
+ };
+
+ timer.schedule(task, new Date(), 10 * 1000);
+ }
+
+ public Broadcaster getInstance() {
+ return BroadcasterHolder.INSTANCE;
+ }
+
+ public static void queueSyncMetadata() {
+ queue(TYPE.METADATA.getType(), EVENT.CREATE.getType(), "metadata");
+ }
+
+ /**
+ * Broadcast the cubedesc event out
+ *
+ * @param action
+ * event action
+ */
+ public static synchronized void queue(String type, String action, String key) {
+ BroadcastEvent event = BroadcasterHolder.INSTANCE.new BroadcastEvent(type, action, key);
+
+ if (!broadcaseEvents.contains(event)) {
+ broadcaseEvents.add(event);
+ }
+ }
+
+ public static synchronized void flush() {
+ String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
+ if (nodes == null)
+ return;
+
+ for (BroadcastEvent event : broadcaseEvents) {
+ for (String nodeUri : nodes) {
+ logger.debug("Broadcast nodeUri: " + nodeUri + ", type: " + event.getType() + ", action: " + event.getAction() + ", name: " + event.getName());
+ WipeCacheThread thread = BroadcasterHolder.INSTANCE.new WipeCacheThread(nodeUri, event.getType(), event.getAction(), event.getName());
+ thread.start();
+ }
+ }
+
+ broadcaseEvents.clear();
+ }
+
+ public static String genEventkey(String type, String action, String name) {
+ String time = String.valueOf(System.currentTimeMillis());
+ return time + "_" + type + "_" + action + "_" + name;
+ }
+
+ protected class WipeCacheThread extends Thread {
+ private String nodeUri;
+ private String type;
+ private String action;
+ private String name;
+
+ public WipeCacheThread(String nodeUri, String type, String action, String name) {
+ this.nodeUri = nodeUri;
+ this.type = type;
+ this.action = action;
+ this.name = name;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Thread#run()
+ */
+ @Override
+ public void run() {
+ RestClient restClient = new RestClient(nodeUri);
+ try {
+ restClient.wipeCache(this.type, this.action, this.name);
+ } catch (IOException e) {
+ logger.warn("Thread failed during wipe cache at " + type + "." + action + "." + name + ", " + e.toString());
+ }
+ }
+ }
+
+ public enum EVENT {
+ CREATE("create"), UPDATE("update"), DROP("drop");
+ private String text;
+
+ private EVENT(String text) {
+ this.text = text;
+ }
+
+ public String getType() {
+ return text;
+ }
+
+ public static EVENT getEvent(String event) {
+ for (EVENT one : values()) {
+ if (one.getType().equalsIgnoreCase(event)) {
+ return one;
+ }
+ }
+
+ return null;
+ }
+ }
+
+ public enum TYPE {
+ CUBE("cube"), METADATA("metadata"), PROJECT("project");
+ private String text;
+
+ private TYPE(String text) {
+ this.text = text;
+ }
+
+ public String getType() {
+ return text;
+ }
+
+ /**
+ * @param type
+ * @return
+ */
+ public static TYPE getType(String type) {
+ for (TYPE one : values()) {
+ if (one.getType().equalsIgnoreCase(type)) {
+ return one;
+ }
+ }
+
+ return null;
+ }
+ }
+
+ public class BroadcastEvent {
+ private String type;
+ private String action;
+ private String name;
+
+ public BroadcastEvent(String type, String action, String name) {
+ super();
+ this.type = type;
+ this.action = action;
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getAction() {
+ return action;
+ }
+
+ public void setAction(String action) {
+ this.action = action;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + getOuterType().hashCode();
+ result = prime * result + ((action == null) ? 0 : action.hashCode());
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + ((type == null) ? 0 : type.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ BroadcastEvent other = (BroadcastEvent) obj;
+ if (!getOuterType().equals(other.getOuterType()))
+ return false;
+ if (action == null) {
+ if (other.action != null)
+ return false;
+ } else if (!action.equals(other.action))
+ return false;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (type == null) {
+ if (other.type != null)
+ return false;
+ } else if (!type.equals(other.type))
+ return false;
+ return true;
+ }
+
+ private Broadcaster getOuterType() {
+ return Broadcaster.this;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
new file mode 100644
index 0000000..3ecaac6
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.util.Set;
+
+import com.google.common.collect.HashMultimap;
+
+/**
+ * @author xjiang
+ *
+ */
+public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
+
+ private final HashMultimap<K, V> innerCache;
+
+ public MultiValueCache(Broadcaster.TYPE syncType) {
+ super(syncType);
+ innerCache = HashMultimap.create();
+ }
+
+ public void put(K key, V value) {
+ Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
+ synchronized (this) {
+ innerCache.put(key, value);
+ }
+ syncRemote(key, eventType);
+ }
+
+ public void putLocal(K key, V value) {
+ synchronized (this) {
+ innerCache.put(key, value);
+ }
+ }
+
+ public void remove(K key) {
+ if (innerCache.containsKey(key)) {
+ innerCache.removeAll(key);
+ syncRemote(key, Broadcaster.EVENT.DROP);
+ }
+ }
+
+ public void removeLocal(K key) {
+ if (innerCache.containsKey(key)) {
+ innerCache.removeAll(key);
+ }
+ }
+
+ public void clear() {
+ innerCache.clear();
+ }
+
+ public int size() {
+ return innerCache.size();
+ }
+
+ public Set<V> get(K key) {
+ return innerCache.get(key);
+ }
+
+ public Set<K> keySet() {
+ return innerCache.keySet();
+ }
+
+ public boolean containsKey(Object key) {
+ return innerCache.containsKey(key);
+ }
+
+ public boolean containsEntry(Object key, Object value) {
+ return innerCache.containsEntry(key, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
new file mode 100644
index 0000000..5f1367f
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpException;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+/**
+ * @author yangli9
+ */
+public class RestClient {
+
+ String host;
+ int port;
+ String baseUrl;
+ String userName;
+ String password;
+ HttpClient client;
+
+ private static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
+
+ public static boolean matchFullRestPattern(String uri) {
+ Matcher m = fullRestPattern.matcher(uri);
+ return m.matches();
+ }
+
+ /**
+ * @param uri
+ * "user:pwd@host:port"
+ */
+ public RestClient(String uri) {
+ Matcher m = fullRestPattern.matcher(uri);
+ if (!m.matches())
+ throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern);
+
+ String user = m.group(1);
+ String pwd = m.group(2);
+ String host = m.group(3);
+ String portStr = m.group(4);
+ int port = Integer.parseInt(portStr == null ? "7070" : portStr);
+
+ init(host, port, user, pwd);
+ }
+
+ private void init(String host, int port, String userName, String password) {
+ this.host = host;
+ this.port = port;
+ this.userName = userName;
+ this.password = password;
+ this.baseUrl = "http://" + host + ":" + port + "/kylin/api";
+
+ client = new HttpClient();
+
+ if (userName != null && password != null) {
+ client.getParams().setAuthenticationPreemptive(true);
+ Credentials creds = new UsernamePasswordCredentials(userName, password);
+ client.getState().setCredentials(new AuthScope(host, port, AuthScope.ANY_REALM), creds);
+ }
+ }
+
+ public void wipeCache(String type, String action, String name) throws IOException {
+ String url = baseUrl + "/cache/" + type + "/" + name + "/" + action;
+ HttpMethod get = new PutMethod(url);
+
+ try {
+ int code = client.executeMethod(get);
+ String msg = Bytes.toString(get.getResponseBody());
+
+ if (code != 200)
+ throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
+
+ } catch (HttpException ex) {
+ throw new IOException(ex);
+ } finally {
+ get.releaseConnection();
+ }
+ }
+
+ public String getKylinProperties() throws IOException {
+ String url = baseUrl + "/admin/config";
+ HttpMethod get = new GetMethod(url);
+ try {
+ int code = client.executeMethod(get);
+ String msg = Bytes.toString(get.getResponseBody());
+ JSONObject obj = new JSONObject(msg);
+ msg = obj.getString("config");
+
+ if (code != 200)
+ throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
+
+ return msg;
+
+ } catch (JSONException e) {
+ throw new IOException("Error when parsing json response from REST");
+ } finally {
+ get.releaseConnection();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
new file mode 100644
index 0000000..394697c
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author xjiang
+ *
+ */
+public class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
+
+ private final Map<K, V> innerCache;
+
+ public SingleValueCache(Broadcaster.TYPE syncType) {
+ super(syncType);
+ innerCache = new ConcurrentHashMap<K, V>();
+ }
+
+ public void put(K key, V value) {
+ Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
+ innerCache.put(key, value);
+ syncRemote(key, eventType);
+ }
+
+ public void putLocal(K key, V value) {
+ innerCache.put(key, value);
+ }
+
+ public void remove(K key) {
+ if (innerCache.containsKey(key)) {
+ innerCache.remove(key);
+ syncRemote(key, Broadcaster.EVENT.DROP);
+ }
+ }
+
+ public void removeLocal(K key) {
+ innerCache.remove(key);
+ }
+
+ public void clear() {
+ innerCache.clear();
+ }
+
+ public int size() {
+ return innerCache.size();
+ }
+
+ public V get(K key) {
+ return innerCache.get(key);
+ }
+
+ public Collection<V> values() {
+ return innerCache.values();
+ }
+
+ public boolean containsKey(String key) {
+ return innerCache.containsKey(key);
+ }
+
+ public Map<K, V> getMap() {
+ return Collections.unmodifiableMap(innerCache);
+ }
+
+ public Set<K> keySet() {
+ return innerCache.keySet();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java b/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
new file mode 100644
index 0000000..9b1005e
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * @author ysong1
+ *
+ */
+public abstract class AbstractKylinTestCase {
+
+ public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
+
+ public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
+
+ public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
+
+ public abstract void createTestMetadata();
+
+ public abstract void cleanupTestMetadata();
+
+ public KylinConfig getTestConfig() {
+ return KylinConfig.getInstanceFromEnv();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/Array.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/Array.java b/common/src/main/java/com/kylinolap/common/util/Array.java
new file mode 100644
index 0000000..0b4165f
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/Array.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.util.Arrays;
+
+/*
+ * An array with correct equals(), hashCode(), compareTo() and toString()
+ */
+public class Array<T> implements Comparable<Array<T>> {
+ public T[] data;
+
+ public Array(T[] data) {
+ this.data = data;
+ }
+
+ public String toString() {
+ return Arrays.toString(data);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o != null && o instanceof Array) {
+ return Arrays.equals(this.data, ((Array<?>) o).data);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(data);
+ }
+
+ @Override
+ public int compareTo(Array<T> other) {
+ return compare(this.data, other.data, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> int compare(T[] a, T[] b, boolean[] ascending) {
+ int r = 0;
+ int n = Math.min(a.length, b.length);
+ boolean asc = true;
+
+ for (int i = 0; i < n; i++) {
+ r = ((Comparable<T>) a[i]).compareTo(b[i]);
+ if (r != 0) {
+ asc = (ascending != null && ascending.length > i) ? ascending[i] : true;
+ return asc ? r : -r;
+ }
+ }
+ return a.length - b.length;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/ByteArray.java b/common/src/main/java/com/kylinolap/common/util/ByteArray.java
new file mode 100644
index 0000000..d6aee89
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/ByteArray.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * @author yangli9
+ */
+public class ByteArray implements Comparable<ByteArray> {
+
+ public byte[] data;
+
+ public ByteArray(byte[] data) {
+ this.data = data;
+ }
+
+ @Override
+ public int hashCode() {
+ return Bytes.hashCode(data);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ByteArray other = (ByteArray) obj;
+ if (!Arrays.equals(data, other.data))
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(ByteArray o) {
+ return Bytes.compareTo(this.data, o.data);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java b/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
new file mode 100644
index 0000000..5d46037
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.nio.ByteBuffer;
+
+/**
+ * @author yangli9
+ *
+ */
+public interface BytesSerializer<T> {
+
+ public static final int SERIALIZE_BUFFER_SIZE = 65536;
+
+ abstract public void serialize(T value, ByteBuffer out);
+
+ abstract public T deserialize(ByteBuffer in);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesUtil.java b/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
new file mode 100644
index 0000000..93d72a4
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+
+public class BytesUtil {
+
+ public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+
+ public static void writeLong(long num, byte[] bytes, int offset, int size) {
+ for (int i = offset + size - 1; i >= offset; i--) {
+ bytes[i] = (byte) num;
+ num >>>= 8;
+ }
+ }
+
+ public static void writeUnsigned(int num, byte[] bytes, int offset, int size) {
+ for (int i = offset + size - 1; i >= offset; i--) {
+ bytes[i] = (byte) num;
+ num >>>= 8;
+ }
+ }
+
+ public static long readLong(byte[] bytes, int offset, int size) {
+ long integer = 0;
+ for (int i = offset, n = offset + size; i < n; i++) {
+ integer <<= 8;
+ integer |= (long) bytes[i] & 0xFF;
+ }
+ return integer;
+ }
+
+ public static int readUnsigned(byte[] bytes, int offset, int size) {
+ int integer = 0;
+ for (int i = offset, n = offset + size; i < n; i++) {
+ integer <<= 8;
+ integer |= (int) bytes[i] & 0xFF;
+ }
+ return integer;
+ }
+
+ public static void writeSigned(int num, byte[] bytes, int offset, int size) {
+ writeUnsigned(num, bytes, offset, size);
+ }
+
+ public static int readSigned(byte[] bytes, int offset, int size) {
+ int integer = (bytes[offset] & 0x80) == 0 ? 0 : -1;
+ for (int i = offset, n = offset + size; i < n; i++) {
+ integer <<= 8;
+ integer |= (int) bytes[i] & 0xFF;
+ }
+ return integer;
+ }
+
+ /**
+ * No. bytes needed to store a value as big as the given
+ */
+ public static int sizeForValue(int maxValue) {
+ int size = 0;
+ while (maxValue > 0) {
+ size++;
+ maxValue >>>= 8;
+ }
+ return size;
+ }
+
+ public static int compareByteUnsigned(byte b1, byte b2) {
+ int i1 = (int) b1 & 0xFF;
+ int i2 = (int) b2 & 0xFF;
+ return i1 - i2;
+ }
+
+ public static byte[] subarray(byte[] bytes, int start, int end) {
+ byte[] r = new byte[end - start];
+ System.arraycopy(bytes, start, r, 0, r.length);
+ return r;
+ }
+
+ public static int compareBytes(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) {
+ int r = 0;
+ for (int i = 0; i < length; i++) {
+ r = src[srcOffset + i] - dst[dstOffset + i];
+ if (r != 0)
+ break;
+ }
+ return r;
+ }
+
+ // from WritableUtils
+ // ============================================================================
+
+ public static void writeVInt(int i, ByteBuffer out) {
+ writeVLong(i, out);
+ }
+
+ public static void writeVLong(long i, ByteBuffer out) {
+ if (i >= -112 && i <= 127) {
+ out.put((byte) i);
+ return;
+ }
+
+ int len = -112;
+ if (i < 0) {
+ i ^= -1L; // take one's complement'
+ len = -120;
+ }
+
+ long tmp = i;
+ while (tmp != 0) {
+ tmp = tmp >> 8;
+ len--;
+ }
+
+ out.put((byte) len);
+
+ len = (len < -120) ? -(len + 120) : -(len + 112);
+
+ for (int idx = len; idx != 0; idx--) {
+ int shiftbits = (idx - 1) * 8;
+ long mask = 0xFFL << shiftbits;
+ out.put((byte) ((i & mask) >> shiftbits));
+ }
+ }
+
+ public static long readVLong(ByteBuffer in) {
+ byte firstByte = in.get();
+ int len = decodeVIntSize(firstByte);
+ if (len == 1) {
+ return firstByte;
+ }
+ long i = 0;
+ for (int idx = 0; idx < len - 1; idx++) {
+ byte b = in.get();
+ i = i << 8;
+ i = i | (b & 0xFF);
+ }
+ return (isNegativeVInt(firstByte) ? (i ^ -1L) : i);
+ }
+
+ public static int readVInt(ByteBuffer in) {
+ long n = readVLong(in);
+ if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) {
+ throw new IllegalArgumentException("value too long to fit in integer");
+ }
+ return (int) n;
+ }
+
+ private static boolean isNegativeVInt(byte value) {
+ return value < -120 || (value >= -112 && value < 0);
+ }
+
+ private static int decodeVIntSize(byte value) {
+ if (value >= -112) {
+ return 1;
+ } else if (value < -120) {
+ return -119 - value;
+ }
+ return -111 - value;
+ }
+
+ public static void writeUnsigned(int num, int size, ByteBuffer out) {
+ for (int i = 0; i < size; i++) {
+ out.put((byte) num);
+ num >>>= 8;
+ }
+ }
+
+ public static int readUnsigned(ByteBuffer in, int size) {
+ int integer = 0;
+ int mask = 0xff;
+ int shift = 0;
+ for (int i = 0; i < size; i++) {
+ integer |= (in.get() << shift) & mask;
+ mask = mask << 8;
+ shift += 8;
+ }
+ return integer;
+ }
+
+ public static void writeLong(long num, ByteBuffer out) {
+ for (int i = 0; i < 8; i++) {
+ out.put((byte) num);
+ num >>>= 8;
+ }
+ }
+
+ public static long readLong(ByteBuffer in) {
+ long integer = 0;
+ int mask = 0xff;
+ int shift = 0;
+ for (int i = 0; i < 8; i++) {
+ integer |= (in.get() << shift) & mask;
+ mask = mask << 8;
+ shift += 8;
+ }
+ return integer;
+ }
+
+ public static void writeUTFString(String str, ByteBuffer out) {
+ byte[] bytes = str == null ? null : Bytes.toBytes(str);
+ writeByteArray(bytes, out);
+ }
+
+ public static String readUTFString(ByteBuffer in) {
+ byte[] bytes = readByteArray(in);
+ return bytes == null ? null : Bytes.toString(bytes);
+ }
+
+ public static void writeAsciiString(String str, ByteBuffer out) {
+ if (str == null) {
+ BytesUtil.writeVInt(-1, out);
+ return;
+ }
+ int len = str.length();
+ BytesUtil.writeVInt(len, out);
+ for (int i = 0; i < len; i++) {
+ out.put((byte) str.charAt(i));
+ }
+ }
+
+ public static String readAsciiString(ByteBuffer in) {
+ int len = BytesUtil.readVInt(in);
+ if (len < 0) {
+ return null;
+ }
+ String result;
+ try {
+ if (in.hasArray()) {
+ int pos = in.position();
+ result = new String(in.array(), pos, len, "ISO-8859-1");
+ in.position(pos + len);
+ } else {
+ byte[] tmp = new byte[len];
+ in.get(tmp);
+ result = new String(tmp, "ISO-8859-1");
+ }
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ return result;
+ }
+
+ public static void writeAsciiStringArray(String[] strs, ByteBuffer out) {
+ writeVInt(strs.length, out);
+ for (int i = 0; i < strs.length; i++)
+ writeAsciiString(strs[i], out);
+ }
+
+ public static String[] readAsciiStringArray(ByteBuffer in) {
+ int len = readVInt(in);
+ String[] strs = new String[len];
+ for (int i = 0; i < len; i++)
+ strs[i] = readAsciiString(in);
+ return strs;
+ }
+
+ public static void writeIntArray(int[] array, ByteBuffer out) {
+ if (array == null) {
+ writeVInt(-1, out);
+ return;
+ }
+ writeVInt(array.length, out);
+ out.asIntBuffer().put(array);
+ }
+
+ public static int[] readIntArray(ByteBuffer in) {
+ int len = readVInt(in);
+ if (len < 0)
+ return null;
+ int[] array = new int[len];
+ in.asIntBuffer().get(array);
+ return array;
+ }
+
+ public static void writeByteArray(byte[] array, ByteBuffer out) {
+ if (array == null) {
+ writeVInt(-1, out);
+ return;
+ }
+ writeVInt(array.length, out);
+ out.put(array);
+ }
+
+ public static byte[] readByteArray(ByteBuffer in) {
+ int len = readVInt(in);
+ if (len < 0)
+ return null;
+
+ byte[] array = new byte[len];
+ in.get(array);
+ return array;
+ }
+
+ public static byte[] toBytes(Writable writable) {
+ try {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(bout);
+ writable.write(out);
+ out.close();
+ bout.close();
+ return bout.toByteArray();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static String toHex(byte[] array) {
+ return toHex(new ImmutableBytesWritable(array));
+ }
+
+ public static String toHex(ImmutableBytesWritable bytes) {
+ byte[] array = bytes.get();
+ int offset = bytes.getOffset();
+ int length = bytes.getLength();
+ StringBuilder sb = new StringBuilder(length * 4);
+ for (int i = 0; i < length; i++) {
+ int b = array[offset + i];
+ sb.append(String.format("\\x%02X", b & 0xFF));
+ }
+ return sb.toString();
+ }
+
+ public static void main(String[] args) throws Exception {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java b/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
new file mode 100644
index 0000000..6be3708
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+/**
+ * @author xduo
+ *
+ */
+public class ClasspathUtil {
+
+ public static void addClasspath(String path) throws Exception {
+ System.out.println("Adding path " + path + " to class path");
+ File file = new File(path);
+
+ if (file.exists()) {
+ URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+ Class<URLClassLoader> urlClass = URLClassLoader.class;
+ Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class });
+ method.setAccessible(true);
+ method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() });
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java b/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
new file mode 100644
index 0000000..3f4ab7b
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * @author yangli9
+ */
+public class CliCommandExecutor {
+
+ private String remoteHost;
+ private String remoteUser;
+ private String remotePwd;
+ private int remoteTimeoutSeconds = 3600;
+
+ public CliCommandExecutor() {
+ }
+
+ public void setRunAtRemote(String host, String user, String pwd) {
+ this.remoteHost = host;
+ this.remoteUser = user;
+ this.remotePwd = pwd;
+ }
+
+ public void setRunAtLocal() {
+ this.remoteHost = null;
+ this.remoteUser = null;
+ this.remotePwd = null;
+ }
+
+ public void copyFile(String localFile, String destDir) throws IOException {
+ if (remoteHost == null)
+ copyNative(localFile, destDir);
+ else
+ copyRemote(localFile, destDir);
+ }
+
+ private void copyNative(String localFile, String destDir) throws IOException {
+ File src = new File(localFile);
+ File dest = new File(destDir, src.getName());
+ FileUtils.copyFile(src, dest);
+ }
+
+ private void copyRemote(String localFile, String destDir) throws IOException {
+ SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
+ try {
+ ssh.scpFileToRemote(localFile, destDir);
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ }
+
+ public String execute(String command) throws IOException {
+ Pair<Integer, String> r;
+ if (remoteHost == null)
+ r = runNativeCommand(command);
+ else
+ r = runRemoteCommand(command);
+
+ if (r.getFirst() != 0)
+ throw new IOException("OS command error exit with " + r.getFirst() + " -- " + command + "\n" + r.getSecond());
+
+ return r.getSecond();
+ }
+
+ private Pair<Integer, String> runRemoteCommand(String command) throws IOException {
+ SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
+
+ SSHClientOutput sshOutput;
+ try {
+ sshOutput = ssh.execCommand(command, remoteTimeoutSeconds);
+ int exitCode = sshOutput.getExitCode();
+ String output = sshOutput.getText();
+ return new Pair<Integer, String>(exitCode, output);
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ }
+
+ private Pair<Integer, String> runNativeCommand(String command) throws IOException {
+ String[] cmd = new String[3];
+ String osName = System.getProperty("os.name");
+ if (osName.startsWith("Windows")) {
+ cmd[0] = "cmd.exe";
+ cmd[1] = "/C";
+ } else {
+ cmd[0] = "/bin/bash";
+ cmd[1] = "-c";
+ }
+ cmd[2] = command;
+
+ ProcessBuilder builder = new ProcessBuilder(cmd);
+ builder.redirectErrorStream(true);
+ Process proc = builder.start();
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ IOUtils.copy(proc.getInputStream(), buf);
+ String output = buf.toString("UTF-8");
+
+ try {
+ int exitCode = proc.waitFor();
+ return new Pair<Integer, String>(exitCode, output);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
new file mode 100644
index 0000000..a1dda9a
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * @author ysong1
+ */
+public class HBaseMetadataTestCase extends AbstractKylinTestCase {
+
+ public static void staticCreateTestMetadata() {
+
+ KylinConfig.destoryInstance();
+
+ if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
+ System.setProperty(KylinConfig.KYLIN_CONF, SANDBOX_TEST_DATA);
+
+ }
+
+ public static void staticCleanupTestMetadata() {
+ System.clearProperty(KylinConfig.KYLIN_CONF);
+ KylinConfig.destoryInstance();
+ }
+
+ @Override
+ public void createTestMetadata() {
+ staticCreateTestMetadata();
+ }
+
+ @Override
+ public void cleanupTestMetadata() {
+ staticCleanupTestMetadata();
+ }
+
+}