You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/07 15:47:06 UTC

[45/51] [partial] incubator-kylin git commit: migrate repo from github.com to apache git

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
new file mode 100644
index 0000000..3e101af
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
@@ -0,0 +1,307 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.BytesUtil;
+import com.kylinolap.common.util.HadoopUtil;
+
+public class HBaseResourceStore extends ResourceStore {
+
+    private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
+    private static final String FAMILY = "f";
+    private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
+    private static final String COLUMN = "c";
+    private static final byte[] B_COLUMN = Bytes.toBytes(COLUMN);
+    private static final String COLUMN_TS = "t";
+    private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS);
+
+    private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>();
+
+    static {
+        TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube");
+        TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict");
+        TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex");
+        TABLE_SUFFIX_MAP.put(JOB_PATH_ROOT + "/", "_job");
+        TABLE_SUFFIX_MAP.put(JOB_OUTPUT_PATH_ROOT + "/", "_job_output");
+        TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj");
+        TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot");
+        TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE
+    }
+
+    final String tableNameBase;
+    final String hbaseUrl;
+
+    final Map<String, String> tableNameMap; // path prefix ==> HBase table name
+
+    private HConnection getConnection() throws IOException {
+        return HBaseConnection.get(hbaseUrl);
+    }
+
+    public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
+        super(kylinConfig);
+
+        String metadataUrl = kylinConfig.getMetadataUrl();
+        // split TABLE@HBASE_URL
+        int cut = metadataUrl.indexOf('@');
+        tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
+        hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+
+        tableNameMap = new LinkedHashMap<String, String>();
+        for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) {
+            String pathPrefix = entry.getKey();
+            String tableName = tableNameBase + entry.getValue();
+            tableNameMap.put(pathPrefix, tableName);
+            createHTableIfNeeded(tableName);
+        }
+
+    }
+
+    private void createHTableIfNeeded(String tableName) throws IOException {
+        HBaseConnection.createHTableIfNeeded(getConnection(), tableName, FAMILY);
+    }
+
+    private String getTableName(String path) {
+        for (Entry<String, String> entry : tableNameMap.entrySet()) {
+            String pathPrefix = entry.getKey();
+            if (path.startsWith(pathPrefix))
+                return entry.getValue();
+        }
+        throw new IllegalStateException("failed to find HBase table for path -- " + path);
+    }
+
+    @Override
+    protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
+        assert resPath.startsWith("/");
+        String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
+        byte[] startRow = Bytes.toBytes(lookForPrefix);
+        byte[] endRow = Bytes.toBytes(lookForPrefix);
+        endRow[endRow.length - 1]++;
+
+        ArrayList<String> result = new ArrayList<String>();
+
+        for (Entry<String, String> entry : tableNameMap.entrySet()) {
+            String pathPrefix = entry.getKey();
+            String tableName = entry.getValue();
+
+            if ((pathPrefix.startsWith(lookForPrefix) || lookForPrefix.startsWith(pathPrefix)) == false)
+                continue;
+
+            HTableInterface table = getConnection().getTable(tableName);
+
+            Scan scan = new Scan(startRow, endRow);
+            scan.setFilter(new KeyOnlyFilter());
+            try {
+                ResultScanner scanner = table.getScanner(scan);
+                for (Result r : scanner) {
+                    String path = Bytes.toString(r.getRow());
+                    assert path.startsWith(lookForPrefix);
+                    int cut = path.indexOf('/', lookForPrefix.length());
+                    String child = cut < 0 ? path : path.substring(0, cut);
+                    if (result.contains(child) == false)
+                        result.add(child);
+                }
+            } finally {
+                IOUtils.closeQuietly(table);
+            }
+        }
+        // return null to indicate not a folder
+        return result.isEmpty() ? null : result;
+    }
+
+    @Override
+    protected boolean existsImpl(String resPath) throws IOException {
+        Result r = getByScan(resPath, null, null);
+        return r != null;
+    }
+
+    @Override
+    protected InputStream getResourceImpl(String resPath) throws IOException {
+        Result r = getByScan(resPath, B_FAMILY, B_COLUMN);
+        if (r == null)
+            return null;
+
+        byte[] value = r.getValue(B_FAMILY, B_COLUMN);
+        if (value.length == 0) {
+            Path redirectPath = bigCellHDFSPath(resPath);
+            Configuration hconf = HadoopUtil.getDefaultConfiguration();
+            FileSystem fileSystem = FileSystem.get(hconf);
+
+            return fileSystem.open(redirectPath);
+        } else {
+            return new ByteArrayInputStream(value);
+        }
+    }
+
+    @Override
+    protected long getResourceTimestampImpl(String resPath) throws IOException {
+        Result r = getByScan(resPath, B_FAMILY, B_COLUMN_TS);
+        if (r == null)
+            return 0;
+        else
+            return Bytes.toLong(r.getValue(B_FAMILY, B_COLUMN_TS));
+    }
+
+    @Override
+    protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
+        ByteArrayOutputStream bout = new ByteArrayOutputStream();
+        IOUtils.copy(content, bout);
+        bout.close();
+
+        HTableInterface table = getConnection().getTable(getTableName(resPath));
+        try {
+            byte[] row = Bytes.toBytes(resPath);
+            Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
+
+            table.put(put);
+            table.flushCommits();
+        } finally {
+            IOUtils.closeQuietly(table);
+        }
+    }
+
+    @Override
+    protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
+        HTableInterface table = getConnection().getTable(getTableName(resPath));
+        try {
+            byte[] row = Bytes.toBytes(resPath);
+            byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
+            Put put = buildPut(resPath, newTS, row, content, table);
+
+            boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
+            if (!ok)
+                throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + getResourceTimestamp(resPath));
+
+            table.flushCommits();
+
+            return newTS;
+        } finally {
+            IOUtils.closeQuietly(table);
+        }
+    }
+
+    @Override
+    protected void deleteResourceImpl(String resPath) throws IOException {
+        HTableInterface table = getConnection().getTable(getTableName(resPath));
+        try {
+            Delete del = new Delete(Bytes.toBytes(resPath));
+            table.delete(del);
+            table.flushCommits();
+        } finally {
+            IOUtils.closeQuietly(table);
+        }
+    }
+
+    @Override
+    protected String getReadableResourcePathImpl(String resPath) {
+        return tableNameBase + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl();
+    }
+
+    private Result getByScan(String path, byte[] family, byte[] column) throws IOException {
+        byte[] startRow = Bytes.toBytes(path);
+        byte[] endRow = plusZero(startRow);
+
+        Scan scan = new Scan(startRow, endRow);
+        if (family == null || column == null) {
+            scan.setFilter(new KeyOnlyFilter());
+        } else {
+            scan.addColumn(family, column);
+        }
+
+        HTableInterface table = getConnection().getTable(getTableName(path));
+        try {
+            ResultScanner scanner = table.getScanner(scan);
+            Result result = null;
+            for (Result r : scanner) {
+                result = r;
+            }
+            return result == null || result.isEmpty() ? null : result;
+        } finally {
+            IOUtils.closeQuietly(table);
+        }
+    }
+
+    private byte[] plusZero(byte[] startRow) {
+        byte[] endRow = Arrays.copyOf(startRow, startRow.length + 1);
+        endRow[endRow.length - 1] = 0;
+        return endRow;
+    }
+
+    private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+        Path redirectPath = bigCellHDFSPath(resPath);
+        Configuration hconf = HadoopUtil.getDefaultConfiguration();
+        FileSystem fileSystem = FileSystem.get(hconf);
+
+        if (fileSystem.exists(redirectPath)) {
+            fileSystem.delete(redirectPath, true);
+        }
+
+        FSDataOutputStream out = fileSystem.create(redirectPath);
+
+        try {
+            out.write(largeColumn);
+        } finally {
+            IOUtils.closeQuietly(out);
+        }
+
+        return redirectPath;
+    }
+
+    public Path bigCellHDFSPath(String resPath) {
+        String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory();
+        Path redirectPath = new Path(hdfsWorkingDirectory, "resources" + resPath);
+        return redirectPath;
+    }
+
+    private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+        int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
+        if (content.length > kvSizeLimit) {
+            writeLargeCellToHdfs(resPath, content, table);
+            content = BytesUtil.EMPTY_BYTE_ARRAY;
+        }
+
+        Put put = new Put(row);
+        put.add(B_FAMILY, B_COLUMN, content);
+        put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+
+        return put;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java b/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
new file mode 100644
index 0000000..1471d09
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.persistence;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import com.kylinolap.common.util.JsonUtil;
+
+/**
+ * @author yangli9
+ */
+public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> {
+
+    Class<T> clz;
+
+    public JsonSerializer(Class<T> clz) {
+        this.clz = clz;
+    }
+
+    @Override
+    public T deserialize(DataInputStream in) throws IOException {
+        return JsonUtil.readValue(in, clz);
+    }
+
+    @Override
+    public void serialize(T obj, DataOutputStream out) throws IOException {
+        JsonUtil.writeValueIndent(out, obj);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
new file mode 100644
index 0000000..36abfdd
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.kylinolap.common.KylinConfig;
+
+abstract public class ResourceStore {
+
+    private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class);
+
+    public static final String CUBE_RESOURCE_ROOT = "/cube";
+    public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc";
+    public static final String DICT_RESOURCE_ROOT = "/dict";
+    public static final String IIDESC_RESOURCE_ROOT = "/invertedindex_desc";
+    public static final String JOB_PATH_ROOT = "/job";
+    public static final String JOB_OUTPUT_PATH_ROOT = "/job_output";
+    public static final String PROJECT_RESOURCE_ROOT = "/project";
+    public static final String SNAPSHOT_RESOURCE_ROOT = "/table_snapshot";
+    public static final String TABLE_EXD_RESOURCE_ROOT = "/table_exd";
+    public static final String TABLE_RESOURCE_ROOT = "/table";
+
+    private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
+
+    public static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
+
+    static {
+        knownImpl.add(HBaseResourceStore.class);
+        knownImpl.add(FileResourceStore.class);
+    }
+
+    public static ResourceStore getStore(KylinConfig kylinConfig) {
+        ResourceStore r = CACHE.get(kylinConfig);
+        List<Throwable> es = new ArrayList<Throwable>();
+        if (r == null) {
+            logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
+            for (Class<? extends ResourceStore> cls : knownImpl) {
+
+                try {
+                    r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
+                } catch (Exception e) {
+                    es.add(e);
+                } catch (NoClassDefFoundError er) {
+                    // may throw NoClassDefFoundError
+                    es.add(er);
+                }
+                if (r != null) {
+                    break;
+                }
+            }
+            if (r == null) {
+                for (Throwable exceptionOrError : es) {
+                    logger.error("Create new store instance failed ", exceptionOrError);
+                }
+                throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
+            }
+
+            CACHE.put(kylinConfig, r);
+        }
+        return r;
+    }
+
+    // ============================================================================
+
+    KylinConfig kylinConfig;
+
+    ResourceStore(KylinConfig kylinConfig) {
+        this.kylinConfig = kylinConfig;
+    }
+
+    /**
+     * return a list of child resources & folders under given path, return null
+     * if given path is not a folder
+     */
+    final public ArrayList<String> listResources(String resPath) throws IOException {
+        resPath = norm(resPath);
+        return listResourcesImpl(resPath);
+    }
+
+    abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException;
+
+    /**
+     * return true if a resource exists, return false in case of folder or
+     * non-exist
+     */
+    final public boolean exists(String resPath) throws IOException {
+        return existsImpl(norm(resPath));
+    }
+
+    abstract protected boolean existsImpl(String resPath) throws IOException;
+
+    /**
+     * read a resource, return null in case of not found
+     */
+    final public <T extends RootPersistentEntity> T getResource(String resPath, Class<T> clz, Serializer<T> serializer) throws IOException {
+        resPath = norm(resPath);
+        InputStream in = getResourceImpl(resPath);
+        if (in == null)
+            return null;
+
+        DataInputStream din = new DataInputStream(in);
+        try {
+            T r = serializer.deserialize(din);
+            r.setLastModified(getResourceTimestamp(resPath));
+            return r;
+        } finally {
+            IOUtils.closeQuietly(din);
+            IOUtils.closeQuietly(in);
+        }
+    }
+
+    final public InputStream getResource(String resPath) throws IOException {
+        return getResourceImpl(norm(resPath));
+    }
+
+    abstract protected InputStream getResourceImpl(String resPath) throws IOException;
+
+    final public long getResourceTimestamp(String resPath) throws IOException {
+        return getResourceTimestampImpl(norm(resPath));
+    }
+
+    abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
+
+    /**
+     * overwrite a resource without write conflict check
+     */
+    final public void putResource(String resPath, InputStream content, long ts) throws IOException {
+        resPath = norm(resPath);
+        logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+        putResourceImpl(resPath, content, ts);
+    }
+
+    abstract protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException;
+
+    /**
+     * check & set, overwrite a resource
+     */
+    final public <T extends RootPersistentEntity> void putResource(String resPath, T obj, Serializer<T> serializer) throws IOException {
+        resPath = norm(resPath);
+        logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+
+        long oldTS = obj.getLastModified();
+        long newTS = System.currentTimeMillis();
+        obj.setLastModified(newTS);
+
+        try {
+            ByteArrayOutputStream buf = new ByteArrayOutputStream();
+            DataOutputStream dout = new DataOutputStream(buf);
+            serializer.serialize(obj, dout);
+            dout.close();
+            buf.close();
+
+            newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS, newTS);
+            obj.setLastModified(newTS); // update again the confirmed TS
+
+        } catch (IOException e) {
+            obj.setLastModified(oldTS); // roll back TS when write fail
+            throw e;
+        } catch (RuntimeException e) {
+            obj.setLastModified(oldTS); // roll back TS when write fail
+            throw e;
+        }
+    }
+
+    /**
+     * checks old timestamp when overwriting existing
+     */
+    abstract protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException;
+
+    /**
+     * delete a resource, does nothing on a folder
+     */
+    final public void deleteResource(String resPath) throws IOException {
+        logger.debug("Deleting resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+        deleteResourceImpl(norm(resPath));
+    }
+
+    abstract protected void deleteResourceImpl(String resPath) throws IOException;
+
+    /**
+     * get a readable string of a resource path
+     */
+    final public String getReadableResourcePath(String resPath) {
+        return getReadableResourcePathImpl(norm(resPath));
+    }
+
+    abstract protected String getReadableResourcePathImpl(String resPath);
+
+    private String norm(String resPath) {
+        resPath = resPath.trim();
+        while (resPath.startsWith("//"))
+            resPath = resPath.substring(1);
+        while (resPath.endsWith("/"))
+            resPath = resPath.substring(0, resPath.length() - 1);
+        if (resPath.startsWith("/") == false)
+            resPath = "/" + resPath;
+        return resPath;
+    }
+
+    // ============================================================================
+
+    public static interface Visitor {
+        void visit(String path) throws IOException;
+    }
+
+    public void scanRecursively(String path, Visitor visitor) throws IOException {
+        ArrayList<String> children = listResources(path);
+        if (children != null) {
+            for (String child : children)
+                scanRecursively(child, visitor);
+            return;
+        }
+
+        if (exists(path))
+            visitor.visit(path);
+    }
+
+    public List<String> collectResourceRecursively(String root, final String suffix) throws IOException {
+        final ArrayList<String> collector = Lists.newArrayList();
+        scanRecursively(root, new Visitor() {
+            @Override
+            public void visit(String path) {
+                if (path.endsWith(suffix))
+                    collector.add(path);
+            }
+        });
+        return collector;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
new file mode 100644
index 0000000..66783ca
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.StringUtil;
+
+public class ResourceTool {
+
+    private static String[] excludes = null;
+
+    public static void main(String[] args) throws IOException {
+        args = StringUtil.filterSystemArgs(args);
+
+        if (args.length == 0) {
+            System.out.println("Usage: MetadataTool reset METADATA_URI");
+            System.out.println("Usage: MetadataTool copy  METADATA_URI_SRC  METADATA_URI_DST");
+            System.out.println("Usage: MetadataTool list  METADATA_URI      PATH");
+            return;
+        }
+
+        String exclude = System.getProperty("exclude");
+        if (exclude != null) {
+            excludes = exclude.split("\\s*,\\s*");
+        }
+
+        String cmd = args[0];
+        if (cmd.equals("reset"))
+            reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1]));
+        else if (cmd.equals("copy"))
+            copy(args[1], args[2]);
+        else if (cmd.equals("list"))
+            list(args[1], args[2]);
+        else if (cmd.equals("download"))
+            copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
+        else if (cmd.equals("upload"))
+            copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
+        else if (cmd.equals("remove"))
+            remove(KylinConfig.getInstanceFromEnv(), args[1]);
+        else
+            System.out.println("Unknown cmd: " + cmd);
+    }
+
+    public static void list(KylinConfig config, String path) throws IOException {
+        ResourceStore store = ResourceStore.getStore(config);
+        ArrayList<String> result = store.listResources(path);
+        System.out.println("" + result);
+    }
+
+    private static void list(String metadataUri, String path) throws IOException {
+        KylinConfig config = KylinConfig.createInstanceFromUri(metadataUri);
+        list(config, path);
+    }
+
+    public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
+
+        ResourceStore src = ResourceStore.getStore(srcConfig);
+        ResourceStore dst = ResourceStore.getStore(dstConfig);
+        copyR(src, dst, "/");
+    }
+
+    private static void copy(String srcUri, String dstUri) throws IOException {
+
+        System.out.println("Copy from " + srcUri + " to " + dstUri);
+
+        KylinConfig srcConfig = KylinConfig.createInstanceFromUri(srcUri);
+        KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dstUri);
+        copy(srcConfig, dstConfig);
+
+    }
+
+    private static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
+        ArrayList<String> children = src.listResources(path);
+
+        // case of resource (not a folder)
+        if (children == null) {
+            if (matchExclude(path) == false) {
+                InputStream content = src.getResource(path);
+                long ts = src.getResourceTimestamp(path);
+                if (content != null)
+                    dst.putResource(path, content, ts);
+                else
+                    System.out.println("Null inputstream for " + path);
+            }
+        }
+        // case of folder
+        else {
+            for (String child : children)
+                copyR(src, dst, child);
+        }
+    }
+
+    private static boolean matchExclude(String path) {
+        if (excludes == null)
+            return false;
+        for (String exclude : excludes) {
+            if (path.startsWith(exclude))
+                return true;
+        }
+        return false;
+    }
+
+    public static void reset(KylinConfig config) throws IOException {
+        ResourceStore store = ResourceStore.getStore(config);
+        resetR(store, "/");
+    }
+
+    private static void resetR(ResourceStore store, String path) throws IOException {
+        ArrayList<String> children = store.listResources(path);
+        if (children == null) { // path is a resource (not a folder)
+            if (matchExclude(path) == false) {
+                store.deleteResource(path);
+            }
+        } else {
+            for (String child : children)
+                resetR(store, child);
+        }
+    }
+    
+    private static void remove(KylinConfig config, String path) throws IOException {
+        ResourceStore store = ResourceStore.getStore(config);
+        resetR(store, path);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
new file mode 100644
index 0000000..d19d583
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+import java.util.UUID;
+
+import org.apache.commons.lang.time.FastDateFormat;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Marks the root entity of JSON persistence. Unit of read, write, cache, and
+ * refresh.
+ * 
+ * - CubeInstance - CubeDesc - SourceTable - JobMeta - Dictionary (not JSON but
+ * also top level persistence)
+ * 
+ * @author yangli9
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+abstract public class RootPersistentEntity implements AclEntity {
+
+    static FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss z");
+
+    public static String formatTime(long millis) {
+        return format.format(millis);
+    }
+
+    // ============================================================================
+
+    @JsonProperty("uuid")
+    protected String uuid;
+
+    @JsonProperty("last_modified")
+    protected long lastModified;
+
+    public String getUuid() {
+        return uuid;
+    }
+
+    public void setUuid(String uuid) {
+        this.uuid = uuid;
+    }
+
+    public String getId() {
+        return uuid;
+    }
+
+    public long getLastModified() {
+        return lastModified;
+    }
+
+    public void setLastModified(long lastModified) {
+        this.lastModified = lastModified;
+    }
+
+    public void updateRandomUuid() {
+        setUuid(UUID.randomUUID().toString());
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + (int) (lastModified ^ (lastModified >>> 32));
+        result = prime * result + ((uuid == null) ? 0 : uuid.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        RootPersistentEntity other = (RootPersistentEntity) obj;
+        if (lastModified != other.lastModified)
+            return false;
+        if (uuid == null) {
+            if (other.uuid != null)
+                return false;
+        } else if (!uuid.equals(other.uuid))
+            return false;
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/Serializer.java b/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
new file mode 100644
index 0000000..d063745
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.persistence;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+/**
+ * @author yangli9
+ * 
+ */
+public interface Serializer<T extends RootPersistentEntity> {
+
+    public void serialize(T obj, DataOutputStream out) throws IOException;
+
+    public T deserialize(DataInputStream in) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/StorageException.java b/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
new file mode 100644
index 0000000..62f12cd
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.persistence;
+
+/**
+ * 
+ * @author xjiang
+ * 
+ */
+public class StorageException extends RuntimeException {
+
+    private static final long serialVersionUID = -3748712888242406257L;
+
+    public StorageException(String msg, Throwable t) {
+        super(msg, t);
+    }
+
+    public StorageException(String msg) {
+        super(msg);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
new file mode 100644
index 0000000..8bc2468
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+/**
+ * @author xjiang
+ * 
+ */
+public abstract class AbstractRestCache<K, V> {
+
+    protected final Broadcaster.TYPE syncType;
+
+    protected AbstractRestCache(Broadcaster.TYPE syncType) {
+        this.syncType = syncType;
+    }
+
+    protected final void syncRemote(Object key, Broadcaster.EVENT syncAction) {
+        String syncKey = (syncType == Broadcaster.TYPE.METADATA) ? "metadata" : key.toString();
+        Broadcaster.queue(syncType.getType(), syncAction.getType(), syncKey);
+    }
+
+    public abstract void put(K key, V value);
+
+    public abstract void putLocal(K key, V value);
+
+    public abstract void remove(K key);
+
+    public abstract void removeLocal(K key);
+
+    public abstract void clear();
+
+    public abstract int size();
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
new file mode 100644
index 0000000..60f0963
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * Broadcast kylin event out
+ * 
+ * @author jianliu
+ * 
+ */
+public class Broadcaster {
+
+    private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
+
+    private static List<BroadcastEvent> broadcaseEvents = new ArrayList<BroadcastEvent>();
+
+    static class BroadcasterHolder {
+        static final Broadcaster INSTANCE = new Broadcaster();
+    }
+
+    private Broadcaster() {
+        Timer timer = new Timer();
+        TimerTask task = new TimerTask() {
+            public void run() {
+                Broadcaster.flush();
+            }
+        };
+
+        timer.schedule(task, new Date(), 10 * 1000);
+    }
+
+    public Broadcaster getInstance() {
+        return BroadcasterHolder.INSTANCE;
+    }
+
+    public static void queueSyncMetadata() {
+        queue(TYPE.METADATA.getType(), EVENT.CREATE.getType(), "metadata");
+    }
+
+    /**
+     * Broadcast the cubedesc event out
+     * 
+     * @param action
+     *            event action
+     */
+    public static synchronized void queue(String type, String action, String key) {
+        BroadcastEvent event = BroadcasterHolder.INSTANCE.new BroadcastEvent(type, action, key);
+
+        if (!broadcaseEvents.contains(event)) {
+            broadcaseEvents.add(event);
+        }
+    }
+
+    public static synchronized void flush() {
+        String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
+        if (nodes == null)
+            return;
+
+        for (BroadcastEvent event : broadcaseEvents) {
+            for (String nodeUri : nodes) {
+                logger.debug("Broadcast nodeUri: " + nodeUri + ", type: " + event.getType() + ", action: " + event.getAction() + ", name: " + event.getName());
+                WipeCacheThread thread = BroadcasterHolder.INSTANCE.new WipeCacheThread(nodeUri, event.getType(), event.getAction(), event.getName());
+                thread.start();
+            }
+        }
+
+        broadcaseEvents.clear();
+    }
+
+    public static String genEventkey(String type, String action, String name) {
+        String time = String.valueOf(System.currentTimeMillis());
+        return time + "_" + type + "_" + action + "_" + name;
+    }
+
+    protected class WipeCacheThread extends Thread {
+        private String nodeUri;
+        private String type;
+        private String action;
+        private String name;
+
+        public WipeCacheThread(String nodeUri, String type, String action, String name) {
+            this.nodeUri = nodeUri;
+            this.type = type;
+            this.action = action;
+            this.name = name;
+        }
+
+        /*
+         * (non-Javadoc)
+         * 
+         * @see java.lang.Thread#run()
+         */
+        @Override
+        public void run() {
+            RestClient restClient = new RestClient(nodeUri);
+            try {
+                restClient.wipeCache(this.type, this.action, this.name);
+            } catch (IOException e) {
+                logger.warn("Thread failed during wipe cache at " + type + "." + action + "." + name + ", " + e.toString());
+            }
+        }
+    }
+
+    public enum EVENT {
+        CREATE("create"), UPDATE("update"), DROP("drop");
+        private String text;
+
+        private EVENT(String text) {
+            this.text = text;
+        }
+
+        public String getType() {
+            return text;
+        }
+
+        public static EVENT getEvent(String event) {
+            for (EVENT one : values()) {
+                if (one.getType().equalsIgnoreCase(event)) {
+                    return one;
+                }
+            }
+
+            return null;
+        }
+    }
+
+    public enum TYPE {
+        CUBE("cube"), METADATA("metadata"), PROJECT("project");
+        private String text;
+
+        private TYPE(String text) {
+            this.text = text;
+        }
+
+        public String getType() {
+            return text;
+        }
+
+        /**
+         * @param type
+         * @return
+         */
+        public static TYPE getType(String type) {
+            for (TYPE one : values()) {
+                if (one.getType().equalsIgnoreCase(type)) {
+                    return one;
+                }
+            }
+
+            return null;
+        }
+    }
+
+    public class BroadcastEvent {
+        private String type;
+        private String action;
+        private String name;
+
+        public BroadcastEvent(String type, String action, String name) {
+            super();
+            this.type = type;
+            this.action = action;
+            this.name = name;
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        public void setType(String type) {
+            this.type = type;
+        }
+
+        public String getAction() {
+            return action;
+        }
+
+        public void setAction(String action) {
+            this.action = action;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + getOuterType().hashCode();
+            result = prime * result + ((action == null) ? 0 : action.hashCode());
+            result = prime * result + ((name == null) ? 0 : name.hashCode());
+            result = prime * result + ((type == null) ? 0 : type.hashCode());
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            BroadcastEvent other = (BroadcastEvent) obj;
+            if (!getOuterType().equals(other.getOuterType()))
+                return false;
+            if (action == null) {
+                if (other.action != null)
+                    return false;
+            } else if (!action.equals(other.action))
+                return false;
+            if (name == null) {
+                if (other.name != null)
+                    return false;
+            } else if (!name.equals(other.name))
+                return false;
+            if (type == null) {
+                if (other.type != null)
+                    return false;
+            } else if (!type.equals(other.type))
+                return false;
+            return true;
+        }
+
+        private Broadcaster getOuterType() {
+            return Broadcaster.this;
+        }
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
new file mode 100644
index 0000000..3ecaac6
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.util.Set;
+
+import com.google.common.collect.HashMultimap;
+
+/**
+ * @author xjiang
+ * 
+ */
+public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
+
+    private final HashMultimap<K, V> innerCache;
+
+    public MultiValueCache(Broadcaster.TYPE syncType) {
+        super(syncType);
+        innerCache = HashMultimap.create();
+    }
+
+    public void put(K key, V value) {
+        Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
+        synchronized (this) {
+            innerCache.put(key, value);
+        }
+        syncRemote(key, eventType);
+    }
+
+    public void putLocal(K key, V value) {
+        synchronized (this) {
+            innerCache.put(key, value);
+        }
+    }
+
+    public void remove(K key) {
+        if (innerCache.containsKey(key)) {
+            innerCache.removeAll(key);
+            syncRemote(key, Broadcaster.EVENT.DROP);
+        }
+    }
+
+    public void removeLocal(K key) {
+        if (innerCache.containsKey(key)) {
+            innerCache.removeAll(key);
+        }
+    }
+
+    public void clear() {
+        innerCache.clear();
+    }
+
+    public int size() {
+        return innerCache.size();
+    }
+
+    public Set<V> get(K key) {
+        return innerCache.get(key);
+    }
+
+    public Set<K> keySet() {
+        return innerCache.keySet();
+    }
+
+    public boolean containsKey(Object key) {
+        return innerCache.containsKey(key);
+    }
+
+    public boolean containsEntry(Object key, Object value) {
+        return innerCache.containsEntry(key, value);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
new file mode 100644
index 0000000..5f1367f
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpException;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+/**
+ * @author yangli9
+ */
+public class RestClient {
+
+    String host;
+    int port;
+    String baseUrl;
+    String userName;
+    String password;
+    HttpClient client;
+
+    private static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
+
+    public static boolean matchFullRestPattern(String uri) {
+        Matcher m = fullRestPattern.matcher(uri);
+        return m.matches();
+    }
+
+    /**
+     * @param uri
+     *            "user:pwd@host:port"
+     */
+    public RestClient(String uri) {
+        Matcher m = fullRestPattern.matcher(uri);
+        if (!m.matches())
+            throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern);
+
+        String user = m.group(1);
+        String pwd = m.group(2);
+        String host = m.group(3);
+        String portStr = m.group(4);
+        int port = Integer.parseInt(portStr == null ? "7070" : portStr);
+
+        init(host, port, user, pwd);
+    }
+
+    private void init(String host, int port, String userName, String password) {
+        this.host = host;
+        this.port = port;
+        this.userName = userName;
+        this.password = password;
+        this.baseUrl = "http://" + host + ":" + port + "/kylin/api";
+
+        client = new HttpClient();
+
+        if (userName != null && password != null) {
+            client.getParams().setAuthenticationPreemptive(true);
+            Credentials creds = new UsernamePasswordCredentials(userName, password);
+            client.getState().setCredentials(new AuthScope(host, port, AuthScope.ANY_REALM), creds);
+        }
+    }
+
+    public void wipeCache(String type, String action, String name) throws IOException {
+        String url = baseUrl + "/cache/" + type + "/" + name + "/" + action;
+        HttpMethod get = new PutMethod(url);
+
+        try {
+            int code = client.executeMethod(get);
+            String msg = Bytes.toString(get.getResponseBody());
+
+            if (code != 200)
+                throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
+
+        } catch (HttpException ex) {
+            throw new IOException(ex);
+        } finally {
+            get.releaseConnection();
+        }
+    }
+
+    public String getKylinProperties() throws IOException {
+        String url = baseUrl + "/admin/config";
+        HttpMethod get = new GetMethod(url);
+        try {
+            int code = client.executeMethod(get);
+            String msg = Bytes.toString(get.getResponseBody());
+            JSONObject obj = new JSONObject(msg);
+            msg = obj.getString("config");
+
+            if (code != 200)
+                throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
+
+            return msg;
+
+        } catch (JSONException e) {
+            throw new IOException("Error when parsing json response from REST");
+        } finally {
+            get.releaseConnection();
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
new file mode 100644
index 0000000..394697c
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.restclient;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author xjiang
+ * 
+ */
+public class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
+
+    private final Map<K, V> innerCache;
+
+    public SingleValueCache(Broadcaster.TYPE syncType) {
+        super(syncType);
+        innerCache = new ConcurrentHashMap<K, V>();
+    }
+
+    public void put(K key, V value) {
+        Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
+        innerCache.put(key, value);
+        syncRemote(key, eventType);
+    }
+
+    public void putLocal(K key, V value) {
+        innerCache.put(key, value);
+    }
+
+    public void remove(K key) {
+        if (innerCache.containsKey(key)) {
+            innerCache.remove(key);
+            syncRemote(key, Broadcaster.EVENT.DROP);
+        }
+    }
+
+    public void removeLocal(K key) {
+        innerCache.remove(key);
+    }
+
+    public void clear() {
+        innerCache.clear();
+    }
+
+    public int size() {
+        return innerCache.size();
+    }
+
+    public V get(K key) {
+        return innerCache.get(key);
+    }
+
+    public Collection<V> values() {
+        return innerCache.values();
+    }
+
+    public boolean containsKey(String key) {
+        return innerCache.containsKey(key);
+    }
+
+    public Map<K, V> getMap() {
+        return Collections.unmodifiableMap(innerCache);
+    }
+
+    public Set<K> keySet() {
+        return innerCache.keySet();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java b/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
new file mode 100644
index 0000000..9b1005e
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * @author ysong1
+ * 
+ */
+public abstract class AbstractKylinTestCase {
+
+    public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
+
+    public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
+    
+    public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
+    
+    public abstract void createTestMetadata();
+
+    public abstract void cleanupTestMetadata();
+
+    public KylinConfig getTestConfig() {
+        return KylinConfig.getInstanceFromEnv();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/Array.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/Array.java b/common/src/main/java/com/kylinolap/common/util/Array.java
new file mode 100644
index 0000000..0b4165f
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/Array.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.util.Arrays;
+
+/*
+ * An array with correct equals(), hashCode(), compareTo() and toString()
+ */
+public class Array<T> implements Comparable<Array<T>> {
+    public T[] data;
+
+    public Array(T[] data) {
+        this.data = data;
+    }
+
+    public String toString() {
+        return Arrays.toString(data);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o != null && o instanceof Array) {
+            return Arrays.equals(this.data, ((Array<?>) o).data);
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(data);
+    }
+
+    @Override
+    public int compareTo(Array<T> other) {
+        return compare(this.data, other.data, null);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> int compare(T[] a, T[] b, boolean[] ascending) {
+        int r = 0;
+        int n = Math.min(a.length, b.length);
+        boolean asc = true;
+
+        for (int i = 0; i < n; i++) {
+            r = ((Comparable<T>) a[i]).compareTo(b[i]);
+            if (r != 0) {
+                asc = (ascending != null && ascending.length > i) ? ascending[i] : true;
+                return asc ? r : -r;
+            }
+        }
+        return a.length - b.length;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/ByteArray.java b/common/src/main/java/com/kylinolap/common/util/ByteArray.java
new file mode 100644
index 0000000..d6aee89
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/ByteArray.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * @author yangli9
+ */
+public class ByteArray implements Comparable<ByteArray> {
+
+    public byte[] data;
+
+    public ByteArray(byte[] data) {
+        this.data = data;
+    }
+
+    @Override
+    public int hashCode() {
+        return Bytes.hashCode(data);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        ByteArray other = (ByteArray) obj;
+        if (!Arrays.equals(data, other.data))
+            return false;
+        return true;
+    }
+
+    @Override
+    public int compareTo(ByteArray o) {
+        return Bytes.compareTo(this.data, o.data);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java b/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
new file mode 100644
index 0000000..5d46037
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.nio.ByteBuffer;
+
+/**
+ * @author yangli9
+ * 
+ */
+public interface BytesSerializer<T> {
+
+    public static final int SERIALIZE_BUFFER_SIZE = 65536;
+
+    abstract public void serialize(T value, ByteBuffer out);
+
+    abstract public T deserialize(ByteBuffer in);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesUtil.java b/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
new file mode 100644
index 0000000..93d72a4
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.common.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+
+public class BytesUtil {
+
+    public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+
+    public static void writeLong(long num, byte[] bytes, int offset, int size) {
+        for (int i = offset + size - 1; i >= offset; i--) {
+            bytes[i] = (byte) num;
+            num >>>= 8;
+        }
+    }
+
+    public static void writeUnsigned(int num, byte[] bytes, int offset, int size) {
+        for (int i = offset + size - 1; i >= offset; i--) {
+            bytes[i] = (byte) num;
+            num >>>= 8;
+        }
+    }
+
+    public static long readLong(byte[] bytes, int offset, int size) {
+        long integer = 0;
+        for (int i = offset, n = offset + size; i < n; i++) {
+            integer <<= 8;
+            integer |= (long) bytes[i] & 0xFF;
+        }
+        return integer;
+    }
+
+    public static int readUnsigned(byte[] bytes, int offset, int size) {
+        int integer = 0;
+        for (int i = offset, n = offset + size; i < n; i++) {
+            integer <<= 8;
+            integer |= (int) bytes[i] & 0xFF;
+        }
+        return integer;
+    }
+
+    public static void writeSigned(int num, byte[] bytes, int offset, int size) {
+        writeUnsigned(num, bytes, offset, size);
+    }
+
+    public static int readSigned(byte[] bytes, int offset, int size) {
+        int integer = (bytes[offset] & 0x80) == 0 ? 0 : -1;
+        for (int i = offset, n = offset + size; i < n; i++) {
+            integer <<= 8;
+            integer |= (int) bytes[i] & 0xFF;
+        }
+        return integer;
+    }
+
+    /**
+     * No. bytes needed to store a value as big as the given
+     */
+    public static int sizeForValue(int maxValue) {
+        int size = 0;
+        while (maxValue > 0) {
+            size++;
+            maxValue >>>= 8;
+        }
+        return size;
+    }
+
+    public static int compareByteUnsigned(byte b1, byte b2) {
+        int i1 = (int) b1 & 0xFF;
+        int i2 = (int) b2 & 0xFF;
+        return i1 - i2;
+    }
+
+    public static byte[] subarray(byte[] bytes, int start, int end) {
+        byte[] r = new byte[end - start];
+        System.arraycopy(bytes, start, r, 0, r.length);
+        return r;
+    }
+
+    public static int compareBytes(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) {
+        int r = 0;
+        for (int i = 0; i < length; i++) {
+            r = src[srcOffset + i] - dst[dstOffset + i];
+            if (r != 0)
+                break;
+        }
+        return r;
+    }
+
+    // from WritableUtils
+    // ============================================================================
+
+    public static void writeVInt(int i, ByteBuffer out) {
+        writeVLong(i, out);
+    }
+
+    public static void writeVLong(long i, ByteBuffer out) {
+        if (i >= -112 && i <= 127) {
+            out.put((byte) i);
+            return;
+        }
+
+        int len = -112;
+        if (i < 0) {
+            i ^= -1L; // take one's complement'
+            len = -120;
+        }
+
+        long tmp = i;
+        while (tmp != 0) {
+            tmp = tmp >> 8;
+            len--;
+        }
+
+        out.put((byte) len);
+
+        len = (len < -120) ? -(len + 120) : -(len + 112);
+
+        for (int idx = len; idx != 0; idx--) {
+            int shiftbits = (idx - 1) * 8;
+            long mask = 0xFFL << shiftbits;
+            out.put((byte) ((i & mask) >> shiftbits));
+        }
+    }
+
+    public static long readVLong(ByteBuffer in) {
+        byte firstByte = in.get();
+        int len = decodeVIntSize(firstByte);
+        if (len == 1) {
+            return firstByte;
+        }
+        long i = 0;
+        for (int idx = 0; idx < len - 1; idx++) {
+            byte b = in.get();
+            i = i << 8;
+            i = i | (b & 0xFF);
+        }
+        return (isNegativeVInt(firstByte) ? (i ^ -1L) : i);
+    }
+
+    public static int readVInt(ByteBuffer in) {
+        long n = readVLong(in);
+        if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) {
+            throw new IllegalArgumentException("value too long to fit in integer");
+        }
+        return (int) n;
+    }
+
+    private static boolean isNegativeVInt(byte value) {
+        return value < -120 || (value >= -112 && value < 0);
+    }
+
+    private static int decodeVIntSize(byte value) {
+        if (value >= -112) {
+            return 1;
+        } else if (value < -120) {
+            return -119 - value;
+        }
+        return -111 - value;
+    }
+
+    public static void writeUnsigned(int num, int size, ByteBuffer out) {
+        for (int i = 0; i < size; i++) {
+            out.put((byte) num);
+            num >>>= 8;
+        }
+    }
+
+    public static int readUnsigned(ByteBuffer in, int size) {
+        int integer = 0;
+        int mask = 0xff;
+        int shift = 0;
+        for (int i = 0; i < size; i++) {
+            integer |= (in.get() << shift) & mask;
+            mask = mask << 8;
+            shift += 8;
+        }
+        return integer;
+    }
+
+    public static void writeLong(long num, ByteBuffer out) {
+        for (int i = 0; i < 8; i++) {
+            out.put((byte) num);
+            num >>>= 8;
+        }
+    }
+
+    public static long readLong(ByteBuffer in) {
+        long integer = 0;
+        int mask = 0xff;
+        int shift = 0;
+        for (int i = 0; i < 8; i++) {
+            integer |= (in.get() << shift) & mask;
+            mask = mask << 8;
+            shift += 8;
+        }
+        return integer;
+    }
+
+    public static void writeUTFString(String str, ByteBuffer out) {
+        byte[] bytes = str == null ? null : Bytes.toBytes(str);
+        writeByteArray(bytes, out);
+    }
+
+    public static String readUTFString(ByteBuffer in) {
+        byte[] bytes = readByteArray(in);
+        return bytes == null ? null : Bytes.toString(bytes);
+    }
+
+    public static void writeAsciiString(String str, ByteBuffer out) {
+        if (str == null) {
+            BytesUtil.writeVInt(-1, out);
+            return;
+        }
+        int len = str.length();
+        BytesUtil.writeVInt(len, out);
+        for (int i = 0; i < len; i++) {
+            out.put((byte) str.charAt(i));
+        }
+    }
+
+    public static String readAsciiString(ByteBuffer in) {
+        int len = BytesUtil.readVInt(in);
+        if (len < 0) {
+            return null;
+        }
+        String result;
+        try {
+            if (in.hasArray()) {
+                int pos = in.position();
+                result = new String(in.array(), pos, len, "ISO-8859-1");
+                in.position(pos + len);
+            } else {
+                byte[] tmp = new byte[len];
+                in.get(tmp);
+                result = new String(tmp, "ISO-8859-1");
+            }
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e); // never happen
+        }
+        return result;
+    }
+
+    public static void writeAsciiStringArray(String[] strs, ByteBuffer out) {
+        writeVInt(strs.length, out);
+        for (int i = 0; i < strs.length; i++)
+            writeAsciiString(strs[i], out);
+    }
+
+    public static String[] readAsciiStringArray(ByteBuffer in) {
+        int len = readVInt(in);
+        String[] strs = new String[len];
+        for (int i = 0; i < len; i++)
+            strs[i] = readAsciiString(in);
+        return strs;
+    }
+
+    public static void writeIntArray(int[] array, ByteBuffer out) {
+        if (array == null) {
+            writeVInt(-1, out);
+            return;
+        }
+        writeVInt(array.length, out);
+        out.asIntBuffer().put(array);
+    }
+
+    public static int[] readIntArray(ByteBuffer in) {
+        int len = readVInt(in);
+        if (len < 0)
+            return null;
+        int[] array = new int[len];
+        in.asIntBuffer().get(array);
+        return array;
+    }
+
+    public static void writeByteArray(byte[] array, ByteBuffer out) {
+        if (array == null) {
+            writeVInt(-1, out);
+            return;
+        }
+        writeVInt(array.length, out);
+        out.put(array);
+    }
+
+    public static byte[] readByteArray(ByteBuffer in) {
+        int len = readVInt(in);
+        if (len < 0)
+            return null;
+
+        byte[] array = new byte[len];
+        in.get(array);
+        return array;
+    }
+
+    public static byte[] toBytes(Writable writable) {
+        try {
+            ByteArrayOutputStream bout = new ByteArrayOutputStream();
+            DataOutputStream out = new DataOutputStream(bout);
+            writable.write(out);
+            out.close();
+            bout.close();
+            return bout.toByteArray();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static String toHex(byte[] array) {
+        return toHex(new ImmutableBytesWritable(array));
+    }
+
+    public static String toHex(ImmutableBytesWritable bytes) {
+        byte[] array = bytes.get();
+        int offset = bytes.getOffset();
+        int length = bytes.getLength();
+        StringBuilder sb = new StringBuilder(length * 4);
+        for (int i = 0; i < length; i++) {
+            int b = array[offset + i];
+            sb.append(String.format("\\x%02X", b & 0xFF));
+        }
+        return sb.toString();
+    }
+
+    public static void main(String[] args) throws Exception {
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java b/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
new file mode 100644
index 0000000..6be3708
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+/**
+ * @author xduo
+ * 
+ */
+public class ClasspathUtil {
+
+    public static void addClasspath(String path) throws Exception {
+        System.out.println("Adding path " + path + " to class path");
+        File file = new File(path);
+
+        if (file.exists()) {
+            URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+            Class<URLClassLoader> urlClass = URLClassLoader.class;
+            Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class });
+            method.setAccessible(true);
+            method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() });
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java b/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
new file mode 100644
index 0000000..3f4ab7b
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * @author yangli9
+ */
+public class CliCommandExecutor {
+
+    private String remoteHost;
+    private String remoteUser;
+    private String remotePwd;
+    private int remoteTimeoutSeconds = 3600;
+
+    public CliCommandExecutor() {
+    }
+
+    public void setRunAtRemote(String host, String user, String pwd) {
+        this.remoteHost = host;
+        this.remoteUser = user;
+        this.remotePwd = pwd;
+    }
+
+    public void setRunAtLocal() {
+        this.remoteHost = null;
+        this.remoteUser = null;
+        this.remotePwd = null;
+    }
+    
+    public void copyFile(String localFile, String destDir) throws IOException {
+        if (remoteHost == null)
+            copyNative(localFile, destDir);
+        else
+            copyRemote(localFile, destDir);
+    }
+
+    private void copyNative(String localFile, String destDir) throws IOException {
+        File src = new File(localFile);
+        File dest = new File(destDir, src.getName());
+        FileUtils.copyFile(src, dest);
+    }
+
+    private void copyRemote(String localFile, String destDir) throws IOException {
+        SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
+        try {
+            ssh.scpFileToRemote(localFile, destDir);
+        } catch (IOException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new IOException(e.getMessage(), e);
+        }
+    }
+
+    public String execute(String command) throws IOException {
+        Pair<Integer, String> r;
+        if (remoteHost == null)
+            r = runNativeCommand(command);
+        else
+            r = runRemoteCommand(command);
+
+        if (r.getFirst() != 0)
+            throw new IOException("OS command error exit with " + r.getFirst() + " -- " + command + "\n" + r.getSecond());
+
+        return r.getSecond();
+    }
+
+    private Pair<Integer, String> runRemoteCommand(String command) throws IOException {
+        SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
+
+        SSHClientOutput sshOutput;
+        try {
+            sshOutput = ssh.execCommand(command, remoteTimeoutSeconds);
+            int exitCode = sshOutput.getExitCode();
+            String output = sshOutput.getText();
+            return new Pair<Integer, String>(exitCode, output);
+        } catch (IOException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new IOException(e.getMessage(), e);
+        }
+    }
+
+    private Pair<Integer, String> runNativeCommand(String command) throws IOException {
+        String[] cmd = new String[3];
+        String osName = System.getProperty("os.name");
+        if (osName.startsWith("Windows")) {
+            cmd[0] = "cmd.exe";
+            cmd[1] = "/C";
+        } else {
+            cmd[0] = "/bin/bash";
+            cmd[1] = "-c";
+        }
+        cmd[2] = command;
+
+        ProcessBuilder builder = new ProcessBuilder(cmd);
+        builder.redirectErrorStream(true);
+        Process proc = builder.start();
+
+        ByteArrayOutputStream buf = new ByteArrayOutputStream();
+        IOUtils.copy(proc.getInputStream(), buf);
+        String output = buf.toString("UTF-8");
+
+        try {
+            int exitCode = proc.waitFor();
+            return new Pair<Integer, String>(exitCode, output);
+        } catch (InterruptedException e) {
+            throw new IOException(e);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
new file mode 100644
index 0000000..a1dda9a
--- /dev/null
+++ b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.common.util;
+
+import com.kylinolap.common.KylinConfig;
+
+/**
+ * @author ysong1
+ */
+public class HBaseMetadataTestCase extends AbstractKylinTestCase {
+    
+    public static void staticCreateTestMetadata() {
+
+        KylinConfig.destoryInstance();
+
+        if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
+            System.setProperty(KylinConfig.KYLIN_CONF, SANDBOX_TEST_DATA);
+
+    }
+    
+    public static void staticCleanupTestMetadata() {
+        System.clearProperty(KylinConfig.KYLIN_CONF);
+        KylinConfig.destoryInstance();
+    }
+
+    @Override
+    public void createTestMetadata() {
+        staticCreateTestMetadata();
+    }
+
+    @Override
+    public void cleanupTestMetadata() {
+        staticCleanupTestMetadata();
+    }
+
+}