You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/14 15:16:20 UTC
[45/51] [partial] incubator-kylin git commit: cleanup for migration
from github.com
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
deleted file mode 100644
index 3e101af..0000000
--- a/common/src/main/java/com/kylinolap/common/persistence/HBaseResourceStore.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.common.util.HadoopUtil;
-
-public class HBaseResourceStore extends ResourceStore {
-
- private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
- private static final String FAMILY = "f";
- private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
- private static final String COLUMN = "c";
- private static final byte[] B_COLUMN = Bytes.toBytes(COLUMN);
- private static final String COLUMN_TS = "t";
- private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS);
-
- private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>();
-
- static {
- TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube");
- TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict");
- TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex");
- TABLE_SUFFIX_MAP.put(JOB_PATH_ROOT + "/", "_job");
- TABLE_SUFFIX_MAP.put(JOB_OUTPUT_PATH_ROOT + "/", "_job_output");
- TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj");
- TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot");
- TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE
- }
-
- final String tableNameBase;
- final String hbaseUrl;
-
- final Map<String, String> tableNameMap; // path prefix ==> HBase table name
-
- private HConnection getConnection() throws IOException {
- return HBaseConnection.get(hbaseUrl);
- }
-
- public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
- super(kylinConfig);
-
- String metadataUrl = kylinConfig.getMetadataUrl();
- // split TABLE@HBASE_URL
- int cut = metadataUrl.indexOf('@');
- tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
- hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
- tableNameMap = new LinkedHashMap<String, String>();
- for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) {
- String pathPrefix = entry.getKey();
- String tableName = tableNameBase + entry.getValue();
- tableNameMap.put(pathPrefix, tableName);
- createHTableIfNeeded(tableName);
- }
-
- }
-
- private void createHTableIfNeeded(String tableName) throws IOException {
- HBaseConnection.createHTableIfNeeded(getConnection(), tableName, FAMILY);
- }
-
- private String getTableName(String path) {
- for (Entry<String, String> entry : tableNameMap.entrySet()) {
- String pathPrefix = entry.getKey();
- if (path.startsWith(pathPrefix))
- return entry.getValue();
- }
- throw new IllegalStateException("failed to find HBase table for path -- " + path);
- }
-
- @Override
- protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
- assert resPath.startsWith("/");
- String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
- byte[] startRow = Bytes.toBytes(lookForPrefix);
- byte[] endRow = Bytes.toBytes(lookForPrefix);
- endRow[endRow.length - 1]++;
-
- ArrayList<String> result = new ArrayList<String>();
-
- for (Entry<String, String> entry : tableNameMap.entrySet()) {
- String pathPrefix = entry.getKey();
- String tableName = entry.getValue();
-
- if ((pathPrefix.startsWith(lookForPrefix) || lookForPrefix.startsWith(pathPrefix)) == false)
- continue;
-
- HTableInterface table = getConnection().getTable(tableName);
-
- Scan scan = new Scan(startRow, endRow);
- scan.setFilter(new KeyOnlyFilter());
- try {
- ResultScanner scanner = table.getScanner(scan);
- for (Result r : scanner) {
- String path = Bytes.toString(r.getRow());
- assert path.startsWith(lookForPrefix);
- int cut = path.indexOf('/', lookForPrefix.length());
- String child = cut < 0 ? path : path.substring(0, cut);
- if (result.contains(child) == false)
- result.add(child);
- }
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
- // return null to indicate not a folder
- return result.isEmpty() ? null : result;
- }
-
- @Override
- protected boolean existsImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, null, null);
- return r != null;
- }
-
- @Override
- protected InputStream getResourceImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, B_FAMILY, B_COLUMN);
- if (r == null)
- return null;
-
- byte[] value = r.getValue(B_FAMILY, B_COLUMN);
- if (value.length == 0) {
- Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HadoopUtil.getDefaultConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
-
- return fileSystem.open(redirectPath);
- } else {
- return new ByteArrayInputStream(value);
- }
- }
-
- @Override
- protected long getResourceTimestampImpl(String resPath) throws IOException {
- Result r = getByScan(resPath, B_FAMILY, B_COLUMN_TS);
- if (r == null)
- return 0;
- else
- return Bytes.toLong(r.getValue(B_FAMILY, B_COLUMN_TS));
- }
-
- @Override
- protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(content, bout);
- bout.close();
-
- HTableInterface table = getConnection().getTable(getTableName(resPath));
- try {
- byte[] row = Bytes.toBytes(resPath);
- Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
-
- table.put(put);
- table.flushCommits();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getTableName(resPath));
- try {
- byte[] row = Bytes.toBytes(resPath);
- byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
- Put put = buildPut(resPath, newTS, row, content, table);
-
- boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
- if (!ok)
- throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + getResourceTimestamp(resPath));
-
- table.flushCommits();
-
- return newTS;
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getTableName(resPath));
- try {
- Delete del = new Delete(Bytes.toBytes(resPath));
- table.delete(del);
- table.flushCommits();
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- protected String getReadableResourcePathImpl(String resPath) {
- return tableNameBase + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl();
- }
-
- private Result getByScan(String path, byte[] family, byte[] column) throws IOException {
- byte[] startRow = Bytes.toBytes(path);
- byte[] endRow = plusZero(startRow);
-
- Scan scan = new Scan(startRow, endRow);
- if (family == null || column == null) {
- scan.setFilter(new KeyOnlyFilter());
- } else {
- scan.addColumn(family, column);
- }
-
- HTableInterface table = getConnection().getTable(getTableName(path));
- try {
- ResultScanner scanner = table.getScanner(scan);
- Result result = null;
- for (Result r : scanner) {
- result = r;
- }
- return result == null || result.isEmpty() ? null : result;
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- private byte[] plusZero(byte[] startRow) {
- byte[] endRow = Arrays.copyOf(startRow, startRow.length + 1);
- endRow[endRow.length - 1] = 0;
- return endRow;
- }
-
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
- Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HadoopUtil.getDefaultConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
-
- if (fileSystem.exists(redirectPath)) {
- fileSystem.delete(redirectPath, true);
- }
-
- FSDataOutputStream out = fileSystem.create(redirectPath);
-
- try {
- out.write(largeColumn);
- } finally {
- IOUtils.closeQuietly(out);
- }
-
- return redirectPath;
- }
-
- public Path bigCellHDFSPath(String resPath) {
- String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory();
- Path redirectPath = new Path(hdfsWorkingDirectory, "resources" + resPath);
- return redirectPath;
- }
-
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
- int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
- if (content.length > kvSizeLimit) {
- writeLargeCellToHdfs(resPath, content, table);
- content = BytesUtil.EMPTY_BYTE_ARRAY;
- }
-
- Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
-
- return put;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java b/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
deleted file mode 100644
index 1471d09..0000000
--- a/common/src/main/java/com/kylinolap/common/persistence/JsonSerializer.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.persistence;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import com.kylinolap.common.util.JsonUtil;
-
-/**
- * @author yangli9
- */
-public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> {
-
- Class<T> clz;
-
- public JsonSerializer(Class<T> clz) {
- this.clz = clz;
- }
-
- @Override
- public T deserialize(DataInputStream in) throws IOException {
- return JsonUtil.readValue(in, clz);
- }
-
- @Override
- public void serialize(T obj, DataOutputStream out) throws IOException {
- JsonUtil.writeValueIndent(out, obj);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
deleted file mode 100644
index 36abfdd..0000000
--- a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.kylinolap.common.KylinConfig;
-
-abstract public class ResourceStore {
-
- private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class);
-
- public static final String CUBE_RESOURCE_ROOT = "/cube";
- public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc";
- public static final String DICT_RESOURCE_ROOT = "/dict";
- public static final String IIDESC_RESOURCE_ROOT = "/invertedindex_desc";
- public static final String JOB_PATH_ROOT = "/job";
- public static final String JOB_OUTPUT_PATH_ROOT = "/job_output";
- public static final String PROJECT_RESOURCE_ROOT = "/project";
- public static final String SNAPSHOT_RESOURCE_ROOT = "/table_snapshot";
- public static final String TABLE_EXD_RESOURCE_ROOT = "/table_exd";
- public static final String TABLE_RESOURCE_ROOT = "/table";
-
- private static ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
-
- public static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
-
- static {
- knownImpl.add(HBaseResourceStore.class);
- knownImpl.add(FileResourceStore.class);
- }
-
- public static ResourceStore getStore(KylinConfig kylinConfig) {
- ResourceStore r = CACHE.get(kylinConfig);
- List<Throwable> es = new ArrayList<Throwable>();
- if (r == null) {
- logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
- for (Class<? extends ResourceStore> cls : knownImpl) {
-
- try {
- r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
- } catch (Exception e) {
- es.add(e);
- } catch (NoClassDefFoundError er) {
- // may throw NoClassDefFoundError
- es.add(er);
- }
- if (r != null) {
- break;
- }
- }
- if (r == null) {
- for (Throwable exceptionOrError : es) {
- logger.error("Create new store instance failed ", exceptionOrError);
- }
- throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
- }
-
- CACHE.put(kylinConfig, r);
- }
- return r;
- }
-
- // ============================================================================
-
- KylinConfig kylinConfig;
-
- ResourceStore(KylinConfig kylinConfig) {
- this.kylinConfig = kylinConfig;
- }
-
- /**
- * return a list of child resources & folders under given path, return null
- * if given path is not a folder
- */
- final public ArrayList<String> listResources(String resPath) throws IOException {
- resPath = norm(resPath);
- return listResourcesImpl(resPath);
- }
-
- abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException;
-
- /**
- * return true if a resource exists, return false in case of folder or
- * non-exist
- */
- final public boolean exists(String resPath) throws IOException {
- return existsImpl(norm(resPath));
- }
-
- abstract protected boolean existsImpl(String resPath) throws IOException;
-
- /**
- * read a resource, return null in case of not found
- */
- final public <T extends RootPersistentEntity> T getResource(String resPath, Class<T> clz, Serializer<T> serializer) throws IOException {
- resPath = norm(resPath);
- InputStream in = getResourceImpl(resPath);
- if (in == null)
- return null;
-
- DataInputStream din = new DataInputStream(in);
- try {
- T r = serializer.deserialize(din);
- r.setLastModified(getResourceTimestamp(resPath));
- return r;
- } finally {
- IOUtils.closeQuietly(din);
- IOUtils.closeQuietly(in);
- }
- }
-
- final public InputStream getResource(String resPath) throws IOException {
- return getResourceImpl(norm(resPath));
- }
-
- abstract protected InputStream getResourceImpl(String resPath) throws IOException;
-
- final public long getResourceTimestamp(String resPath) throws IOException {
- return getResourceTimestampImpl(norm(resPath));
- }
-
- abstract protected long getResourceTimestampImpl(String resPath) throws IOException;
-
- /**
- * overwrite a resource without write conflict check
- */
- final public void putResource(String resPath, InputStream content, long ts) throws IOException {
- resPath = norm(resPath);
- logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
- putResourceImpl(resPath, content, ts);
- }
-
- abstract protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException;
-
- /**
- * check & set, overwrite a resource
- */
- final public <T extends RootPersistentEntity> void putResource(String resPath, T obj, Serializer<T> serializer) throws IOException {
- resPath = norm(resPath);
- logger.debug("Saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
-
- long oldTS = obj.getLastModified();
- long newTS = System.currentTimeMillis();
- obj.setLastModified(newTS);
-
- try {
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- DataOutputStream dout = new DataOutputStream(buf);
- serializer.serialize(obj, dout);
- dout.close();
- buf.close();
-
- newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS, newTS);
- obj.setLastModified(newTS); // update again the confirmed TS
-
- } catch (IOException e) {
- obj.setLastModified(oldTS); // roll back TS when write fail
- throw e;
- } catch (RuntimeException e) {
- obj.setLastModified(oldTS); // roll back TS when write fail
- throw e;
- }
- }
-
- /**
- * checks old timestamp when overwriting existing
- */
- abstract protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException;
-
- /**
- * delete a resource, does nothing on a folder
- */
- final public void deleteResource(String resPath) throws IOException {
- logger.debug("Deleting resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
- deleteResourceImpl(norm(resPath));
- }
-
- abstract protected void deleteResourceImpl(String resPath) throws IOException;
-
- /**
- * get a readable string of a resource path
- */
- final public String getReadableResourcePath(String resPath) {
- return getReadableResourcePathImpl(norm(resPath));
- }
-
- abstract protected String getReadableResourcePathImpl(String resPath);
-
- private String norm(String resPath) {
- resPath = resPath.trim();
- while (resPath.startsWith("//"))
- resPath = resPath.substring(1);
- while (resPath.endsWith("/"))
- resPath = resPath.substring(0, resPath.length() - 1);
- if (resPath.startsWith("/") == false)
- resPath = "/" + resPath;
- return resPath;
- }
-
- // ============================================================================
-
- public static interface Visitor {
- void visit(String path) throws IOException;
- }
-
- public void scanRecursively(String path, Visitor visitor) throws IOException {
- ArrayList<String> children = listResources(path);
- if (children != null) {
- for (String child : children)
- scanRecursively(child, visitor);
- return;
- }
-
- if (exists(path))
- visitor.visit(path);
- }
-
- public List<String> collectResourceRecursively(String root, final String suffix) throws IOException {
- final ArrayList<String> collector = Lists.newArrayList();
- scanRecursively(root, new Visitor() {
- @Override
- public void visit(String path) {
- if (path.endsWith(suffix))
- collector.add(path);
- }
- });
- return collector;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
deleted file mode 100644
index 66783ca..0000000
--- a/common/src/main/java/com/kylinolap/common/persistence/ResourceTool.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.util.StringUtil;
-
-public class ResourceTool {
-
- private static String[] excludes = null;
-
- public static void main(String[] args) throws IOException {
- args = StringUtil.filterSystemArgs(args);
-
- if (args.length == 0) {
- System.out.println("Usage: MetadataTool reset METADATA_URI");
- System.out.println("Usage: MetadataTool copy METADATA_URI_SRC METADATA_URI_DST");
- System.out.println("Usage: MetadataTool list METADATA_URI PATH");
- return;
- }
-
- String exclude = System.getProperty("exclude");
- if (exclude != null) {
- excludes = exclude.split("\\s*,\\s*");
- }
-
- String cmd = args[0];
- if (cmd.equals("reset"))
- reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1]));
- else if (cmd.equals("copy"))
- copy(args[1], args[2]);
- else if (cmd.equals("list"))
- list(args[1], args[2]);
- else if (cmd.equals("download"))
- copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
- else if (cmd.equals("upload"))
- copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
- else if (cmd.equals("remove"))
- remove(KylinConfig.getInstanceFromEnv(), args[1]);
- else
- System.out.println("Unknown cmd: " + cmd);
- }
-
- public static void list(KylinConfig config, String path) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- ArrayList<String> result = store.listResources(path);
- System.out.println("" + result);
- }
-
- private static void list(String metadataUri, String path) throws IOException {
- KylinConfig config = KylinConfig.createInstanceFromUri(metadataUri);
- list(config, path);
- }
-
- public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
-
- ResourceStore src = ResourceStore.getStore(srcConfig);
- ResourceStore dst = ResourceStore.getStore(dstConfig);
- copyR(src, dst, "/");
- }
-
- private static void copy(String srcUri, String dstUri) throws IOException {
-
- System.out.println("Copy from " + srcUri + " to " + dstUri);
-
- KylinConfig srcConfig = KylinConfig.createInstanceFromUri(srcUri);
- KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dstUri);
- copy(srcConfig, dstConfig);
-
- }
-
- private static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
- ArrayList<String> children = src.listResources(path);
-
- // case of resource (not a folder)
- if (children == null) {
- if (matchExclude(path) == false) {
- InputStream content = src.getResource(path);
- long ts = src.getResourceTimestamp(path);
- if (content != null)
- dst.putResource(path, content, ts);
- else
- System.out.println("Null inputstream for " + path);
- }
- }
- // case of folder
- else {
- for (String child : children)
- copyR(src, dst, child);
- }
- }
-
- private static boolean matchExclude(String path) {
- if (excludes == null)
- return false;
- for (String exclude : excludes) {
- if (path.startsWith(exclude))
- return true;
- }
- return false;
- }
-
- public static void reset(KylinConfig config) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- resetR(store, "/");
- }
-
- private static void resetR(ResourceStore store, String path) throws IOException {
- ArrayList<String> children = store.listResources(path);
- if (children == null) { // path is a resource (not a folder)
- if (matchExclude(path) == false) {
- store.deleteResource(path);
- }
- } else {
- for (String child : children)
- resetR(store, child);
- }
- }
-
- private static void remove(KylinConfig config, String path) throws IOException {
- ResourceStore store = ResourceStore.getStore(config);
- resetR(store, path);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
deleted file mode 100644
index d19d583..0000000
--- a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-import java.util.UUID;
-
-import org.apache.commons.lang.time.FastDateFormat;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Marks the root entity of JSON persistence. Unit of read, write, cache, and
- * refresh.
- *
- * - CubeInstance - CubeDesc - SourceTable - JobMeta - Dictionary (not JSON but
- * also top level persistence)
- *
- * @author yangli9
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-abstract public class RootPersistentEntity implements AclEntity {
-
- static FastDateFormat format = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss z");
-
- public static String formatTime(long millis) {
- return format.format(millis);
- }
-
- // ============================================================================
-
- @JsonProperty("uuid")
- protected String uuid;
-
- @JsonProperty("last_modified")
- protected long lastModified;
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String uuid) {
- this.uuid = uuid;
- }
-
- public String getId() {
- return uuid;
- }
-
- public long getLastModified() {
- return lastModified;
- }
-
- public void setLastModified(long lastModified) {
- this.lastModified = lastModified;
- }
-
- public void updateRandomUuid() {
- setUuid(UUID.randomUUID().toString());
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (int) (lastModified ^ (lastModified >>> 32));
- result = prime * result + ((uuid == null) ? 0 : uuid.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- RootPersistentEntity other = (RootPersistentEntity) obj;
- if (lastModified != other.lastModified)
- return false;
- if (uuid == null) {
- if (other.uuid != null)
- return false;
- } else if (!uuid.equals(other.uuid))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/Serializer.java b/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
deleted file mode 100644
index d063745..0000000
--- a/common/src/main/java/com/kylinolap/common/persistence/Serializer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.persistence;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/**
- * @author yangli9
- *
- */
-public interface Serializer<T extends RootPersistentEntity> {
-
- public void serialize(T obj, DataOutputStream out) throws IOException;
-
- public T deserialize(DataInputStream in) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/StorageException.java b/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
deleted file mode 100644
index 62f12cd..0000000
--- a/common/src/main/java/com/kylinolap/common/persistence/StorageException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.persistence;
-
-/**
- *
- * @author xjiang
- *
- */
-public class StorageException extends RuntimeException {
-
- private static final long serialVersionUID = -3748712888242406257L;
-
- public StorageException(String msg, Throwable t) {
- super(msg, t);
- }
-
- public StorageException(String msg) {
- super(msg);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java b/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
deleted file mode 100644
index 8bc2468..0000000
--- a/common/src/main/java/com/kylinolap/common/restclient/AbstractRestCache.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-/**
- * @author xjiang
- *
- */
-public abstract class AbstractRestCache<K, V> {
-
- protected final Broadcaster.TYPE syncType;
-
- protected AbstractRestCache(Broadcaster.TYPE syncType) {
- this.syncType = syncType;
- }
-
- protected final void syncRemote(Object key, Broadcaster.EVENT syncAction) {
- String syncKey = (syncType == Broadcaster.TYPE.METADATA) ? "metadata" : key.toString();
- Broadcaster.queue(syncType.getType(), syncAction.getType(), syncKey);
- }
-
- public abstract void put(K key, V value);
-
- public abstract void putLocal(K key, V value);
-
- public abstract void remove(K key);
-
- public abstract void removeLocal(K key);
-
- public abstract void clear();
-
- public abstract int size();
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java b/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
deleted file mode 100644
index 60f0963..0000000
--- a/common/src/main/java/com/kylinolap/common/restclient/Broadcaster.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.kylinolap.common.KylinConfig;
-
-/**
- * Broadcast kylin event out
- *
- * @author jianliu
- *
- */
-public class Broadcaster {
-
- private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
-
- private static List<BroadcastEvent> broadcaseEvents = new ArrayList<BroadcastEvent>();
-
- static class BroadcasterHolder {
- static final Broadcaster INSTANCE = new Broadcaster();
- }
-
- private Broadcaster() {
- Timer timer = new Timer();
- TimerTask task = new TimerTask() {
- public void run() {
- Broadcaster.flush();
- }
- };
-
- timer.schedule(task, new Date(), 10 * 1000);
- }
-
- public Broadcaster getInstance() {
- return BroadcasterHolder.INSTANCE;
- }
-
- public static void queueSyncMetadata() {
- queue(TYPE.METADATA.getType(), EVENT.CREATE.getType(), "metadata");
- }
-
- /**
- * Broadcast the cubedesc event out
- *
- * @param action
- * event action
- */
- public static synchronized void queue(String type, String action, String key) {
- BroadcastEvent event = BroadcasterHolder.INSTANCE.new BroadcastEvent(type, action, key);
-
- if (!broadcaseEvents.contains(event)) {
- broadcaseEvents.add(event);
- }
- }
-
- public static synchronized void flush() {
- String[] nodes = KylinConfig.getInstanceFromEnv().getRestServers();
- if (nodes == null)
- return;
-
- for (BroadcastEvent event : broadcaseEvents) {
- for (String nodeUri : nodes) {
- logger.debug("Broadcast nodeUri: " + nodeUri + ", type: " + event.getType() + ", action: " + event.getAction() + ", name: " + event.getName());
- WipeCacheThread thread = BroadcasterHolder.INSTANCE.new WipeCacheThread(nodeUri, event.getType(), event.getAction(), event.getName());
- thread.start();
- }
- }
-
- broadcaseEvents.clear();
- }
-
- public static String genEventkey(String type, String action, String name) {
- String time = String.valueOf(System.currentTimeMillis());
- return time + "_" + type + "_" + action + "_" + name;
- }
-
- protected class WipeCacheThread extends Thread {
- private String nodeUri;
- private String type;
- private String action;
- private String name;
-
- public WipeCacheThread(String nodeUri, String type, String action, String name) {
- this.nodeUri = nodeUri;
- this.type = type;
- this.action = action;
- this.name = name;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Thread#run()
- */
- @Override
- public void run() {
- RestClient restClient = new RestClient(nodeUri);
- try {
- restClient.wipeCache(this.type, this.action, this.name);
- } catch (IOException e) {
- logger.warn("Thread failed during wipe cache at " + type + "." + action + "." + name + ", " + e.toString());
- }
- }
- }
-
- public enum EVENT {
- CREATE("create"), UPDATE("update"), DROP("drop");
- private String text;
-
- private EVENT(String text) {
- this.text = text;
- }
-
- public String getType() {
- return text;
- }
-
- public static EVENT getEvent(String event) {
- for (EVENT one : values()) {
- if (one.getType().equalsIgnoreCase(event)) {
- return one;
- }
- }
-
- return null;
- }
- }
-
- public enum TYPE {
- CUBE("cube"), METADATA("metadata"), PROJECT("project");
- private String text;
-
- private TYPE(String text) {
- this.text = text;
- }
-
- public String getType() {
- return text;
- }
-
- /**
- * @param type
- * @return
- */
- public static TYPE getType(String type) {
- for (TYPE one : values()) {
- if (one.getType().equalsIgnoreCase(type)) {
- return one;
- }
- }
-
- return null;
- }
- }
-
- public class BroadcastEvent {
- private String type;
- private String action;
- private String name;
-
- public BroadcastEvent(String type, String action, String name) {
- super();
- this.type = type;
- this.action = action;
- this.name = name;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getAction() {
- return action;
- }
-
- public void setAction(String action) {
- this.action = action;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + getOuterType().hashCode();
- result = prime * result + ((action == null) ? 0 : action.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((type == null) ? 0 : type.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- BroadcastEvent other = (BroadcastEvent) obj;
- if (!getOuterType().equals(other.getOuterType()))
- return false;
- if (action == null) {
- if (other.action != null)
- return false;
- } else if (!action.equals(other.action))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (type == null) {
- if (other.type != null)
- return false;
- } else if (!type.equals(other.type))
- return false;
- return true;
- }
-
- private Broadcaster getOuterType() {
- return Broadcaster.this;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
deleted file mode 100644
index 3ecaac6..0000000
--- a/common/src/main/java/com/kylinolap/common/restclient/MultiValueCache.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.util.Set;
-
-import com.google.common.collect.HashMultimap;
-
-/**
- * @author xjiang
- *
- */
-public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final HashMultimap<K, V> innerCache;
-
- public MultiValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = HashMultimap.create();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- synchronized (this) {
- innerCache.put(key, value);
- }
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- synchronized (this) {
- innerCache.put(key, value);
- }
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- }
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public Set<V> get(K key) {
- return innerCache.get(key);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-
- public boolean containsKey(Object key) {
- return innerCache.containsKey(key);
- }
-
- public boolean containsEntry(Object key, Object value) {
- return innerCache.containsEntry(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
deleted file mode 100644
index 5f1367f..0000000
--- a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.httpclient.Credentials;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.UsernamePasswordCredentials;
-import org.apache.commons.httpclient.auth.AuthScope;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PutMethod;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-
-/**
- * @author yangli9
- */
-public class RestClient {
-
- String host;
- int port;
- String baseUrl;
- String userName;
- String password;
- HttpClient client;
-
- private static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
-
- public static boolean matchFullRestPattern(String uri) {
- Matcher m = fullRestPattern.matcher(uri);
- return m.matches();
- }
-
- /**
- * @param uri
- * "user:pwd@host:port"
- */
- public RestClient(String uri) {
- Matcher m = fullRestPattern.matcher(uri);
- if (!m.matches())
- throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern);
-
- String user = m.group(1);
- String pwd = m.group(2);
- String host = m.group(3);
- String portStr = m.group(4);
- int port = Integer.parseInt(portStr == null ? "7070" : portStr);
-
- init(host, port, user, pwd);
- }
-
- private void init(String host, int port, String userName, String password) {
- this.host = host;
- this.port = port;
- this.userName = userName;
- this.password = password;
- this.baseUrl = "http://" + host + ":" + port + "/kylin/api";
-
- client = new HttpClient();
-
- if (userName != null && password != null) {
- client.getParams().setAuthenticationPreemptive(true);
- Credentials creds = new UsernamePasswordCredentials(userName, password);
- client.getState().setCredentials(new AuthScope(host, port, AuthScope.ANY_REALM), creds);
- }
- }
-
- public void wipeCache(String type, String action, String name) throws IOException {
- String url = baseUrl + "/cache/" + type + "/" + name + "/" + action;
- HttpMethod get = new PutMethod(url);
-
- try {
- int code = client.executeMethod(get);
- String msg = Bytes.toString(get.getResponseBody());
-
- if (code != 200)
- throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
-
- } catch (HttpException ex) {
- throw new IOException(ex);
- } finally {
- get.releaseConnection();
- }
- }
-
- public String getKylinProperties() throws IOException {
- String url = baseUrl + "/admin/config";
- HttpMethod get = new GetMethod(url);
- try {
- int code = client.executeMethod(get);
- String msg = Bytes.toString(get.getResponseBody());
- JSONObject obj = new JSONObject(msg);
- msg = obj.getString("config");
-
- if (code != 200)
- throw new IOException("Invalid response " + code + " with cache wipe url " + url + "\n" + msg);
-
- return msg;
-
- } catch (JSONException e) {
- throw new IOException("Error when parsing json response from REST");
- } finally {
- get.releaseConnection();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java b/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
deleted file mode 100644
index 394697c..0000000
--- a/common/src/main/java/com/kylinolap/common/restclient/SingleValueCache.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.restclient;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * @author xjiang
- *
- */
-public class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final Map<K, V> innerCache;
-
- public SingleValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = new ConcurrentHashMap<K, V>();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- innerCache.put(key, value);
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- innerCache.put(key, value);
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.remove(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- innerCache.remove(key);
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public V get(K key) {
- return innerCache.get(key);
- }
-
- public Collection<V> values() {
- return innerCache.values();
- }
-
- public boolean containsKey(String key) {
- return innerCache.containsKey(key);
- }
-
- public Map<K, V> getMap() {
- return Collections.unmodifiableMap(innerCache);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java b/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
deleted file mode 100644
index 9b1005e..0000000
--- a/common/src/main/java/com/kylinolap/common/util/AbstractKylinTestCase.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import com.kylinolap.common.KylinConfig;
-
-/**
- * @author ysong1
- *
- */
-public abstract class AbstractKylinTestCase {
-
- public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
-
- public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
-
- public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
-
- public abstract void createTestMetadata();
-
- public abstract void cleanupTestMetadata();
-
- public KylinConfig getTestConfig() {
- return KylinConfig.getInstanceFromEnv();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/Array.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/Array.java b/common/src/main/java/com/kylinolap/common/util/Array.java
deleted file mode 100644
index 0b4165f..0000000
--- a/common/src/main/java/com/kylinolap/common/util/Array.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.util.Arrays;
-
-/*
- * An array with correct equals(), hashCode(), compareTo() and toString()
- */
-public class Array<T> implements Comparable<Array<T>> {
- public T[] data;
-
- public Array(T[] data) {
- this.data = data;
- }
-
- public String toString() {
- return Arrays.toString(data);
- }
-
- @Override
- public boolean equals(Object o) {
- if (o != null && o instanceof Array) {
- return Arrays.equals(this.data, ((Array<?>) o).data);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(data);
- }
-
- @Override
- public int compareTo(Array<T> other) {
- return compare(this.data, other.data, null);
- }
-
- @SuppressWarnings("unchecked")
- public static <T> int compare(T[] a, T[] b, boolean[] ascending) {
- int r = 0;
- int n = Math.min(a.length, b.length);
- boolean asc = true;
-
- for (int i = 0; i < n; i++) {
- r = ((Comparable<T>) a[i]).compareTo(b[i]);
- if (r != 0) {
- asc = (ascending != null && ascending.length > i) ? ascending[i] : true;
- return asc ? r : -r;
- }
- }
- return a.length - b.length;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/ByteArray.java b/common/src/main/java/com/kylinolap/common/util/ByteArray.java
deleted file mode 100644
index d6aee89..0000000
--- a/common/src/main/java/com/kylinolap/common/util/ByteArray.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * @author yangli9
- */
-public class ByteArray implements Comparable<ByteArray> {
-
- public byte[] data;
-
- public ByteArray(byte[] data) {
- this.data = data;
- }
-
- @Override
- public int hashCode() {
- return Bytes.hashCode(data);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- ByteArray other = (ByteArray) obj;
- if (!Arrays.equals(data, other.data))
- return false;
- return true;
- }
-
- @Override
- public int compareTo(ByteArray o) {
- return Bytes.compareTo(this.data, o.data);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java b/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
deleted file mode 100644
index 5d46037..0000000
--- a/common/src/main/java/com/kylinolap/common/util/BytesSerializer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.nio.ByteBuffer;
-
-/**
- * @author yangli9
- *
- */
-public interface BytesSerializer<T> {
-
- public static final int SERIALIZE_BUFFER_SIZE = 65536;
-
- abstract public void serialize(T value, ByteBuffer out);
-
- abstract public T deserialize(ByteBuffer in);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/BytesUtil.java b/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
deleted file mode 100644
index 93d72a4..0000000
--- a/common/src/main/java/com/kylinolap/common/util/BytesUtil.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.common.util;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.Writable;
-
-public class BytesUtil {
-
- public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
-
- public static void writeLong(long num, byte[] bytes, int offset, int size) {
- for (int i = offset + size - 1; i >= offset; i--) {
- bytes[i] = (byte) num;
- num >>>= 8;
- }
- }
-
- public static void writeUnsigned(int num, byte[] bytes, int offset, int size) {
- for (int i = offset + size - 1; i >= offset; i--) {
- bytes[i] = (byte) num;
- num >>>= 8;
- }
- }
-
- public static long readLong(byte[] bytes, int offset, int size) {
- long integer = 0;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (long) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- public static int readUnsigned(byte[] bytes, int offset, int size) {
- int integer = 0;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (int) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- public static void writeSigned(int num, byte[] bytes, int offset, int size) {
- writeUnsigned(num, bytes, offset, size);
- }
-
- public static int readSigned(byte[] bytes, int offset, int size) {
- int integer = (bytes[offset] & 0x80) == 0 ? 0 : -1;
- for (int i = offset, n = offset + size; i < n; i++) {
- integer <<= 8;
- integer |= (int) bytes[i] & 0xFF;
- }
- return integer;
- }
-
- /**
- * No. bytes needed to store a value as big as the given
- */
- public static int sizeForValue(int maxValue) {
- int size = 0;
- while (maxValue > 0) {
- size++;
- maxValue >>>= 8;
- }
- return size;
- }
-
- public static int compareByteUnsigned(byte b1, byte b2) {
- int i1 = (int) b1 & 0xFF;
- int i2 = (int) b2 & 0xFF;
- return i1 - i2;
- }
-
- public static byte[] subarray(byte[] bytes, int start, int end) {
- byte[] r = new byte[end - start];
- System.arraycopy(bytes, start, r, 0, r.length);
- return r;
- }
-
- public static int compareBytes(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) {
- int r = 0;
- for (int i = 0; i < length; i++) {
- r = src[srcOffset + i] - dst[dstOffset + i];
- if (r != 0)
- break;
- }
- return r;
- }
-
- // from WritableUtils
- // ============================================================================
-
- public static void writeVInt(int i, ByteBuffer out) {
- writeVLong(i, out);
- }
-
- public static void writeVLong(long i, ByteBuffer out) {
- if (i >= -112 && i <= 127) {
- out.put((byte) i);
- return;
- }
-
- int len = -112;
- if (i < 0) {
- i ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = i;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- out.put((byte) len);
-
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- out.put((byte) ((i & mask) >> shiftbits));
- }
- }
-
- public static long readVLong(ByteBuffer in) {
- byte firstByte = in.get();
- int len = decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = in.get();
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (isNegativeVInt(firstByte) ? (i ^ -1L) : i);
- }
-
- public static int readVInt(ByteBuffer in) {
- long n = readVLong(in);
- if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) {
- throw new IllegalArgumentException("value too long to fit in integer");
- }
- return (int) n;
- }
-
- private static boolean isNegativeVInt(byte value) {
- return value < -120 || (value >= -112 && value < 0);
- }
-
- private static int decodeVIntSize(byte value) {
- if (value >= -112) {
- return 1;
- } else if (value < -120) {
- return -119 - value;
- }
- return -111 - value;
- }
-
- public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static int readUnsigned(ByteBuffer in, int size) {
- int integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-
- public static void writeLong(long num, ByteBuffer out) {
- for (int i = 0; i < 8; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static long readLong(ByteBuffer in) {
- long integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < 8; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-
- public static void writeUTFString(String str, ByteBuffer out) {
- byte[] bytes = str == null ? null : Bytes.toBytes(str);
- writeByteArray(bytes, out);
- }
-
- public static String readUTFString(ByteBuffer in) {
- byte[] bytes = readByteArray(in);
- return bytes == null ? null : Bytes.toString(bytes);
- }
-
- public static void writeAsciiString(String str, ByteBuffer out) {
- if (str == null) {
- BytesUtil.writeVInt(-1, out);
- return;
- }
- int len = str.length();
- BytesUtil.writeVInt(len, out);
- for (int i = 0; i < len; i++) {
- out.put((byte) str.charAt(i));
- }
- }
-
- public static String readAsciiString(ByteBuffer in) {
- int len = BytesUtil.readVInt(in);
- if (len < 0) {
- return null;
- }
- String result;
- try {
- if (in.hasArray()) {
- int pos = in.position();
- result = new String(in.array(), pos, len, "ISO-8859-1");
- in.position(pos + len);
- } else {
- byte[] tmp = new byte[len];
- in.get(tmp);
- result = new String(tmp, "ISO-8859-1");
- }
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- return result;
- }
-
- public static void writeAsciiStringArray(String[] strs, ByteBuffer out) {
- writeVInt(strs.length, out);
- for (int i = 0; i < strs.length; i++)
- writeAsciiString(strs[i], out);
- }
-
- public static String[] readAsciiStringArray(ByteBuffer in) {
- int len = readVInt(in);
- String[] strs = new String[len];
- for (int i = 0; i < len; i++)
- strs[i] = readAsciiString(in);
- return strs;
- }
-
- public static void writeIntArray(int[] array, ByteBuffer out) {
- if (array == null) {
- writeVInt(-1, out);
- return;
- }
- writeVInt(array.length, out);
- out.asIntBuffer().put(array);
- }
-
- public static int[] readIntArray(ByteBuffer in) {
- int len = readVInt(in);
- if (len < 0)
- return null;
- int[] array = new int[len];
- in.asIntBuffer().get(array);
- return array;
- }
-
- public static void writeByteArray(byte[] array, ByteBuffer out) {
- if (array == null) {
- writeVInt(-1, out);
- return;
- }
- writeVInt(array.length, out);
- out.put(array);
- }
-
- public static byte[] readByteArray(ByteBuffer in) {
- int len = readVInt(in);
- if (len < 0)
- return null;
-
- byte[] array = new byte[len];
- in.get(array);
- return array;
- }
-
- public static byte[] toBytes(Writable writable) {
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(bout);
- writable.write(out);
- out.close();
- bout.close();
- return bout.toByteArray();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static String toHex(byte[] array) {
- return toHex(new ImmutableBytesWritable(array));
- }
-
- public static String toHex(ImmutableBytesWritable bytes) {
- byte[] array = bytes.get();
- int offset = bytes.getOffset();
- int length = bytes.getLength();
- StringBuilder sb = new StringBuilder(length * 4);
- for (int i = 0; i < length; i++) {
- int b = array[offset + i];
- sb.append(String.format("\\x%02X", b & 0xFF));
- }
- return sb.toString();
- }
-
- public static void main(String[] args) throws Exception {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java b/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
deleted file mode 100644
index 6be3708..0000000
--- a/common/src/main/java/com/kylinolap/common/util/ClasspathUtil.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.net.URL;
-import java.net.URLClassLoader;
-
-/**
- * @author xduo
- *
- */
-public class ClasspathUtil {
-
- public static void addClasspath(String path) throws Exception {
- System.out.println("Adding path " + path + " to class path");
- File file = new File(path);
-
- if (file.exists()) {
- URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
- Class<URLClassLoader> urlClass = URLClassLoader.class;
- Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class });
- method.setAccessible(true);
- method.invoke(urlClassLoader, new Object[] { file.toURI().toURL() });
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java b/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
deleted file mode 100644
index 3f4ab7b..0000000
--- a/common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * @author yangli9
- */
-public class CliCommandExecutor {
-
- private String remoteHost;
- private String remoteUser;
- private String remotePwd;
- private int remoteTimeoutSeconds = 3600;
-
- public CliCommandExecutor() {
- }
-
- public void setRunAtRemote(String host, String user, String pwd) {
- this.remoteHost = host;
- this.remoteUser = user;
- this.remotePwd = pwd;
- }
-
- public void setRunAtLocal() {
- this.remoteHost = null;
- this.remoteUser = null;
- this.remotePwd = null;
- }
-
- public void copyFile(String localFile, String destDir) throws IOException {
- if (remoteHost == null)
- copyNative(localFile, destDir);
- else
- copyRemote(localFile, destDir);
- }
-
- private void copyNative(String localFile, String destDir) throws IOException {
- File src = new File(localFile);
- File dest = new File(destDir, src.getName());
- FileUtils.copyFile(src, dest);
- }
-
- private void copyRemote(String localFile, String destDir) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
- try {
- ssh.scpFileToRemote(localFile, destDir);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- public String execute(String command) throws IOException {
- Pair<Integer, String> r;
- if (remoteHost == null)
- r = runNativeCommand(command);
- else
- r = runRemoteCommand(command);
-
- if (r.getFirst() != 0)
- throw new IOException("OS command error exit with " + r.getFirst() + " -- " + command + "\n" + r.getSecond());
-
- return r.getSecond();
- }
-
- private Pair<Integer, String> runRemoteCommand(String command) throws IOException {
- SSHClient ssh = new SSHClient(remoteHost, remoteUser, remotePwd, null);
-
- SSHClientOutput sshOutput;
- try {
- sshOutput = ssh.execCommand(command, remoteTimeoutSeconds);
- int exitCode = sshOutput.getExitCode();
- String output = sshOutput.getText();
- return new Pair<Integer, String>(exitCode, output);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
- private Pair<Integer, String> runNativeCommand(String command) throws IOException {
- String[] cmd = new String[3];
- String osName = System.getProperty("os.name");
- if (osName.startsWith("Windows")) {
- cmd[0] = "cmd.exe";
- cmd[1] = "/C";
- } else {
- cmd[0] = "/bin/bash";
- cmd[1] = "-c";
- }
- cmd[2] = command;
-
- ProcessBuilder builder = new ProcessBuilder(cmd);
- builder.redirectErrorStream(true);
- Process proc = builder.start();
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- IOUtils.copy(proc.getInputStream(), buf);
- String output = buf.toString("UTF-8");
-
- try {
- int exitCode = proc.waitFor();
- return new Pair<Integer, String>(exitCode, output);
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
deleted file mode 100644
index a1dda9a..0000000
--- a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.common.util;
-
-import com.kylinolap.common.KylinConfig;
-
-/**
- * @author ysong1
- */
-public class HBaseMetadataTestCase extends AbstractKylinTestCase {
-
- public static void staticCreateTestMetadata() {
-
- KylinConfig.destoryInstance();
-
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, SANDBOX_TEST_DATA);
-
- }
-
- public static void staticCleanupTestMetadata() {
- System.clearProperty(KylinConfig.KYLIN_CONF);
- KylinConfig.destoryInstance();
- }
-
- @Override
- public void createTestMetadata() {
- staticCreateTestMetadata();
- }
-
- @Override
- public void cleanupTestMetadata() {
- staticCleanupTestMetadata();
- }
-
-}