You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/03/20 08:17:31 UTC
kylin git commit: KYLIN-1504 Use NavigableSet instead
ArrayList to keep rowkey original order instead of urgly using
Collections.sort
Repository: kylin
Updated Branches:
refs/heads/master 5227fb156 -> 801fb83b2
KYLIN-1504 Use NavigableSet<String> instead ArrayList<String> to keep rowkey original order instead of urgly using Collections.sort
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/801fb83b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/801fb83b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/801fb83b
Branch: refs/heads/master
Commit: 801fb83b22e6a737ca9c43155a4860951bf370a2
Parents: 5227fb1
Author: Hao Chen <ha...@apache.org>
Authored: Thu Mar 17 17:54:08 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Mar 20 15:16:49 2016 +0800
----------------------------------------------------------------------
.../common/persistence/FileResourceStore.java | 8 ++--
.../kylin/common/persistence/ResourceStore.java | 7 ++--
.../kylin/common/persistence/ResourceTool.java | 7 ++--
.../persistence/LocalFileResourceStoreTest.java | 3 +-
.../common/MetadataVersionRefresher.java | 5 +--
.../org/apache/kylin/cube/CubeManagerTest.java | 29 +++++++-------
.../apache/kylin/dict/DictionaryManager.java | 42 ++++++++------------
.../kylin/dict/lookup/SnapshotManager.java | 5 ++-
.../org/apache/kylin/job/dao/ExecutableDao.java | 31 +++++++--------
.../engine/mr/steps/MetadataCleanupJob.java | 15 +++----
.../storage/hbase/ITHBaseResourceStoreTest.java | 3 +-
.../apache/kylin/rest/service/JobService.java | 1 +
.../kylin/storage/hbase/HBaseResourceStore.java | 40 +++----------------
13 files changed, 79 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index 49ff441..1ab659f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -26,6 +26,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -49,12 +51,12 @@ public class FileResourceStore extends ResourceStore {
}
@Override
- protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
+ protected NavigableSet<String> listResourcesImpl(String resPath) throws IOException {
String[] names = file(resPath).list();
if (names == null) // not a directory
return null;
- ArrayList<String> r = new ArrayList<String>(names.length);
+ TreeSet<String> r = new TreeSet<>();
String prefix = resPath.endsWith("/") ? resPath : resPath + "/";
for (String n : names) {
r.add(prefix + n);
@@ -75,7 +77,7 @@ public class FileResourceStore extends ResourceStore {
try {
String commonPrefix = StringUtils.getCommonPrefix(rangeEnd, rangeStart);
commonPrefix = commonPrefix.substring(0, commonPrefix.lastIndexOf("/") + 1);
- final ArrayList<String> resources = listResourcesImpl(commonPrefix);
+ final NavigableSet<String> resources = listResourcesImpl(commonPrefix);
for (String resource : resources) {
if (resource.compareTo(rangeStart) >= 0 && resource.compareTo(rangeEnd) <= 0) {
if (existsImpl(resource)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 88ee553..4e9e904 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -26,6 +26,7 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.io.IOUtils;
@@ -116,12 +117,12 @@ abstract public class ResourceStore {
* return a list of child resources & folders under given path, return null
* if given path is not a folder
*/
- final public ArrayList<String> listResources(String resPath) throws IOException {
+ final public NavigableSet<String> listResources(String resPath) throws IOException {
resPath = norm(resPath);
return listResourcesImpl(resPath);
}
- abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException;
+ abstract protected NavigableSet<String> listResourcesImpl(String resPath) throws IOException;
/**
* return true if a resource exists, return false in case of folder or
@@ -278,7 +279,7 @@ abstract public class ResourceStore {
}
public void scanRecursively(String path, Visitor visitor) throws IOException {
- ArrayList<String> children = listResources(path);
+ NavigableSet<String> children = listResources(path);
if (children != null) {
for (String child : children)
scanRecursively(child, visitor);
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index b11fe1a..56f855c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
+import java.util.NavigableSet;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
@@ -99,7 +100,7 @@ public class ResourceTool {
public static void list(KylinConfig config, String path) throws IOException {
ResourceStore store = ResourceStore.getStore(config);
- ArrayList<String> result = store.listResources(path);
+ NavigableSet<String> result = store.listResources(path);
System.out.println("" + result);
}
@@ -119,7 +120,7 @@ public class ResourceTool {
}
public static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
- ArrayList<String> children = src.listResources(path);
+ NavigableSet<String> children = src.listResources(path);
// case of resource (not a folder)
if (children == null) {
@@ -169,7 +170,7 @@ public class ResourceTool {
}
private static void resetR(ResourceStore store, String path) throws IOException {
- ArrayList<String> children = store.listResources(path);
+ NavigableSet<String> children = store.listResources(path);
if (children == null) { // path is a resource (not a folder)
if (matchFilter(path)) {
store.deleteResource(path);
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
index a62a578..7ba5329 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
@@ -27,6 +27,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.NavigableSet;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -91,7 +92,7 @@ public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
}
// list
- ArrayList<String> list;
+ NavigableSet<String> list;
list = store.listResources(dir1);
assertTrue(list.contains(path1));
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
index 50833e3..02f084c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
@@ -21,10 +21,9 @@ package org.apache.kylin.cube.upgrade.common;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
+import java.util.NavigableSet;
-import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.metadata.MetadataConstants;
import org.slf4j.Logger;
@@ -73,7 +72,7 @@ public class MetadataVersionRefresher {
}
public static void collectFiles(ResourceStore src, String path, List<String> ret) throws IOException {
- ArrayList<String> children = src.listResources(path);
+ NavigableSet<String> children = src.listResources(path);
// case of resource (not a folder)
if (children == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 15c6deb..18a8150 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -18,17 +18,6 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.TimeZone;
-
-import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.JsonUtil;
@@ -41,6 +30,12 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableSet;
+
+import static org.junit.Assert.*;
+
/**
* @author yangli9
*/
@@ -130,12 +125,16 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
@Test
public void testGetAllCubes() throws Exception {
final ResourceStore store = ResourceStore.getStore(getTestConfig());
- final ArrayList<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT);
+ final NavigableSet<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT);
+ final Iterator<String> iterator = cubePath.iterator();
+ final String firstPath = iterator.next();
+ final String secondPath = iterator.next();
+ final String lastPath = cubePath.last();
assertTrue(cubePath.size() > 1);
- Collections.sort(cubePath);
- final List<CubeInstance> cubes = store.getAllResources(cubePath.get(0), cubePath.get(cubePath.size() - 1), CubeInstance.class, CubeManager.CUBE_SERIALIZER);
+
+ final List<CubeInstance> cubes = store.getAllResources(firstPath, lastPath, CubeInstance.class, CubeManager.CUBE_SERIALIZER);
assertEquals(cubePath.size(), cubes.size());
- assertEquals(cubePath.size() - 1, store.getAllResources(cubePath.get(1), cubePath.get(cubePath.size() - 1), CubeInstance.class, CubeManager.CUBE_SERIALIZER).size());
+ assertEquals(cubePath.size() - 1, store.getAllResources(secondPath, lastPath, CubeInstance.class, CubeManager.CUBE_SERIALIZER).size());
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index b7c95da..b3ed2ea 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -18,17 +18,8 @@
package org.apache.kylin.dict;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
+import com.google.common.cache.*;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.dimension.Dictionary;
@@ -43,12 +34,15 @@ import org.apache.kylin.source.SourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.common.collect.Lists;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
public class DictionaryManager {
@@ -175,7 +169,7 @@ public class DictionaryManager {
private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException {
ResourceStore store = MetadataManager.getInstance(config).getStore();
- ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
+ NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir());
if (existings == null)
return null;
@@ -318,13 +312,12 @@ public class DictionaryManager {
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
ResourceStore store = MetadataManager.getInstance(config).getStore();
- ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
+ NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir());
if (existings == null || existings.isEmpty()) {
return null;
}
- Collections.sort(existings);
- final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(existings.get(0), existings.get(existings.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
+ final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(existings.first(), existings.last(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
TableSignature input = dictInfo.getInput();
@@ -338,13 +331,12 @@ public class DictionaryManager {
private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException {
ResourceStore store = MetadataManager.getInstance(config).getStore();
- ArrayList<String> dictInfos = store.listResources(dictInfo.getResourceDir());
+ NavigableSet<String> dictInfos = store.listResources(dictInfo.getResourceDir());
if (dictInfos == null || dictInfos.isEmpty()) {
return null;
}
- Collections.sort(dictInfos);
- final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.get(0), dictInfos.get(dictInfos.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
+ final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.first(), dictInfos.last(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
DictionaryInfo largestDict = null;
for (DictionaryInfo dictionaryInfo : allResources) {
@@ -373,7 +365,7 @@ public class DictionaryManager {
info.setSourceColumn(srcCol);
ResourceStore store = MetadataManager.getInstance(config).getStore();
- ArrayList<String> existings = store.listResources(info.getResourceDir());
+ NavigableSet<String> existings = store.listResources(info.getResourceDir());
if (existings == null)
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 1123c2d..b9c17dd 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -20,6 +20,7 @@ package org.apache.kylin.dict.lookup;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.common.KylinConfig;
@@ -139,7 +140,7 @@ public class SnapshotManager {
private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
ResourceStore store = MetadataManager.getInstance(this.config).getStore();
String resourceDir = snapshot.getResourceDir();
- ArrayList<String> existings = store.listResources(resourceDir);
+ NavigableSet<String> existings = store.listResources(resourceDir);
if (existings == null)
return null;
@@ -157,7 +158,7 @@ public class SnapshotManager {
private String checkDupByContent(SnapshotTable snapshot) throws IOException {
ResourceStore store = MetadataManager.getInstance(this.config).getStore();
String resourceDir = snapshot.getResourceDir();
- ArrayList<String> existings = store.listResources(resourceDir);
+ NavigableSet<String> existings = store.listResources(resourceDir);
if (existings == null)
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index d063ee7..493555d 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.common.KylinConfig;
@@ -95,13 +96,12 @@ public class ExecutableDao {
public List<ExecutableOutputPO> getJobOutputs() throws PersistentException {
try {
- ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
+ NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
if (resources == null || resources.isEmpty()) {
return Collections.emptyList();
}
- Collections.sort(resources);
- String rangeStart = resources.get(0);
- String rangeEnd = resources.get(resources.size() - 1);
+ String rangeStart = resources.first();
+ String rangeEnd = resources.last();
return store.getAllResources(rangeStart, rangeEnd, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER);
} catch (IOException e) {
logger.error("error get all Jobs:", e);
@@ -111,13 +111,12 @@ public class ExecutableDao {
public List<ExecutableOutputPO> getJobOutputs(long timeStartInMillis, long timeEndInMillis) throws PersistentException {
try {
- ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
+ NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
if (resources == null || resources.isEmpty()) {
return Collections.emptyList();
}
- Collections.sort(resources);
- String rangeStart = resources.get(0);
- String rangeEnd = resources.get(resources.size() - 1);
+ String rangeStart = resources.first();
+ String rangeEnd = resources.last();
return store.getAllResources(rangeStart, rangeEnd, timeStartInMillis, timeEndInMillis, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER);
} catch (IOException e) {
logger.error("error get all Jobs:", e);
@@ -127,13 +126,12 @@ public class ExecutableDao {
public List<ExecutablePO> getJobs() throws PersistentException {
try {
- final List<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
+ final NavigableSet<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
if (jobIds == null || jobIds.isEmpty()) {
return Collections.emptyList();
}
- Collections.sort(jobIds);
- String rangeStart = jobIds.get(0);
- String rangeEnd = jobIds.get(jobIds.size() - 1);
+ String rangeStart = jobIds.first();
+ String rangeEnd = jobIds.last();
return store.getAllResources(rangeStart, rangeEnd, ExecutablePO.class, JOB_SERIALIZER);
} catch (IOException e) {
logger.error("error get all Jobs:", e);
@@ -143,13 +141,12 @@ public class ExecutableDao {
public List<ExecutablePO> getJobs(long timeStartInMillis, long timeEndInMillis) throws PersistentException {
try {
- final List<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
+ final NavigableSet<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
if (jobIds == null || jobIds.isEmpty()) {
return Collections.emptyList();
}
- Collections.sort(jobIds);
- String rangeStart = jobIds.get(0);
- String rangeEnd = jobIds.get(jobIds.size() - 1);
+ String rangeStart = jobIds.first();
+ String rangeEnd = jobIds.last();
return store.getAllResources(rangeStart, rangeEnd, timeStartInMillis, timeEndInMillis, ExecutablePO.class, JOB_SERIALIZER);
} catch (IOException e) {
logger.error("error get all Jobs:", e);
@@ -159,7 +156,7 @@ public class ExecutableDao {
public List<String> getJobIds() throws PersistentException {
try {
- ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
+ NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
if (resources == null) {
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
index 99f8a9f..b88a170 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java
@@ -18,10 +18,7 @@
package org.apache.kylin.engine.mr.steps;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
@@ -114,11 +111,11 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
// two level resources, snapshot tables and cube statistics
for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
- ArrayList<String> snapshotTables = getStore().listResources(resourceRoot);
+ NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);
if (snapshotTables != null) {
for (String snapshotTable : snapshotTables) {
- ArrayList<String> snapshotNames = getStore().listResources(snapshotTable);
+ NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
if (snapshotNames != null)
for (String snapshot : snapshotNames) {
if (!activeResourceList.contains(snapshot)) {
@@ -131,13 +128,13 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
}
// three level resources, only dictionaries
- ArrayList<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
+ NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
for (String table : dictTables) {
- ArrayList<String> tableColNames = getStore().listResources(table);
+ NavigableSet<String> tableColNames = getStore().listResources(table);
if (tableColNames != null)
for (String tableCol : tableColNames) {
- ArrayList<String> dictionaries = getStore().listResources(tableCol);
+ NavigableSet<String> dictionaries = getStore().listResources(tableCol);
if (dictionaries != null)
for (String dict : dictionaries)
if (!activeResourceList.contains(dict)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
index c560bb8..dccdca9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
@@ -27,6 +27,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.NavigableSet;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -133,7 +134,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
}
// list
- ArrayList<String> list;
+ NavigableSet<String> list;
list = store.listResources(dir1);
assertTrue(list.contains(path1));
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index cd8eef9..d0c29a4 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -90,6 +90,7 @@ public class JobService extends BasicService {
return jobs.subList(offset, offset + limit);
}
+
public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date());
http://git-wip-us.apache.org/repos/asf/kylin/blob/801fb83b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 6d77240..bbb5e21 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -22,11 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -41,10 +37,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.filter.*;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -67,22 +60,9 @@ public class HBaseResourceStore extends ResourceStore {
private static final String COLUMN_TS = "t";
private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS);
- private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>();
-
- static {
- TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube");
- TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict");
- TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex");
- TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj");
- TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot");
- TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE
- }
-
final String tableNameBase;
final String hbaseUrl;
- // final Map<String, String> tableNameMap; // path prefix ==> HBase table name
-
private HConnection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -97,15 +77,6 @@ public class HBaseResourceStore extends ResourceStore {
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
createHTableIfNeeded(getAllInOneTableName());
-
- // tableNameMap = new LinkedHashMap<String, String>();
- // for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) {
- // String pathPrefix = entry.getKey();
- // String tableName = tableNameBase + entry.getValue();
- // tableNameMap.put(pathPrefix, tableName);
- // createHTableIfNeeded(tableName);
- // }
-
}
private void createHTableIfNeeded(String tableName) throws IOException {
@@ -117,14 +88,14 @@ public class HBaseResourceStore extends ResourceStore {
}
@Override
- protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
+ protected NavigableSet<String> listResourcesImpl(String resPath) throws IOException {
assert resPath.startsWith("/");
String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
byte[] startRow = Bytes.toBytes(lookForPrefix);
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- ArrayList<String> result = new ArrayList<String>();
+ TreeSet<String> result = new TreeSet<>();
HTableInterface table = getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
@@ -136,8 +107,7 @@ public class HBaseResourceStore extends ResourceStore {
assert path.startsWith(lookForPrefix);
int cut = path.indexOf('/', lookForPrefix.length());
String child = cut < 0 ? path : path.substring(0, cut);
- if (result.contains(child) == false)
- result.add(child);
+ result.add(child);
}
} finally {
IOUtils.closeQuietly(table);