You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/06/25 06:00:49 UTC
[kylin] 01/02: KYLIN-3271 KYLIN-3454 minor refactor,
accelerate ResourceTool
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 95dc0bf8d5dfa05d840409f5ab82862b25f53d80
Author: jie.zou <ji...@kyligence.io>
AuthorDate: Wed Jun 5 14:58:01 2019 +0800
KYLIN-3271 KYLIN-3454 minor refactor,accelerate ResourceTool
---
.../kylin/common/persistence/ResourceTool.java | 31 +++++++---------------
.../kylin/common/persistence/ResourceToolTest.java | 23 ++++++++++++++--
2 files changed, 30 insertions(+), 24 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 3ff0694..c282a79 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Locale;
import java.util.NavigableSet;
import java.util.Set;
-import java.util.TreeSet;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceParallelCopier.Stats;
@@ -208,7 +207,7 @@ public class ResourceTool {
logger.info("Copy from {} to {}", src, dst);
- copyR(src, dst, path, getPathsSkipChildren(src), copyImmutableResource);
+ copyR(src, dst, path, copyImmutableResource);
}
public void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException {
@@ -224,7 +223,7 @@ public class ResourceTool {
logger.info("Copy from {} to {}", src, dst);
for (String path : paths) {
- copyR(src, dst, path, getPathsSkipChildren(src), copyImmutableResource);
+ copyR(src, dst, path, copyImmutableResource);
}
}
@@ -238,18 +237,19 @@ public class ResourceTool {
copy(srcConfig, dstConfig, "/", copyImmutableResource);
}
- private void copyR(ResourceStore src, ResourceStore dst, String path, TreeSet<String> pathsSkipChildrenCheck, boolean copyImmutableResource)
+ private void copyR(ResourceStore src, ResourceStore dst, String path, boolean copyImmutableResource)
throws IOException {
if (!copyImmutableResource && IMMUTABLE_PREFIX.contains(path)) {
return;
}
- NavigableSet<String> children = null;
+ boolean isSkip = SKIP_CHILDREN_CHECK_RESOURCE_ROOT.stream()
+ .anyMatch(prefixToSkip -> (path.startsWith(prefixToSkip)));
+ if (isSkip)
+ return;
- if (!pathsSkipChildrenCheck.contains(path)) {
- children = src.listResources(path);
- }
+ NavigableSet<String> children = src.listResources(path);
if (children == null) {
// case of resource (not a folder)
@@ -273,22 +273,9 @@ public class ResourceTool {
} else {
// case of folder
for (String child : children)
- copyR(src, dst, child, pathsSkipChildrenCheck, copyImmutableResource);
- }
-
- }
-
- private TreeSet<String> getPathsSkipChildren(ResourceStore src) throws IOException {
- TreeSet<String> pathsSkipChildrenCheck = new TreeSet<>();
-
- for (String resourceRoot : SKIP_CHILDREN_CHECK_RESOURCE_ROOT) {
- NavigableSet<String> all = src.listResourcesRecursively(resourceRoot);
- if (all != null) {
- pathsSkipChildrenCheck.addAll(src.listResourcesRecursively(resourceRoot));
- }
+ copyR(src, dst, child, copyImmutableResource);
}
- return pathsSkipChildrenCheck;
}
static boolean matchFilter(String path, String[] includePrefix, String[] excludePrefix) {
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
index e4efe32..dac69b8 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.common.persistence;
+import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -28,19 +29,32 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.util.List;
+import java.util.NavigableSet;
public class ResourceToolTest extends LocalFileMetadataTestCase {
private static final String dstPath = "../examples/test_metadata2/";
+ private static final File DIR_1 = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + ResourceStore.EXECUTE_RESOURCE_ROOT);
+ private static final File DIR_2 = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
+ private static final String FILE_1 = ResourceStore.EXECUTE_RESOURCE_ROOT + "/1.json";
+ private static final String FILE_2 = ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/2.json";
+ private static final List<String> EXEC_FILES = Lists.newArrayList(FILE_1, FILE_2);
@Before
public void setup() throws Exception {
- this.createTestMetadata();
+ FileUtils.forceMkdir(DIR_1);
+ FileUtils.forceMkdir(DIR_2);
+ FileUtils.write(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_1), "");
+ FileUtils.write(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_2), "");
FileUtils.forceMkdir(new File(dstPath));
FileUtils.cleanDirectory(new File(dstPath));
+ this.createTestMetadata();
}
@After
public void after() throws Exception {
+ FileUtils.deleteQuietly(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_1));
+ FileUtils.deleteQuietly(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_2));
File directory = new File(dstPath);
try {
FileUtils.deleteDirectory(directory);
@@ -63,6 +77,11 @@ public class ResourceToolTest extends LocalFileMetadataTestCase {
new ResourceTool().copy(KylinConfig.getInstanceFromEnv(), dstConfig, "/");
//After copy, two paths have same metadata
- Assert.assertEquals(srcStore.listResources("/"), dstStore.listResources("/"));
+ NavigableSet<String> dstFiles = dstStore.listResourcesRecursively("/");
+ NavigableSet<String> srcFiles = srcStore.listResourcesRecursively("/");
+ Assert.assertTrue(srcFiles.containsAll(EXEC_FILES));
+ Assert.assertFalse(dstFiles.containsAll(EXEC_FILES));
+ srcFiles.removeAll(EXEC_FILES);
+ Assert.assertEquals(srcFiles, dstFiles);
}
}