You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/06/25 06:00:48 UTC
[kylin] branch 2.6.x updated (f346884 -> c0b69b0)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from f346884 Minor, remove duplicate code
new 95dc0bf KYLIN-3271 KYLIN-3454 minor refactor,accelerate ResourceTool
new c0b69b0 KYLIN-4024 Support pushdown to presto
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../kylin/common/persistence/ResourceTool.java | 31 +++++++---------------
.../kylin/common/persistence/ResourceToolTest.java | 23 ++++++++++++++--
.../kylin/query/adhoc/PushDownRunnerJdbcImpl.java | 9 ++++++-
3 files changed, 38 insertions(+), 25 deletions(-)
[kylin] 01/02: KYLIN-3271 KYLIN-3454 minor refactor,
accelerate ResourceTool
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 95dc0bf8d5dfa05d840409f5ab82862b25f53d80
Author: jie.zou <ji...@kyligence.io>
AuthorDate: Wed Jun 5 14:58:01 2019 +0800
KYLIN-3271 KYLIN-3454 minor refactor,accelerate ResourceTool
---
.../kylin/common/persistence/ResourceTool.java | 31 +++++++---------------
.../kylin/common/persistence/ResourceToolTest.java | 23 ++++++++++++++--
2 files changed, 30 insertions(+), 24 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 3ff0694..c282a79 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Locale;
import java.util.NavigableSet;
import java.util.Set;
-import java.util.TreeSet;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceParallelCopier.Stats;
@@ -208,7 +207,7 @@ public class ResourceTool {
logger.info("Copy from {} to {}", src, dst);
- copyR(src, dst, path, getPathsSkipChildren(src), copyImmutableResource);
+ copyR(src, dst, path, copyImmutableResource);
}
public void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException {
@@ -224,7 +223,7 @@ public class ResourceTool {
logger.info("Copy from {} to {}", src, dst);
for (String path : paths) {
- copyR(src, dst, path, getPathsSkipChildren(src), copyImmutableResource);
+ copyR(src, dst, path, copyImmutableResource);
}
}
@@ -238,18 +237,19 @@ public class ResourceTool {
copy(srcConfig, dstConfig, "/", copyImmutableResource);
}
- private void copyR(ResourceStore src, ResourceStore dst, String path, TreeSet<String> pathsSkipChildrenCheck, boolean copyImmutableResource)
+ private void copyR(ResourceStore src, ResourceStore dst, String path, boolean copyImmutableResource)
throws IOException {
if (!copyImmutableResource && IMMUTABLE_PREFIX.contains(path)) {
return;
}
- NavigableSet<String> children = null;
+ boolean isSkip = SKIP_CHILDREN_CHECK_RESOURCE_ROOT.stream()
+ .anyMatch(prefixToSkip -> (path.startsWith(prefixToSkip)));
+ if (isSkip)
+ return;
- if (!pathsSkipChildrenCheck.contains(path)) {
- children = src.listResources(path);
- }
+ NavigableSet<String> children = src.listResources(path);
if (children == null) {
// case of resource (not a folder)
@@ -273,22 +273,9 @@ public class ResourceTool {
} else {
// case of folder
for (String child : children)
- copyR(src, dst, child, pathsSkipChildrenCheck, copyImmutableResource);
- }
-
- }
-
- private TreeSet<String> getPathsSkipChildren(ResourceStore src) throws IOException {
- TreeSet<String> pathsSkipChildrenCheck = new TreeSet<>();
-
- for (String resourceRoot : SKIP_CHILDREN_CHECK_RESOURCE_ROOT) {
- NavigableSet<String> all = src.listResourcesRecursively(resourceRoot);
- if (all != null) {
- pathsSkipChildrenCheck.addAll(src.listResourcesRecursively(resourceRoot));
- }
+ copyR(src, dst, child, copyImmutableResource);
}
- return pathsSkipChildrenCheck;
}
static boolean matchFilter(String path, String[] includePrefix, String[] excludePrefix) {
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
index e4efe32..dac69b8 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.common.persistence;
+import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -28,19 +29,32 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.util.List;
+import java.util.NavigableSet;
public class ResourceToolTest extends LocalFileMetadataTestCase {
private static final String dstPath = "../examples/test_metadata2/";
+ private static final File DIR_1 = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + ResourceStore.EXECUTE_RESOURCE_ROOT);
+ private static final File DIR_2 = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
+ private static final String FILE_1 = ResourceStore.EXECUTE_RESOURCE_ROOT + "/1.json";
+ private static final String FILE_2 = ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/2.json";
+ private static final List<String> EXEC_FILES = Lists.newArrayList(FILE_1, FILE_2);
@Before
public void setup() throws Exception {
- this.createTestMetadata();
+ FileUtils.forceMkdir(DIR_1);
+ FileUtils.forceMkdir(DIR_2);
+ FileUtils.write(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_1), "");
+ FileUtils.write(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_2), "");
FileUtils.forceMkdir(new File(dstPath));
FileUtils.cleanDirectory(new File(dstPath));
+ this.createTestMetadata();
}
@After
public void after() throws Exception {
+ FileUtils.deleteQuietly(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_1));
+ FileUtils.deleteQuietly(new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + FILE_2));
File directory = new File(dstPath);
try {
FileUtils.deleteDirectory(directory);
@@ -63,6 +77,11 @@ public class ResourceToolTest extends LocalFileMetadataTestCase {
new ResourceTool().copy(KylinConfig.getInstanceFromEnv(), dstConfig, "/");
//After copy, two paths have same metadata
- Assert.assertEquals(srcStore.listResources("/"), dstStore.listResources("/"));
+ NavigableSet<String> dstFiles = dstStore.listResourcesRecursively("/");
+ NavigableSet<String> srcFiles = srcStore.listResourcesRecursively("/");
+ Assert.assertTrue(srcFiles.containsAll(EXEC_FILES));
+ Assert.assertFalse(dstFiles.containsAll(EXEC_FILES));
+ srcFiles.removeAll(EXEC_FILES);
+ Assert.assertEquals(srcFiles, dstFiles);
}
}
[kylin] 02/02: KYLIN-4024 Support pushdown to presto
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c0b69b08db52190614f9356c8614a31c76ea48c0
Author: hit-lacus <hi...@126.com>
AuthorDate: Sun Jun 9 18:54:12 2019 +0800
KYLIN-4024 Support pushdown to presto
---
.../org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index abdeeb3..de73327 100644
--- a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
+++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -75,7 +75,8 @@ public class PushDownRunnerJdbcImpl extends AbstractPushdownRunner {
}
}
- private SelectedColumnMeta extractColumnMeta(ResultSetMetaData resultSetMetaData, int columnIndex) throws SQLException {
+ private SelectedColumnMeta extractColumnMeta(ResultSetMetaData resultSetMetaData, int columnIndex)
+ throws SQLException {
boolean isAutoIncrement = false;
try {
isAutoIncrement = resultSetMetaData.isAutoIncrement(columnIndex);
@@ -256,6 +257,12 @@ public class PushDownRunnerJdbcImpl extends AbstractPushdownRunner {
return Types.ARRAY;
} else if ("struct".equalsIgnoreCase(type)) {
return Types.STRUCT;
+ } else if ("integer".equalsIgnoreCase(type)) {
+ return Types.INTEGER;
+ } else if ("time".equalsIgnoreCase(type)) {
+ return Types.VARCHAR;
+ } else if ("varbinary".equalsIgnoreCase(type)) {
+ return Types.BINARY;
}
throw new SQLException("Unrecognized column type: " + type);
}