You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/03/11 06:28:10 UTC
[kylin] 01/04: KYLIN-3275, add ut for storage clean job.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch sync
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9ab6b52c1a40a69e6793a76e3fc2f53d18d1b57e
Author: Jiatao Tao <24...@qq.com>
AuthorDate: Mon Mar 5 18:40:07 2018 +0800
KYLIN-3275, add ut for storage clean job.
---
.../kylin/rest/job/StorageCleanJobHbaseUtil.java | 10 +-
.../apache/kylin/rest/job/StorageCleanupJob.java | 63 +++++---
.../apache/kylin/rest/service/AdminService.java | 7 +-
.../rest/job/StorageCleanJobHbaseUtilTest.java | 75 +++++++++
.../kylin/rest/job/StorageCleanupJobTest.java | 129 ++++++++++++++++
.../hbase_storage_ut/cube/ci_inner_join_cube.json | 51 +++++++
.../execute/091a0322-249c-43e7-91df-205603ab6883 | 28 ++++
.../execute/f8edd777-8756-40d5-be19-3159120e4f7b | 167 +++++++++++++++++++++
.../091a0322-249c-43e7-91df-205603ab6883 | 12 ++
.../f8edd777-8756-40d5-be19-3159120e4f7b | 14 ++
.../execute/d9a2b721-9916-4607-8047-148ceb2473b1 | 14 ++
11 files changed, 544 insertions(+), 26 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 9933fb4..61351ae 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -28,7 +28,6 @@ import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -45,11 +44,16 @@ public class StorageCleanJobHbaseUtil {
protected static final Logger logger = LoggerFactory.getLogger(StorageCleanJobHbaseUtil.class);
public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException {
- Configuration conf = HBaseConfiguration.create();
+ try (HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create())) {
+ cleanUnusedHBaseTables(hbaseAdmin, delete, deleteTimeout);
+ }
+ }
+
+ static void cleanUnusedHBaseTables(HBaseAdmin hbaseAdmin, boolean delete, int deleteTimeout) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
// get all kylin hbase tables
- try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
+ try {
String namespace = kylinConfig.getHBaseStorageNameSpace();
String tableNamePrefix = (namespace.equals("default") || namespace.equals(""))
? kylinConfig.getHBaseTableNamePrefix() : (namespace + ":" + kylinConfig.getHBaseTableNamePrefix());
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 7730ee3..59bd21f 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -32,7 +32,6 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -71,12 +70,27 @@ public class StorageCleanupJob extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class);
public static final int deleteTimeout = 10; // Unit minute
-
+ protected FileSystem hbaseFs;
+ protected FileSystem defaultFs;
protected boolean delete = false;
protected boolean force = false;
protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig
.getInstanceFromEnv());
+ public StorageCleanupJob(FileSystem defaultFs, FileSystem hbaseFs) {
+ this.defaultFs = defaultFs;
+ this.hbaseFs = hbaseFs;
+ }
+
+ public StorageCleanupJob() throws IOException {
+ this.defaultFs = HadoopUtil.getWorkingFileSystem();
+ this.hbaseFs = HadoopUtil.getWorkingFileSystem(HBaseConfiguration.create());
+ }
+
+ public void setDelete(boolean delete) {
+ this.delete = delete;
+ }
+
protected void cleanUnusedHBaseTables() throws IOException {
KylinConfig config = KylinConfig.getInstanceFromEnv();
if ("hbase".equals(config.getMetadataUrl().getScheme())) {
@@ -107,21 +121,29 @@ public class StorageCleanupJob extends AbstractApplication {
logger.info("force option value: '" + optionsHelper.getOptionValue(OPTION_FORCE) + "'");
delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
force = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_FORCE));
- cleanUnusedIntermediateHiveTable();
+
KylinConfig config = KylinConfig.getInstanceFromEnv();
+ cleanUnusedIntermediateHiveTable(getHiveTables(), getCliCommandExecutor());
+
if (StringUtils.isNotEmpty(config.getHBaseClusterFs())) {
- cleanUnusedHdfsFiles(HBaseConfiguration.create());
+ cleanUnusedHdfsFiles(hbaseFs);
}
- Configuration conf = HadoopUtil.getCurrentConfiguration();
- cleanUnusedHdfsFiles(conf);
+ cleanUnusedHdfsFiles(defaultFs);
cleanUnusedHBaseTables();
}
- private void cleanUnusedHdfsFiles(Configuration conf) throws IOException {
+ protected List<String> getHiveTables() throws Exception {
+ ISourceMetadataExplorer explr = SourceFactory.getDefaultSource().getSourceMetadataExplorer();
+ return explr.listTables(KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable());
+ }
+ protected CliCommandExecutor getCliCommandExecutor() throws IOException {
+ return KylinConfig.getInstanceFromEnv().getCliCommandExecutor();
+ }
+
+ void cleanUnusedHdfsFiles(FileSystem fs) throws IOException {
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
@@ -129,12 +151,14 @@ public class StorageCleanupJob extends AbstractApplication {
// TODO: when first use, /kylin/kylin_metadata does not exist.
try {
FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
- for (FileStatus status : fStatus) {
- String path = status.getPath().getName();
- // System.out.println(path);
- if (path.startsWith("kylin-")) {
- String kylinJobPath = engineConfig.getHdfsWorkingDirectory() + path;
- allHdfsPathsNeedToBeDeleted.add(kylinJobPath);
+ if (fStatus != null) {
+ for (FileStatus status : fStatus) {
+ String path = status.getPath().getName();
+ // System.out.println(path);
+ if (path.startsWith("kylin-")) {
+ String kylinJobPath = engineConfig.getHdfsWorkingDirectory() + path;
+ allHdfsPathsNeedToBeDeleted.add(kylinJobPath);
+ }
}
}
} catch (FileNotFoundException e) {
@@ -187,17 +211,13 @@ public class StorageCleanupJob extends AbstractApplication {
}
}
- private void cleanUnusedIntermediateHiveTable() throws Exception {
- Configuration conf = HadoopUtil.getCurrentConfiguration();
+ void cleanUnusedIntermediateHiveTable(List<String> hiveTableNames, CliCommandExecutor cmdExec) throws Exception {
final KylinConfig config = KylinConfig.getInstanceFromEnv();
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
- final CliCommandExecutor cmdExec = config.getCliCommandExecutor();
final int uuidLength = 36;
final String preFix = MetadataConstants.KYLIN_INTERMEDIATE_PREFIX;
final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
- ISourceMetadataExplorer explr = SourceFactory.getDefaultSource().getSourceMetadataExplorer();
- List<String> hiveTableNames = explr.listTables(config.getHiveDatabaseForIntermediateTable());
Iterable<String> kylinIntermediates = Iterables.filter(hiveTableNames, new Predicate<String>() {
@Override
public boolean apply(@Nullable String input) {
@@ -294,9 +314,8 @@ public class StorageCleanupJob extends AbstractApplication {
String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(),
segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
Path externalDataPath = new Path(path);
- FileSystem fs = HadoopUtil.getWorkingFileSystem();
- if (fs.exists(externalDataPath)) {
- fs.delete(externalDataPath, true);
+ if (defaultFs.exists(externalDataPath)) {
+ defaultFs.delete(externalDataPath, true);
logger.info("Hive table {}'s external path {} deleted", tableToDelete, path);
} else {
logger.info(
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index 1da423b..70afefd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -89,7 +89,12 @@ public class AdminService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void cleanupStorage() {
- StorageCleanupJob job = new StorageCleanupJob();
+ StorageCleanupJob job = null;
+ try {
+ job = new StorageCleanupJob();
+ } catch (IOException e) {
+ logger.error("can not init StorageCleanupJob", e);
+ }
String[] args = new String[] { "-delete", "true" };
job.execute(args);
}
diff --git a/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtilTest.java
new file mode 100644
index 0000000..5ce8813
--- /dev/null
+++ b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtilTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.job;
+
+import static org.apache.kylin.common.util.LocalFileMetadataTestCase.cleanAfterClass;
+import static org.apache.kylin.common.util.LocalFileMetadataTestCase.staticCreateTestMetadata;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase.OverlayMetaHook;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import com.google.common.collect.Lists;
+
+public class StorageCleanJobHbaseUtilTest {
+ @Before
+ public void setup() {
+ staticCreateTestMetadata(true, new OverlayMetaHook("src/test/resources/ut_meta/hbase_storage_ut/"));
+ }
+
+ @After
+ public void after() {
+ cleanAfterClass();
+ }
+
+ @Test
+ public void test() throws IOException {
+ HBaseAdmin hBaseAdmin = mock(HBaseAdmin.class);
+ HTableDescriptor[] hds = new HTableDescriptor[2];
+ HTableDescriptor d1 = mock(HTableDescriptor.class);
+ HTableDescriptor d2 = mock(HTableDescriptor.class);
+ hds[0] = d1;
+ hds[1] = d2;
+ when(d1.getValue("KYLIN_HOST")).thenReturn("../examples/test_metadata/");
+ when(d2.getValue("KYLIN_HOST")).thenReturn("../examples/test_metadata/");
+ when(d1.getTableName()).thenReturn(TableName.valueOf("KYLIN_J9TE08D9IA"));
+ String toBeDel = "to-be-del";
+ when(d2.getTableName()).thenReturn(TableName.valueOf(toBeDel));
+ when(hBaseAdmin.listTables("KYLIN_.*")).thenReturn(hds);
+
+ when(hBaseAdmin.tableExists(toBeDel)).thenReturn(true);
+ when(hBaseAdmin.isTableEnabled(toBeDel)).thenReturn(false);
+ StorageCleanJobHbaseUtil.cleanUnusedHBaseTables(hBaseAdmin, true, 100000);
+
+ ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
+ verify(hBaseAdmin).deleteTable(captor.capture());
+ assertEquals(Lists.newArrayList(toBeDel), captor.getAllValues());
+ }
+}
diff --git a/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
new file mode 100644
index 0000000..4bb426b
--- /dev/null
+++ b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.job;
+
+import static org.apache.kylin.common.util.LocalFileMetadataTestCase.cleanAfterClass;
+import static org.apache.kylin.common.util.LocalFileMetadataTestCase.staticCreateTestMetadata;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.notNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase.OverlayMetaHook;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import com.google.common.collect.Lists;
+
+public class StorageCleanupJobTest {
+ @Before
+ public void setup() {
+ staticCreateTestMetadata(true, new OverlayMetaHook("src/test/resources/ut_meta/storage_ut/"));
+ }
+
+ @After
+ public void after() {
+ cleanAfterClass();
+ }
+
+ @Test
+ public void test() throws Exception {
+ FileSystem mockFs = mock(FileSystem.class);
+ prepareUnusedIntermediateHiveTable(mockFs);
+ prepareUnusedHDFSFiles(mockFs);
+
+ MockStorageCleanupJob job = new MockStorageCleanupJob(mockFs, mockFs);
+ job.execute(new String[] { "--delete", "true" });
+
+ ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
+ verify(mockFs, times(2)).delete(pathCaptor.capture(), eq(true));
+ ArrayList<Path> expected = Lists.newArrayList(
+ // verifyCleanUnusedIntermediateHiveTable
+ new Path("file:///tmp/examples/test_metadata/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/kylin_intermediate_2838c7fc-722a-48fa-9d1a-8ab37837a952"),
+
+ // verifyCleanUnusedHdfsFiles
+ new Path("file:///tmp/examples/test_metadata/kylin-to-be-delete")
+ );
+ assertEquals(expected, pathCaptor.getAllValues());
+ }
+
+ private void prepareUnusedHDFSFiles(FileSystem mockFs) throws IOException {
+ Path p1 = new Path("file:///tmp/examples/test_metadata/");
+ FileStatus[] statuses = new FileStatus[3];
+ FileStatus f1 = mock(FileStatus.class);
+ FileStatus f2 = mock(FileStatus.class);
+ FileStatus f3 = mock(FileStatus.class);
+ // only remove FINISHED and DISCARDED job intermediate files, so this exclude.
+ when(f1.getPath()).thenReturn(new Path("kylin-091a0322-249c-43e7-91df-205603ab6883"));
+ // remove every segment working dir from deletion list, so this exclude.
+ when(f2.getPath()).thenReturn(new Path("kylin-bcf2f125-9b0b-40dd-9509-95ec59b31333"));
+ when(f3.getPath()).thenReturn(new Path("kylin-to-be-delete"));
+ statuses[0] = f1;
+ statuses[1] = f2;
+ statuses[2] = f3;
+
+ when(mockFs.listStatus(p1)).thenReturn(statuses);
+ Path p2 = new Path("file:///tmp/examples/test_metadata/kylin-to-be-delete");
+ when(mockFs.exists(p2)).thenReturn(true);
+ }
+
+ private void prepareUnusedIntermediateHiveTable(FileSystem mockFs) throws IOException {
+ Path p1 = new Path(
+ "file:///tmp/examples/test_metadata/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/kylin_intermediate_2838c7fc-722a-48fa-9d1a-8ab37837a952");
+ when(mockFs.exists(p1)).thenReturn(true);
+ }
+
+ class MockStorageCleanupJob extends StorageCleanupJob {
+
+ MockStorageCleanupJob(FileSystem defaultFs, FileSystem hbaseFs) {
+ super(defaultFs, hbaseFs);
+ }
+
+ @Override
+ protected List<String> getHiveTables() throws Exception {
+ List<String> l = new ArrayList<>();
+ l.add("kylin_intermediate_2838c7fc-722a-48fa-9d1a-8ab37837a952");
+ // wrong prefix, so this is exclude.
+ l.add("wrong_prefix_6219a647-d8be-49bb-8562-3f4976922a96");
+ // intermediate table still in use, so this is exclude.
+ l.add("kylin_intermediate_091a0322-249c-43e7-91df-205603ab6883");
+ return l;
+ }
+
+ @Override
+ protected CliCommandExecutor getCliCommandExecutor() throws IOException {
+ CliCommandExecutor mockCli = mock(CliCommandExecutor.class);
+ when(mockCli.execute((String) notNull())).thenReturn(null);
+ return mockCli;
+ }
+ }
+}
diff --git a/server-base/src/test/resources/ut_meta/hbase_storage_ut/cube/ci_inner_join_cube.json b/server-base/src/test/resources/ut_meta/hbase_storage_ut/cube/ci_inner_join_cube.json
new file mode 100644
index 0000000..27315e4
--- /dev/null
+++ b/server-base/src/test/resources/ut_meta/hbase_storage_ut/cube/ci_inner_join_cube.json
@@ -0,0 +1,51 @@
+{
+ "uuid" : "8372c3b7-a33e-4b69-83dd-0bb8b1f8117e",
+ "last_modified" : 0,
+ "name" : "ci_inner_join_cube",
+ "owner" : null,
+ "descriptor" : "ci_inner_join_cube",
+ "display_name" : "ci_inner_join_cube",
+ "cost" : 50,
+ "status" : "READY",
+ "segments" : [{
+ "uuid" : "adb2275e-c1d5-498b-b8ea-0201586868aa",
+ "name" : "19700101000000_20220701000000",
+ "storage_location_identifier" : "KYLIN_J9TE08D9IA",
+ "date_range_start" : 0,
+ "date_range_end" : 1656633600000,
+ "source_offset_start" : 0,
+ "source_offset_end" : 0,
+ "status" : "READY",
+ "size_kb" : 78965,
+ "input_records" : 9911,
+ "input_records_size" : 3955601,
+ "last_build_time" : 1511364122982,
+ "last_build_job_id" : "b58ed952-3041-4431-b560-09b78fa1fca0",
+ "create_time_utc" : 1511363249614,
+ "cuboid_shard_nums" : {
+ "7864320" : 2,
+ "8847375" : 6,
+ "41943504" : 6,
+ "458784" : 2,
+ "3145728" : 2,
+ "8323104" : 4,
+ "2097152" : 2,
+ "8388608" : 2,
+ "16252928" : 6,
+ "25230848" : 6,
+ "3670016" : 2,
+ "67108863" : 6
+ },
+ "total_shards" : 0,
+ "blackout_cuboids" : [ ],
+ "binary_signature" : null,
+ "dictionaries" : {
+ "BUYER_COUNTRY.NAME" : "/dict/DEFAULT.TEST_COUNTRY/NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict"
+ },
+ "snapshots" : {
+ "DEFAULT.TEST_COUNTRY" : "/table_snapshot/DEFAULT.TEST_COUNTRY/7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot"
+ },
+ "rowkey_stats" : [ [ "TEST_KYLIN_FACT.LEAF_CATEG_ID", 134, 1 ], [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", 44, 1 ], [ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 94, 1 ], [ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 127, 1 ], [ "TEST_KYLIN_FACT.LSTG_SITE_ID", 8, 1 ], [ "TEST_KYLIN_FACT.SLR_SEGMENT_CD", 8, 1 ], [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", 5, 1 ], [ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", 5, 1 ], [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", 8, 1 ], [ "BUYER_COUNTRY.NAME", 8, 1 ], [ "SE [...]
+ }],
+ "create_time" : null
+}
\ No newline at end of file
diff --git a/server-base/src/test/resources/ut_meta/storage_ut/execute/091a0322-249c-43e7-91df-205603ab6883 b/server-base/src/test/resources/ut_meta/storage_ut/execute/091a0322-249c-43e7-91df-205603ab6883
new file mode 100644
index 0000000..e5febde
--- /dev/null
+++ b/server-base/src/test/resources/ut_meta/storage_ut/execute/091a0322-249c-43e7-91df-205603ab6883
@@ -0,0 +1,28 @@
+{
+ "uuid" : "091a0322-249c-43e7-91df-205603ab6883",
+ "last_modified" : 1508913805780,
+ "version" : "2.3.0.20500",
+ "name" : "BUILD CUBE - sample_cube - FULL_BUILD - GMT+08:00 2017-10-25 14:43:25",
+ "tasks" : [ {
+ "uuid" : "091a0322-249c-43e7-91df-205603ab6883-00",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Create Intermediate Flat Hive Table",
+ "tasks" : null,
+ "type" : "org.apache.kylin.source.hive.CreateFlatHiveTableStep",
+ "params" : {
+ "HiveInit" : "USE default;\n",
+ "HiveRedistributeData" : "DROP TABLE IF EXISTS kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952;\nCREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952\n(\nSAMPLE_07_DESCRIPTION string\n,SAMPLE_07_TOTAL_EMP int\n,SAMPLE_07_SALARY int\n)\nSTORED AS SEQUENCEFILE\nLOCATION 'hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-091a0322-249c-43e7-91df-205603ab6883/kylin_intermediate_sample_cube [...]
+ "cubeName" : "sample_cube"
+ }
+ } ],
+ "type" : "org.apache.kylin.engine.mr.CubingJob",
+ "params" : {
+ "submitter" : "ADMIN",
+ "envName" : "DEV",
+ "segmentId" : "2838c7fc-722a-48fa-9d1a-8ab37837a123",
+ "notify_list" : "",
+ "projectName" : "default",
+ "cubeName" : "sample_cube"
+ }
+}
diff --git a/server-base/src/test/resources/ut_meta/storage_ut/execute/f8edd777-8756-40d5-be19-3159120e4f7b b/server-base/src/test/resources/ut_meta/storage_ut/execute/f8edd777-8756-40d5-be19-3159120e4f7b
new file mode 100644
index 0000000..aec9220
--- /dev/null
+++ b/server-base/src/test/resources/ut_meta/storage_ut/execute/f8edd777-8756-40d5-be19-3159120e4f7b
@@ -0,0 +1,167 @@
+{
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b",
+ "last_modified" : 1508913805780,
+ "version" : "2.3.0.20500",
+ "name" : "BUILD CUBE - sample_cube - FULL_BUILD - GMT+08:00 2017-10-25 14:43:25",
+ "tasks" : [ {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-00",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Create Intermediate Flat Hive Table",
+ "tasks" : null,
+ "type" : "org.apache.kylin.source.hive.CreateFlatHiveTableStep",
+ "params" : {
+ "HiveInit" : "USE default;\n",
+ "HiveRedistributeData" : "DROP TABLE IF EXISTS kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952;\nCREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952\n(\nSAMPLE_07_DESCRIPTION string\n,SAMPLE_07_TOTAL_EMP int\n,SAMPLE_07_SALARY int\n)\nSTORED AS SEQUENCEFILE\nLOCATION 'hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/kylin_intermediate_sample_cube [...]
+ "cubeName" : "sample_cube"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-01",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Extract Fact Table Distinct Columns",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.MapReduceExecutable",
+ "params" : {
+ "MR_JOB_PARAMS" : " -conf /Users/wangcheng/Developments/kylin/server/../examples/test_case_data/sandbox/kylin_job_conf.xml -cubename sample_cube -output hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/fact_distinct_columns -segmentid 2838c7fc-722a-48fa-9d1a-8ab37837a952 -statisticsenabled true -statisticsoutput hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40 [...]
+ "MR_COUNTER_SAVEAS" : "sourceRecordCount,sourceSizeBytes",
+ "MR_JOB_CLASS" : "org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-02",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Build Dimension Dictionary",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.HadoopShellExecutable",
+ "params" : {
+ "HADOOP_SHELL_JOB_CLASS" : "org.apache.kylin.engine.mr.steps.CreateDictionaryJob",
+ "HADOOP_SHELL_JOB_PARAMS" : " -cubename sample_cube -segmentid 2838c7fc-722a-48fa-9d1a-8ab37837a952 -input hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/fact_distinct_columns"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-03",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Save Cuboid Statistics",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.steps.SaveStatisticsStep",
+ "params" : {
+ "cubingJobId" : "f8edd777-8756-40d5-be19-3159120e4f7b",
+ "segmentId" : "2838c7fc-722a-48fa-9d1a-8ab37837a952",
+ "statisticsPath" : "hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/fact_distinct_columns/statistics",
+ "cubeName" : "sample_cube"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-04",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Create HTable",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.HadoopShellExecutable",
+ "params" : {
+ "HADOOP_SHELL_JOB_CLASS" : "org.apache.kylin.storage.hbase.steps.CreateHTableJob",
+ "HADOOP_SHELL_JOB_PARAMS" : " -cubename sample_cube -segmentid 2838c7fc-722a-48fa-9d1a-8ab37837a952 -partitions hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/rowkey_stats/part-r-00000 -statisticsenabled true"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-05",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Build Base Cuboid",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.MapReduceExecutable",
+ "params" : {
+ "MR_JOB_PARAMS" : " -conf /Users/wangcheng/Developments/kylin/server/../examples/test_case_data/sandbox/kylin_job_conf.xml -cubename sample_cube -segmentid 2838c7fc-722a-48fa-9d1a-8ab37837a952 -input FLAT_TABLE -output hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/cuboid/level_base_cuboid -jobname Kylin_Base_Cuboid_Builder_sample_cube -level 0 -cubingJobId f8edd777-8756-40d5-be19-3159120e4f7b",
+ "MR_JOB_CLASS" : "org.apache.kylin.engine.mr.steps.BaseCuboidJob"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-06",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Build N-Dimension Cuboid : level 1",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.MapReduceExecutable",
+ "params" : {
+ "MR_JOB_PARAMS" : " -conf /Users/wangcheng/Developments/kylin/server/../examples/test_case_data/sandbox/kylin_job_conf.xml -cubename sample_cube -segmentid 2838c7fc-722a-48fa-9d1a-8ab37837a952 -input hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/cuboid/level_base_cuboid -output hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube [...]
+ "MR_JOB_CLASS" : "org.apache.kylin.engine.mr.steps.NDCuboidJob"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-07",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Build N-Dimension Cuboid : level 2",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.MapReduceExecutable",
+ "params" : {
+ "MR_JOB_PARAMS" : " -conf /Users/wangcheng/Developments/kylin/server/../examples/test_case_data/sandbox/kylin_job_conf.xml -cubename sample_cube -segmentid 2838c7fc-722a-48fa-9d1a-8ab37837a952 -input hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/cuboid/level_1_cuboid -output hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/cu [...]
+ "MR_JOB_CLASS" : "org.apache.kylin.engine.mr.steps.NDCuboidJob"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-08",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Build Cube In-Mem",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.MapReduceExecutable",
+ "params" : {
+ "MR_JOB_PARAMS" : " -conf /Users/wangcheng/Developments/kylin/server/../examples/test_case_data/sandbox/kylin_job_conf_inmem.xml -cubename sample_cube -segmentid 2838c7fc-722a-48fa-9d1a-8ab37837a952 -output hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/cuboid/ -jobname Kylin_Cube_Builder_sample_cube -cubingJobId f8edd777-8756-40d5-be19-3159120e4f7b",
+ "MR_JOB_CLASS" : "org.apache.kylin.engine.mr.steps.InMemCuboidJob"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-09",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Convert Cuboid Data to HFile",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.MapReduceExecutable",
+ "params" : {
+ "MR_JOB_PARAMS" : " -conf /Users/wangcheng/Developments/kylin/server/../examples/test_case_data/sandbox/kylin_job_conf.xml -cubename sample_cube -partitions hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/rowkey_stats/part-r-00000_hfile -input hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/cuboid/* -output hdfs://sandbox.hort [...]
+ "MR_COUNTER_SAVEAS" : ",,byteSizeBytes",
+ "MR_JOB_CLASS" : "org.apache.kylin.storage.hbase.steps.CubeHFileJob"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-10",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Load HFile to HBase Table",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.common.HadoopShellExecutable",
+ "params" : {
+ "HADOOP_SHELL_JOB_CLASS" : "org.apache.kylin.storage.hbase.steps.BulkLoadJob",
+ "HADOOP_SHELL_JOB_PARAMS" : " -input hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/hfile -htablename KYLIN_7SJCQA7S8W -cubename sample_cube"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-11",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Update Cube Info",
+ "tasks" : null,
+ "type" : "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep",
+ "params" : {
+ "cubingJobId" : "f8edd777-8756-40d5-be19-3159120e4f7b",
+ "output.path" : "hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/sample_cube/fact_distinct_columns",
+ "segmentId" : "2838c7fc-722a-48fa-9d1a-8ab37837a952",
+ "cubeName" : "sample_cube"
+ }
+ }, {
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b-12",
+ "last_modified" : 0,
+ "version" : "2.3.0.20500",
+ "name" : "Hive Cleanup",
+ "tasks" : null,
+ "type" : "org.apache.kylin.source.hive.HiveMRInput$GarbageCollectionStep",
+ "params" : {
+ "oldHiveTable" : "default.kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952",
+ "oldHiveViewIntermediateTables" : "",
+ "externalDataPath" : "hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be19-3159120e4f7b/kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952"
+ }
+ } ],
+ "type" : "org.apache.kylin.engine.mr.CubingJob",
+ "params" : {
+ "submitter" : "ADMIN",
+ "envName" : "DEV",
+ "segmentId" : "2838c7fc-722a-48fa-9d1a-8ab37837a952",
+ "notify_list" : "",
+ "projectName" : "default",
+ "cubeName" : "sample_cube"
+ }
+}
\ No newline at end of file
diff --git a/server-base/src/test/resources/ut_meta/storage_ut/execute_output/091a0322-249c-43e7-91df-205603ab6883 b/server-base/src/test/resources/ut_meta/storage_ut/execute_output/091a0322-249c-43e7-91df-205603ab6883
new file mode 100644
index 0000000..ded2764
--- /dev/null
+++ b/server-base/src/test/resources/ut_meta/storage_ut/execute_output/091a0322-249c-43e7-91df-205603ab6883
@@ -0,0 +1,12 @@
+{
+ "uuid" : "091a0322-249c-43e7-91df-205603ab6883",
+ "last_modified" : 1517798371078,
+ "version" : "2.3.0.20505",
+ "content" : null,
+ "status" : "READY",
+ "info" : {
+ "startTime" : "1517798275124",
+ "mapReduceWaitTime" : "34392",
+ "endTime" : "1517798371069"
+ }
+}
\ No newline at end of file
diff --git a/server-base/src/test/resources/ut_meta/storage_ut/execute_output/f8edd777-8756-40d5-be19-3159120e4f7b b/server-base/src/test/resources/ut_meta/storage_ut/execute_output/f8edd777-8756-40d5-be19-3159120e4f7b
new file mode 100644
index 0000000..fb26725
--- /dev/null
+++ b/server-base/src/test/resources/ut_meta/storage_ut/execute_output/f8edd777-8756-40d5-be19-3159120e4f7b
@@ -0,0 +1,14 @@
+{
+ "uuid" : "f8edd777-8756-40d5-be19-3159120e4f7b",
+ "last_modified" : 1508914247021,
+ "version" : "2.3.0.20500",
+ "content" : "Create and distribute table, cmd: \nhive -e \"USE default;\n\nDROP TABLE IF EXISTS kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952;\nCREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_sample_cube_2838c7fc_722a_48fa_9d1a_8ab37837a952\n(\nSAMPLE_07_DESCRIPTION string\n,SAMPLE_07_TOTAL_EMP int\n,SAMPLE_07_SALARY int\n)\nSTORED AS SEQUENCEFILE\nLOCATION 'hdfs://sandbox.hortonworks.com:8020/kylin/g-dev_kylinmaster_ci_instance/kylin-f8edd777-8756-40d5-be1 [...]
+ "status" : "SUCCEED",
+ "info" : {
+ "startTime" : "1508913834417",
+ "mapReduceWaitTime" : "88082",
+ "interruptTime" : "193157",
+ "algorithm" : "LAYER",
+ "endTime" : "1508914247000"
+ }
+}
\ No newline at end of file
diff --git a/server/src/test/resources/ut_meta/storage_ut/execute/d9a2b721-9916-4607-8047-148ceb2473b1 b/server/src/test/resources/ut_meta/storage_ut/execute/d9a2b721-9916-4607-8047-148ceb2473b1
new file mode 100644
index 0000000..fb8ff30
--- /dev/null
+++ b/server/src/test/resources/ut_meta/storage_ut/execute/d9a2b721-9916-4607-8047-148ceb2473b1
@@ -0,0 +1,14 @@
+{
+ "uuid" : "d9a2b721-9916-4607-8047-148ceb2473b1",
+ "last_modified" : 1516778161249,
+ "version" : "2.3.0",
+ "name" : null,
+ "tasks" : null,
+ "tasks_check" : null,
+ "type" : "org.apache.kylin.job.BadClassName",
+ "params" : {
+ "test2" : "test2",
+ "test3" : "test3",
+ "test1" : "test1"
+ }
+}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
liyang@apache.org.