You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/01/13 06:02:41 UTC
[kylin] 02/02: KYLIN-5391 Kylin metadata tool for read specific file
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 80ff2285027d1c55aa07bb07c0afd64c823a71ea
Author: zhennzhang <zh...@ebay.com>
AuthorDate: Fri Jan 13 11:50:43 2023 +0800
KYLIN-5391 Kylin metadata tool for read specific file
---
build/bin/metastore.sh | 43 ++++++++++-
.../java/org/apache/kylin/tool/MetadataTool.java | 84 +++++++++++++++++++++-
.../org/apache/kylin/tool/MetadataToolTest.java | 32 +++++++++
3 files changed, 157 insertions(+), 2 deletions(-)
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 05c998eae1..9e791eea71 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -26,6 +26,8 @@ fi
function help {
echo "usage: metastore.sh backup METADATA_BACKUP_PATH(the default path is KYLIN_HOME/meta_backups/)"
+ echo " metastore.sh fetch TARGET_FILE_PATH METADATA_FETCH_PATH(the default path is KYLIN_HOME/meta_fetch/)"
+ echo " metastore.sh list TARGET_FOLDER_PATH"
echo " metastore.sh restore METADATA_RESTORE_PATH [--after-truncate]"
echo " metastore.sh backup-project PROJECT_NAME METADATA_BACKUP_PATH(the default path is KYLIN_HOME/meta_backups/)"
echo " metastore.sh restore-project PROJECT_NAME METADATA_RESTORE_PATH [--after-truncate]"
@@ -44,6 +46,18 @@ function printBackupResult() {
fi
}
+function printFetchResult() {
+ error=$1
+ if [[ $error == 0 ]]; then
+ if [[ -z "$path" ]]; then
+ path="\${KYLIN_HOME}/meta_fetch"
+ fi
+ echo -e "${YELLOW} Fetch at local dist succeed.${RESTORE}"
+ else
+ echo -e "${YELLOW} Fetch failed.${RESTORE}"
+ fi
+}
+
function printRestoreResult() {
error=$1
@@ -134,6 +148,34 @@ then
${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.MetadataTool ${BACKUP_OPTS}
printBackupResult $?
+elif [ "$1" == "fetch" ]
+then
+ FETCH_OPTS="-fetch"
+ if [ $# -eq 2 ]; then
+ _file=$2
+ FETCH_OPTS="${FETCH_OPTS} -target ${_file}"
+ elif [ $# -eq 3 ]; then
+ _file=$2
+ path=`cd $3 && pwd -P`
+ check_path_empty ${path}
+ FETCH_OPTS="${FETCH_OPTS} -target ${_file} -dir ${path}"
+ else
+ help
+ fi
+
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.MetadataTool ${FETCH_OPTS}
+ printFetchResult $?
+
+elif [ "$1" == "list" ]
+then
+ if [ $# -eq 2 ]; then
+ _folder=$2
+ LIST_OPTS="-list -target ${_folder}"
+ else
+ help
+ fi
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.MetadataTool ${LIST_OPTS}
+
elif [ "$1" == "restore" ]
then
if [ $# -eq 2 ]; then
@@ -171,4 +213,3 @@ else
help
fi
-
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
index 66aa1d8b3d..daa8aed307 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
@@ -87,6 +87,12 @@ public class MetadataTool extends ExecutableApplication {
private static final Option OPERATE_COMPRESS = OptionBuilder.getInstance()
.withDescription("Backup compressed metadata to HDFS path").isRequired(false).create("compress");
+ private static final Option OPERATE_FETCH = OptionBuilder.getInstance()
+ .withDescription("Fetch part of metadata to local path").isRequired(false).create("fetch");
+
+ private static final Option OPERATE_LIST = OptionBuilder.getInstance()
+ .withDescription("List children of target folder").isRequired(false).create("list");
+
private static final Option OPERATE_RESTORE = OptionBuilder.getInstance()
.withDescription("Restore metadata from local path or HDFS path").isRequired(false).create("restore");
@@ -100,6 +106,9 @@ public class MetadataTool extends ExecutableApplication {
private static final Option OPTION_PROJECT = OptionBuilder.getInstance().hasArg().withArgName("PROJECT_NAME")
.withDescription("Specify project level backup and restore (optional)").isRequired(false).create("project");
+ private static final Option OPTION_TARGET = OptionBuilder.getInstance().hasArg().withArgName("TARGET_FILE")
+ .withDescription("Specify part of metadata for fetch to local path").isRequired(false).create("target");
+
private static final Option FOLDER_NAME = OptionBuilder.getInstance().hasArg().withArgName("FOLDER_NAME")
.withDescription("Specify the folder name for backup").isRequired(false).create("folder");
@@ -116,6 +125,9 @@ public class MetadataTool extends ExecutableApplication {
@Getter
private String backupPath;
+ @Getter
+ private String fetchPath;
+
MetadataTool() {
kylinConfig = KylinConfig.getInstanceFromEnv();
this.options = new Options();
@@ -154,7 +166,8 @@ public class MetadataTool extends ExecutableApplication {
val optionsHelper = new OptionsHelper();
optionsHelper.parseOptions(tool.getOptions(), args);
boolean isBackup = optionsHelper.hasOption(OPERATE_BACKUP);
- if (isBackup && ScreenPrintUtil.isMainThread()) {
+ boolean isFetch = optionsHelper.hasOption(OPERATE_FETCH);
+ if ((isBackup || isFetch) && ScreenPrintUtil.isMainThread()) {
config.setProperty("kylin.env.metadata.only-for-read", "true");
}
val resourceStore = ResourceStore.getKylinMetaStore(config);
@@ -258,12 +271,15 @@ public class MetadataTool extends ExecutableApplication {
final OptionGroup optionGroup = new OptionGroup();
optionGroup.setRequired(true);
optionGroup.addOption(OPERATE_BACKUP);
+ optionGroup.addOption(OPERATE_FETCH);
+ optionGroup.addOption(OPERATE_LIST);
optionGroup.addOption(OPERATE_RESTORE);
options.addOptionGroup(optionGroup);
options.addOption(OPTION_DIR);
options.addOption(OPTION_PROJECT);
options.addOption(FOLDER_NAME);
+ options.addOption(OPTION_TARGET);
options.addOption(OPERATE_COMPRESS);
options.addOption(OPTION_EXCLUDE_TABLE_EXD);
options.addOption(OPTION_AFTER_TRUNCATE);
@@ -305,6 +321,10 @@ public class MetadataTool extends ExecutableApplication {
}
}
+ } else if (optionsHelper.hasOption(OPERATE_FETCH)) {
+ fetch(optionsHelper);
+ } else if (optionsHelper.hasOption(OPERATE_LIST)) {
+ list(optionsHelper);
} else if (optionsHelper.hasOption(OPERATE_RESTORE)) {
restore(optionsHelper, optionsHelper.hasOption(OPTION_AFTER_TRUNCATE));
} else {
@@ -330,6 +350,68 @@ public class MetadataTool extends ExecutableApplication {
}
}
+ private void fetch(OptionsHelper optionsHelper) throws Exception {
+ var path = optionsHelper.getOptionValue(OPTION_DIR);
+ var folder = optionsHelper.getOptionValue(FOLDER_NAME);
+ val excludeTableExd = optionsHelper.hasOption(OPTION_EXCLUDE_TABLE_EXD);
+ val target = optionsHelper.getOptionValue(OPTION_TARGET);
+ if (StringUtils.isBlank(path)) {
+ path = KylinConfigBase.getKylinHome() + File.separator + "meta_fetch";
+ }
+ if (StringUtils.isEmpty(folder)) {
+ folder = LocalDateTime.now(Clock.systemDefaultZone()).format(DATE_TIME_FORMATTER) + "_fetch";
+ }
+ if (target == null) {
+ System.out.println("target file must be set with fetch mode");
+ } else {
+ fetchPath = StringUtils.appendIfMissing(path, "/") + folder;
+ // currently do not support compress with fetch
+ val fetchMetadataUrl = getMetadataUrl(fetchPath, false);
+ val fetchConfig = KylinConfig.createKylinConfig(kylinConfig);
+ fetchConfig.setMetadataUrl(fetchMetadataUrl);
+ abortIfAlreadyExists(fetchPath);
+ logger.info("The fetch metadataUrl is {} and backup path is {}", fetchMetadataUrl, fetchPath);
+
+ try (val fetchResourceStore = ResourceStore.getKylinMetaStore(fetchConfig)) {
+
+ val fetchMetadataStore = fetchResourceStore.getMetadataStore();
+
+ String targetPath = target.startsWith("/") ? target.substring(1) : target;
+
+ logger.info("start to copy target file {} from ResourceStore.", target);
+ UnitOfWork.doInTransactionWithRetry(
+ UnitOfWorkParams.builder().readonly(true).unitName(target).processor(() -> {
+ copyResourceStore("/" + targetPath, resourceStore, fetchResourceStore, true, excludeTableExd);
+ // uuid
+ val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
+ fetchResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
+ uuid.getTimestamp(), -1);
+ return null;
+ }).build());
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("metadata task is interrupt");
+ }
+ logger.info("start to fetch target file {}", target);
+
+ // fetchResourceStore is read-only, currently we don't do any write operation on it.
+ // fetchResourceStore.deleteResource(ResourceStore.METASTORE_TRASH_RECORD);
+ fetchMetadataStore.dump(fetchResourceStore);
+ logger.info("fetch successfully at {}", fetchPath);
+ }
+ }
+ }
+
+ private NavigableSet<String> list(OptionsHelper optionsHelper) throws Exception {
+ val target = optionsHelper.getOptionValue(OPTION_TARGET);
+ var res = resourceStore.listResources(target);
+ if (res == null) {
+ System.out.printf("%s is not exist%n", target);
+ } else {
+ System.out.println("" + res);
+ }
+ return res;
+ }
+
private void backup(OptionsHelper optionsHelper) throws Exception {
val project = optionsHelper.getOptionValue(OPTION_PROJECT);
var path = optionsHelper.getOptionValue(OPTION_DIR);
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
index b82216ab9f..d8ddd329aa 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
@@ -124,6 +124,38 @@ public class MetadataToolTest extends NLocalFileMetadataTestCase {
return tool;
}
+ @Test
+ public void testFetchTargetFile() throws IOException {
+ val junitFolder = temporaryFolder.getRoot();
+ val tool = new MetadataTool(getTestConfig());
+ // test case for fetching a specific file
+ tool.execute(new String[] {
+ "-fetch", "-target", "default/table/DEFAULT.STREAMING_TABLE.json", "-dir", junitFolder.getAbsolutePath(), "-folder", "target_fetch"
+ });
+ //test case for fetching a folder
+ tool.execute(new String[] {
+ "-fetch", "-target", "_global", "-dir", junitFolder.getAbsolutePath(), "-folder", "target_fetch_global"
+ });
+
+ Assertions.assertThat(junitFolder.listFiles()).hasSize(2);
+ val archiveFolder = junitFolder.listFiles()[1];
+ val globleFolder = junitFolder.listFiles()[0];
+ Assertions.assertThat(archiveFolder).exists();
+
+ Assertions.assertThat(archiveFolder.list()).isNotEmpty().containsOnly("default", "UUID");
+
+ val projectFolder = findFile(archiveFolder.listFiles(), f -> f.getName().equals("default"));
+ assertProjectFolder(projectFolder, globleFolder);
+ }
+
+ @Test
+ public void testListFile() {
+ val tool = new MetadataTool(getTestConfig());
+ tool.execute(new String[] {
+ "-list", "-target", "default"
+ });
+ }
+
@Test
public void testBackupProject() throws IOException {
val junitFolder = temporaryFolder.getRoot();