You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/04/12 07:09:57 UTC
(doris) 11/27: [case](regression) Add backup restore with NGRAM bloom filter (#33479)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 031f1e6a6531170b6cea946f17dccd8b7e54b119
Author: walter <w4...@gmail.com>
AuthorDate: Wed Apr 10 21:56:18 2024 +0800
[case](regression) Add backup restore with NGRAM bloom filter (#33479)
---
.../test_backup_restore_ngram_bloom_filter.groovy | 168 +++++++++++++++++++++
1 file changed, 168 insertions(+)
diff --git a/regression-test/suites/backup_restore/test_backup_restore_ngram_bloom_filter.groovy b/regression-test/suites/backup_restore/test_backup_restore_ngram_bloom_filter.groovy
new file mode 100644
index 00000000000..ea97c7c2c07
--- /dev/null
+++ b/regression-test/suites/backup_restore/test_backup_restore_ngram_bloom_filter.groovy
@@ -0,0 +1,168 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_backup_restore_ngram_bloom_filter", "backup_restore") {
+ String suiteName = "test_backup_restore_ngram_bloom_filter"
+ String repoName = "${suiteName}_repo"
+ String dbName = "${suiteName}_db"
+ String tableName = "${suiteName}_table"
+ String snapshotName = "${suiteName}_snapshot"
+
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE if NOT EXISTS ${dbName}.${tableName}
+ (
+ `test` INT,
+ `id` INT,
+ `username` varchar(32) NULL DEFAULT "",
+ `only4test` varchar(32) NULL DEFAULT "",
+ INDEX idx_ngrambf (`username`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256")
+ )
+ ENGINE=OLAP
+ DUPLICATE KEY(`test`, `id`)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "bloom_filter_columns" = "id"
+ )
+ """
+ for (int index = 0; index < 10; index++) {
+ sql """
+ INSERT INTO ${dbName}.${tableName} VALUES (${index}, ${index}, "test_${index}", "${index}_test")
+ """
+ }
+
+ def checkNgramBf = { inputRes -> Boolean
+ for (List<Object> row : inputRes) {
+ if (row[2] == "idx_ngrambf" && row[10] == "NGRAM_BF") {
+ return true
+ }
+ }
+ return false
+ }
+ def checkBloomFilter = { inputRes -> Boolean
+ for (List<Object> row : inputRes) {
+ if ((row[1] as String).contains("\"bloom_filter_columns\" = \"id\"")) {
+ return true
+ }
+ }
+ return false
+ }
+ List<List<Object>> res = sql "SHOW INDEXES FROM ${dbName}.${tableName}"
+ assertTrue(checkNgramBf(res));
+ res = sql "SHOW CREATE TABLE ${dbName}.${tableName}"
+ assertTrue(checkBloomFilter(res));
+
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ PROPERTIES ("type" = "full")
+ """
+
+ syncer.waitSnapshotFinish(dbName)
+
+ def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+
+ assertTrue(snapshot != null)
+
+ sql "TRUNCATE TABLE ${dbName}.${tableName}"
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON ( `${tableName}`)
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "reserve_replica" = "true"
+ )
+ """
+
+ syncer.waitAllRestoreFinish(dbName)
+
+ res = sql "SHOW INDEXES FROM ${dbName}.${tableName}"
+ assertTrue(checkNgramBf(res));
+ res = sql "SHOW CREATE TABLE ${dbName}.${tableName}"
+ assertTrue(checkBloomFilter(res));
+
+ sql """
+ ALTER TABLE ${dbName}.${tableName}
+ ADD INDEX idx_only4test(`only4test`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256")
+ """
+ def checkNgramBf1 = { inputRes -> Boolean
+ for (List<Object> row : inputRes) {
+ if (row[2] == "idx_only4test" && row[10] == "NGRAM_BF") {
+ return true
+ }
+ }
+ return false
+ }
+
+ int count = 0;
+ while (true) {
+ res = sql "SHOW INDEXES FROM ${dbName}.${tableName}"
+ if (checkNgramBf1(res)) {
+ break
+ }
+ count += 1;
+ if (count >= 30) {
+ throw new IllegalStateException("alter table add index timeouted")
+ }
+ Thread.sleep(1000);
+ }
+
+ snapshotName = "${snapshotName}_1"
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ PROPERTIES ("type" = "full")
+ """
+
+ syncer.waitSnapshotFinish(dbName)
+
+ snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+
+ assertTrue(snapshot != null)
+
+ sql "TRUNCATE TABLE ${dbName}.${tableName}"
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON ( `${tableName}`)
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "reserve_replica" = "true"
+ )
+ """
+
+ syncer.waitAllRestoreFinish(dbName)
+
+ res = sql "SHOW INDEXES FROM ${dbName}.${tableName}"
+ assertTrue(checkNgramBf1(res));
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org