You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2024/07/09 01:49:08 UTC
(doris) branch branch-3.0 updated (098d8f594b2 -> 2c35e89f247)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 098d8f594b2 [fix](recycler) Fix missing provider in S3Conf (#37468)
new a3065881142 [fix](create table) create table fail not write drop table editlog (#37488)
new d71a78e36ca [fix](Azure) Enhance the glob list's logic for azure file system in FE (#37490)
new 2c35e89f247 bump to 3.0.0-rc09
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/doris/datasource/InternalCatalog.java | 29 ++++++----
.../org/apache/doris/fs/obj/AzureObjStorage.java | 62 +++++++++++++++++-----
gensrc/script/gen_build_version.sh | 2 +-
.../test_create_table_exception.groovy | 43 ++++++++++-----
4 files changed, 98 insertions(+), 38 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/03: bump to 3.0.0-rc09
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2c35e89f247bd2c960b39a2421c63934d48daf94
Author: Yongqiang YANG <da...@gmail.com>
AuthorDate: Tue Jul 9 09:48:45 2024 +0800
bump to 3.0.0-rc09
---
gensrc/script/gen_build_version.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh
index 104feccd85a..cb7277ecaa5 100755
--- a/gensrc/script/gen_build_version.sh
+++ b/gensrc/script/gen_build_version.sh
@@ -31,7 +31,7 @@ build_version_prefix="doris"
build_version_major=3
build_version_minor=0
build_version_patch=0
-build_version_rc_version="rc08"
+build_version_rc_version="rc09"
build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}-${build_version_rc_version}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/03: [fix](create table) create table fail not write drop table editlog (#37488)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a3065881142d002982c62560121ea3a02dbef439
Author: yujun <yu...@gmail.com>
AuthorDate: Tue Jul 9 09:38:52 2024 +0800
[fix](create table) create table fail not write drop table editlog (#37488)
Fix: for creating table, if create dynamic partition throw exception (pr
#35778), then create table will fail, then drop table, but it forget to
write an editlog.
---
.../apache/doris/datasource/InternalCatalog.java | 29 ++++++++++-----
.../test_create_table_exception.groovy | 43 +++++++++++++++-------
2 files changed, 49 insertions(+), 23 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 68d7db6256c..2ffc472fab0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1001,13 +1001,13 @@ public class InternalCatalog implements CatalogIf<Database> {
return true;
}
- public void dropTable(Database db, long tableId, boolean isForceDrop,
+ private void dropTable(Database db, long tableId, boolean isForceDrop, boolean isReplay,
Long recycleTime) throws MetaNotFoundException {
Table table = db.getTableOrMetaException(tableId);
db.writeLock();
table.writeLock();
try {
- unprotectDropTable(db, table, isForceDrop, true, recycleTime);
+ unprotectDropTable(db, table, isForceDrop, isReplay, recycleTime);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), tableId);
Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId());
} finally {
@@ -1018,7 +1018,7 @@ public class InternalCatalog implements CatalogIf<Database> {
public void replayDropTable(Database db, long tableId, boolean isForceDrop,
Long recycleTime) throws MetaNotFoundException {
- dropTable(db, tableId, isForceDrop, recycleTime);
+ dropTable(db, tableId, isForceDrop, true, recycleTime);
}
public void replayEraseTable(long tableId) {
@@ -2805,6 +2805,7 @@ public class InternalCatalog implements CatalogIf<Database> {
// if failed in any step, use this set to do clear things
Set<Long> tabletIdSet = new HashSet<>();
// create partition
+ boolean hadLogEditCreateTable = false;
try {
if (partitionInfo.getType() == PartitionType.UNPARTITIONED) {
if (properties != null && !properties.isEmpty()) {
@@ -2936,11 +2937,6 @@ public class InternalCatalog implements CatalogIf<Database> {
if (!result.first) {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
}
- if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) {
- LOG.info("debug point FE.createOlapTable.exception, throw e");
- // not commit, not log edit
- throw new DdlException("debug point FE.createOlapTable.exception");
- }
if (result.second) {
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
@@ -2953,6 +2949,9 @@ public class InternalCatalog implements CatalogIf<Database> {
}
LOG.info("duplicate create table[{};{}], skip next steps", tableName, tableId);
} else {
+ // if table not exists, then db.createTableWithLock will write an editlog.
+ hadLogEditCreateTable = true;
+
// we have added these index to memory, only need to persist here
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
GroupId groupId = Env.getCurrentColocateIndex().getGroup(tableId);
@@ -2971,17 +2970,27 @@ public class InternalCatalog implements CatalogIf<Database> {
.createOrUpdateRuntimeInfo(tableId, DynamicPartitionScheduler.LAST_UPDATE_TIME,
TimeUtils.getCurrentFormatTime());
}
+
+ if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) {
+ LOG.info("debug point FE.createOlapTable.exception, throw e");
+ throw new DdlException("debug point FE.createOlapTable.exception");
+ }
} catch (DdlException e) {
LOG.warn("create table failed {} - {}", tabletIdSet, e.getMessage());
for (Long tabletId : tabletIdSet) {
Env.getCurrentInvertedIndex().deleteTablet(tabletId);
}
- // only remove from memory, because we have not persist it
+ // edit log write DropTableInfo will result in deleting colocate group,
+ // but follow fe may need wait 30s (recycle bin mgr run every 30s).
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
Env.getCurrentColocateIndex().removeTable(tableId);
}
try {
- dropTable(db, tableId, true, 0L);
+ dropTable(db, tableId, true, false, 0L);
+ if (hadLogEditCreateTable) {
+ DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, true, 0L);
+ Env.getCurrentEnv().getEditLog().logDropTable(info);
+ }
} catch (Exception ex) {
LOG.warn("drop table", ex);
}
diff --git a/regression-test/suites/partition_p0/test_create_table_exception.groovy b/regression-test/suites/partition_p0/test_create_table_exception.groovy
index 49cadcd3af4..96f097c7670 100644
--- a/regression-test/suites/partition_p0/test_create_table_exception.groovy
+++ b/regression-test/suites/partition_p0/test_create_table_exception.groovy
@@ -31,9 +31,9 @@ suite("test_create_table_exception") {
def table3 = "dynamic_partition_table"
try {
GetDebugPoint().enableDebugPointForAllFEs('FE.createOlapTable.exception', null)
- def createTable = { ->
+ def createTable = { tableIdx ->
try_sql """
- CREATE TABLE $table1 (
+ CREATE TABLE ${table1}_${tableIdx} (
`k1` int(11) NULL,
`k2` int(11) NULL
)
@@ -41,12 +41,13 @@ suite("test_create_table_exception") {
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 10
PROPERTIES (
+ "colocate_with" = "col_grp_${tableIdx}",
"replication_num"="3"
);
"""
try_sql """
- CREATE TABLE IF NOT EXISTS $table2 (
+ CREATE TABLE IF NOT EXISTS ${table2}_${tableIdx} (
lo_orderdate int(11) NOT NULL COMMENT "",
lo_orderkey bigint(20) NOT NULL COMMENT "",
lo_linenumber bigint(20) NOT NULL COMMENT "",
@@ -79,7 +80,7 @@ suite("test_create_table_exception") {
"""
try_sql """
- CREATE TABLE $table3 (
+ CREATE TABLE ${table3}_${tableIdx} (
time date,
key1 int,
key2 int,
@@ -109,19 +110,35 @@ suite("test_create_table_exception") {
);
"""
}
- createTable()
+ createTable(1)
def result = sql """show tables;"""
assertEquals(result.size(), 0)
+
+ def checkResult = { ->
+ def tables = sql """show tables;"""
+ log.info("tables=" + tables)
+ assertEquals(3, tables.size())
+
+ def groups = sql """ show proc "/colocation_group" """
+ log.info("groups=" + groups)
+ assertEquals(1, groups.size())
+ }
+
GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception')
- createTable()
- result = sql """show tables;"""
- log.info(result.toString())
- assertEquals(result.size(), 3)
+ createTable(2)
+ checkResult()
+
+ sleep 1000
+ cluster.restartFrontends(cluster.getMasterFe().index)
+ sleep 32_000
+ def newMasterFe = cluster.getMasterFe()
+ def newMasterFeUrl = "jdbc:mysql://${newMasterFe.host}:${newMasterFe.queryPort}/?useLocalSessionState=false&allowLoadLocalInfile=false"
+ newMasterFeUrl = context.config.buildUrlWithDb(newMasterFeUrl, context.dbName)
+ connect('root', '', newMasterFeUrl) {
+ checkResult()
+ }
+
} finally {
- GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception')
- sql """drop table if exists ${table1}"""
- sql """drop table if exists ${table2}"""
- sql """drop table if exists ${table3}"""
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/03: [fix](Azure) Enhance the glob list's logic for azure file system in FE (#37490)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d71a78e36ca0baa5f304ce827f1c9dab938893e8
Author: AlexYue <yj...@gmail.com>
AuthorDate: Tue Jul 9 09:46:24 2024 +0800
[fix](Azure) Enhance the glob list's logic for azure file system in FE (#37490)
Previously in fe, for files like
`s3://qa-build/regression/tpcds/sf100_split/catalog_sales.dat.*.gz` it
can not work.
---
.../org/apache/doris/fs/obj/AzureObjStorage.java | 62 +++++++++++++++++-----
1 file changed, 48 insertions(+), 14 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java
index 358b66b44b2..73d9444cc8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/AzureObjStorage.java
@@ -299,47 +299,81 @@ public class AzureObjStorage implements ObjStorage<BlobServiceClient> {
return String.format("s3://%s/%s", bucket, fileName);
}
+ public static String getLongestPrefix(String globPattern) {
+ int length = globPattern.length();
+ int earliestSpecialCharIndex = length;
+
+ char[] specialChars = {'*', '?', '[', '{', '\\'};
+
+ for (char specialChar : specialChars) {
+ int index = globPattern.indexOf(specialChar);
+ if (index != -1 && index < earliestSpecialCharIndex) {
+ earliestSpecialCharIndex = index;
+ }
+ }
+
+ return globPattern.substring(0, earliestSpecialCharIndex);
+ }
+
public Status globList(String remotePath, List<RemoteFile> result, boolean fileNameOnly) {
+ long roundCnt = 0;
+ long elementCnt = 0;
+ long matchCnt = 0;
+ long startTime = System.nanoTime();
+ Status st = Status.OK;
try {
S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri);
String globPath = uri.getKey();
+ String bucket = uri.getBucket();
LOG.info("try to glob list for azure, remote path {}, orig {}", globPath, remotePath);
- BlobContainerClient client = getClient().getBlobContainerClient(uri.getBucket());
+ BlobContainerClient client = getClient().getBlobContainerClient(bucket);
java.nio.file.Path pathPattern = Paths.get(globPath);
LOG.info("path pattern {}", pathPattern.toString());
PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + pathPattern.toString());
- ListBlobsOptions options = new ListBlobsOptions().setPrefix(globPath);
+ String listPrefix = getLongestPrefix(globPath);
+ LOG.info("azure glob list prefix is {}", listPrefix);
+ ListBlobsOptions options = new ListBlobsOptions().setPrefix(listPrefix);
String newContinuationToken = null;
do {
+ roundCnt++;
PagedIterable<BlobItem> pagedBlobs = client.listBlobs(options, newContinuationToken, null);
PagedResponse<BlobItem> pagedResponse = pagedBlobs.iterableByPage().iterator().next();
for (BlobItem blobItem : pagedResponse.getElements()) {
+ elementCnt++;
java.nio.file.Path blobPath = Paths.get(blobItem.getName());
- if (matcher.matches(blobPath)) {
- RemoteFile remoteFile = new RemoteFile(
- fileNameOnly ? blobPath.getFileName().toString() : constructS3Path(blobPath.toString(),
- uri.getBucket()),
- !blobItem.isPrefix(),
- blobItem.isPrefix() ? -1 : blobItem.getProperties().getContentLength(),
- blobItem.getProperties().getContentLength(),
- blobItem.getProperties().getLastModified().getSecond());
- result.add(remoteFile);
+ if (!matcher.matches(blobPath)) {
+ continue;
}
+ matchCnt++;
+ RemoteFile remoteFile = new RemoteFile(
+ fileNameOnly ? blobPath.getFileName().toString() : constructS3Path(blobPath.toString(),
+ uri.getBucket()),
+ !blobItem.isPrefix(),
+ blobItem.isPrefix() ? -1 : blobItem.getProperties().getContentLength(),
+ blobItem.getProperties().getContentLength(),
+ blobItem.getProperties().getLastModified().getSecond());
+ result.add(remoteFile);
}
newContinuationToken = pagedResponse.getContinuationToken();
} while (newContinuationToken != null);
} catch (BlobStorageException e) {
LOG.warn("glob file " + remotePath + " failed because azure error: " + e.getMessage());
- return new Status(Status.ErrCode.COMMON_ERROR, "glob file " + remotePath
+ st = new Status(Status.ErrCode.COMMON_ERROR, "glob file " + remotePath
+ " failed because azure error: " + e.getMessage());
} catch (Exception e) {
LOG.warn("errors while glob file " + remotePath, e);
- return new Status(Status.ErrCode.COMMON_ERROR, "errors while glob file " + remotePath + e.getMessage());
+ st = new Status(Status.ErrCode.COMMON_ERROR, "errors while glob file " + remotePath + e.getMessage());
+ } finally {
+ long endTime = System.nanoTime();
+ long duration = endTime - startTime;
+ LOG.info("process {} elements under prefix {} for {} round, match {} elements, take {} micro second",
+ remotePath, elementCnt, matchCnt, roundCnt,
+ duration / 1000);
}
- return Status.OK;
+ return st;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org