You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/05/18 05:29:00 UTC
[incubator-doris] branch revert-3613-revert-3523-fix_codestyle
created (now 36947fd)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch revert-3613-revert-3523-fix_codestyle
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git.
at 36947fd Revert "Revert "Refactor some redunant code && Replace some UT by UtFrameUtils" (#3613)"
This branch includes the following new commits:
new 36947fd Revert "Revert "Refactor some redunant code && Replace some UT by UtFrameUtils" (#3613)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/01: Revert "Revert "Refactor some redunant
code && Replace some UT by UtFrameUtils" (#3613)"
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch revert-3613-revert-3523-fix_codestyle
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 36947fd076ca38da7ca4cc0a09ce9275372e960c
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Mon May 18 13:28:53 2020 +0800
Revert "Revert "Refactor some redunant code && Replace some UT by UtFrameUtils" (#3613)"
This reverts commit 903592d82bf5f73e8856763bc2c36a1793780b10.
---
.../main/java/org/apache/doris/alter/AlterJob.java | 18 +-
.../java/org/apache/doris/backup/RestoreJob.java | 175 +++---
.../java/org/apache/doris/catalog/Database.java | 2 +-
.../apache/doris/catalog/MaterializedIndex.java | 8 +-
.../java/org/apache/doris/catalog/OlapTable.java | 6 +-
.../java/org/apache/doris/catalog/Partition.java | 7 +-
.../org/apache/doris/catalog/PartitionInfo.java | 8 +-
.../java/org/apache/doris/catalog/Replica.java | 1 +
.../main/java/org/apache/doris/catalog/Tablet.java | 19 +-
.../java/org/apache/doris/catalog/TabletMeta.java | 12 +-
.../apache/doris/clone/BackendLoadStatistic.java | 25 +-
.../apache/doris/clone/RootPathLoadStatistic.java | 8 +-
.../org/apache/doris/clone/TabletScheduler.java | 89 ++-
.../common/proc/ClusterLoadStatisticProcDir.java | 6 +-
.../org/apache/doris/system/BackendHbResponse.java | 1 +
.../org/apache/doris/system/BrokerHbResponse.java | 1 +
.../doris/catalog/DynamicPartitionTableTest.java | 684 ++++++++-------------
.../apache/doris/common/PropertyAnalyzerTest.java | 10 +-
18 files changed, 426 insertions(+), 654 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJob.java b/fe/src/main/java/org/apache/doris/alter/AlterJob.java
index ab927cc..7b10fd5 100644
--- a/fe/src/main/java/org/apache/doris/alter/AlterJob.java
+++ b/fe/src/main/java/org/apache/doris/alter/AlterJob.java
@@ -189,18 +189,14 @@ public abstract class AlterJob implements Writable {
return false;
} else if (!backend.isAlive()) {
long currentTime = System.currentTimeMillis();
- if (backend.getLastUpdateMs() > 0
- && currentTime - backend.getLastUpdateMs() > Config.max_backend_down_time_second * 1000) {
- // this backend is done for a long time and not restart automatically.
- // we consider it as dead
- return false;
- }
- return true;
- } else if (backend.isDecommissioned()) {
- return false;
+ // If this backend is done for a long time and not restart automatically.
+ // we consider it as dead and return false.
+ return backend.getLastUpdateMs() <= 0
+ || currentTime - backend.getLastUpdateMs() <= Config.max_backend_down_time_second * 1000;
+ } else {
+ return !backend.isDecommissioned();
}
-
- return true;
+
}
public static AlterJob read(DataInput in) throws IOException {
diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
index ef576b4..335ac17 100644
--- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -172,11 +172,7 @@ public class RestoreJob extends AbstractJob {
}
public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) {
- Preconditions.checkState(task.getJobId() == jobId);
- Preconditions.checkState(task.getDbId() == dbId);
-
- if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
- taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ if (checkTaskStatus(task, task.getJobId(), request)) {
return false;
}
@@ -198,19 +194,15 @@ public class RestoreJob extends AbstractJob {
taskErrMsg.remove(task.getSignature());
Preconditions.checkState(task.getTabletId() == removedTabletId, removedTabletId);
LOG.debug("get finished snapshot info: {}, unfinished tasks num: {}, remove result: {}. {}",
- info, unfinishedSignatureToId.size(), this);
+ info, unfinishedSignatureToId.size(), this, removedTabletId);
return true;
}
-
return false;
}
- public synchronized boolean finishTabletDownloadTask(DownloadTask task, TFinishTaskRequest request) {
- Preconditions.checkState(task.getJobId() == jobId);
- Preconditions.checkState(task.getDbId() == dbId);
- if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
- taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ public synchronized boolean finishTabletDownloadTask(DownloadTask task, TFinishTaskRequest request) {
+ if (checkTaskStatus(task, task.getJobId(), request)) {
return false;
}
@@ -237,11 +229,7 @@ public class RestoreJob extends AbstractJob {
}
public synchronized boolean finishDirMoveTask(DirMoveTask task, TFinishTaskRequest request) {
- Preconditions.checkState(task.getJobId() == jobId);
- Preconditions.checkState(task.getDbId() == dbId);
-
- if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
- taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ if (checkTaskStatus(task, task.getJobId(), request)) {
return false;
}
@@ -256,6 +244,17 @@ public class RestoreJob extends AbstractJob {
return true;
}
+ private boolean checkTaskStatus(AgentTask task, long jobId, TFinishTaskRequest request) {
+ Preconditions.checkState(jobId == this.jobId);
+ Preconditions.checkState(dbId == task.getDbId());
+
+ if (request.getTask_status().getStatus_code() != TStatusCode.OK) {
+ taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs()));
+ return true;
+ }
+ return false;
+ }
+
@Override
public synchronized void replayRun() {
LOG.info("replay run restore job: {}", this);
@@ -342,7 +341,7 @@ public class RestoreJob extends AbstractJob {
}
}
- /*
+ /**
* return true if some restored objs have been dropped.
*/
private void checkIfNeedCancel() {
@@ -353,6 +352,7 @@ public class RestoreJob extends AbstractJob {
Database db = catalog.getDb(dbId);
if (db == null) {
status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " has been dropped");
+ return;
}
db.readLock();
@@ -381,7 +381,7 @@ public class RestoreJob extends AbstractJob {
}
}
- /*
+ /**
* Restore rules as follow:
* A. Table already exist
* A1. Partition already exist, generate file mapping
@@ -505,19 +505,9 @@ public class RestoreJob extends AbstractJob {
Range<PartitionKey> remoteRange = remoteRangePartInfo.getRange(backupPartInfo.id);
if (localRange.equals(remoteRange)) {
// Same partition, same range
- if (localRangePartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) {
- status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name
- + " in table " + localTbl.getName()
- + " has different replication num '"
- + localRangePartInfo.getReplicationNum(localPartition.getId())
- + "' with partition in repository, which is " + restoreReplicationNum);
+ if (genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl, backupPartInfo, tblInfo)) {
return;
}
- genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo,
- true /* overwrite when commit */);
- restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(),
- Pair.create(backupPartInfo.version,
- backupPartInfo.versionHash));
} else {
// Same partition name, different range
status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name
@@ -527,21 +517,9 @@ public class RestoreJob extends AbstractJob {
}
} else {
// If this is a single partitioned table.
- if (localPartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) {
- status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name
- + " in table " + localTbl.getName()
- + " has different replication num '"
- + localPartInfo.getReplicationNum(localPartition.getId())
- + "' with partition in repository, which is " + restoreReplicationNum);
+ if (genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl, backupPartInfo, tblInfo)) {
return;
}
-
- // No need to check range, just generate file mapping
- genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo,
- true /* overwrite when commit */);
- restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(),
- Pair.create(backupPartInfo.version,
- backupPartInfo.versionHash));
}
} else {
// partitions does not exist
@@ -616,65 +594,20 @@ public class RestoreJob extends AbstractJob {
BackupPartitionInfo backupPartitionInfo
= jobInfo.getTableInfo(entry.first).getPartInfo(restorePart.getName());
- Set<String> bfColumns = localTbl.getCopiedBfColumns();
- double bfFpp = localTbl.getBfFpp();
- for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
- MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId());
- TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(),
- restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD);
- for (Tablet restoreTablet : restoredIdx.getTablets()) {
- Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
- for (Replica restoreReplica : restoreTablet.getReplicas()) {
- Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica);
- CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendId(), dbId,
- localTbl.getId(), restorePart.getId(), restoredIdx.getId(),
- restoreTablet.getId(), indexMeta.getShortKeyColumnCount(),
- indexMeta.getSchemaHash(), restoreReplica.getVersion(),
- restoreReplica.getVersionHash(), indexMeta.getKeysType(), TStorageType.COLUMN,
- TStorageMedium.HDD /* all restored replicas will be saved to HDD */,
- indexMeta.getSchema(), bfColumns, bfFpp, null,
- localTbl.getCopiedIndexes(),
- localTbl.isInMemory());
- task.setInRestoreMode(true);
- batchTask.addTask(task);
- }
- }
- }
+ createReplicas(db, batchTask, localTbl, restorePart);
genFileMapping(localTbl, restorePart, remoteTbl.getId(), backupPartitionInfo,
- allowLoad ? false : true /* if allow load, do not overwrite when commit */);
+ !allowLoad /* if allow load, do not overwrite when commit */);
}
// generate create replica task for all restored tables
for (OlapTable restoreTbl : restoredTbls) {
for (Partition restorePart : restoreTbl.getPartitions()) {
- Set<String> bfColumns = restoreTbl.getCopiedBfColumns();
- double bfFpp = restoreTbl.getBfFpp();
- for (MaterializedIndex index : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
- MaterializedIndexMeta indexMeta = restoreTbl.getIndexMetaByIndexId(index.getId());
- TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), restorePart.getId(),
- index.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD);
- for (Tablet tablet : index.getTablets()) {
- Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta);
- for (Replica replica : tablet.getReplicas()) {
- Catalog.getCurrentInvertedIndex().addReplica(tablet.getId(), replica);
- CreateReplicaTask task = new CreateReplicaTask(replica.getBackendId(), dbId,
- restoreTbl.getId(), restorePart.getId(), index.getId(), tablet.getId(),
- indexMeta.getShortKeyColumnCount(), indexMeta.getSchemaHash(),
- replica.getVersion(), replica.getVersionHash(),
- indexMeta.getKeysType(), TStorageType.COLUMN, TStorageMedium.HDD,
- indexMeta.getSchema(), bfColumns, bfFpp, null,
- restoreTbl.getCopiedIndexes(),
- restoreTbl.isInMemory());
- task.setInRestoreMode(true);
- batchTask.addTask(task);
- }
- }
- }
+ createReplicas(db, batchTask, restoreTbl, restorePart);
BackupTableInfo backupTableInfo = jobInfo.getTableInfo(restoreTbl.getName());
genFileMapping(restoreTbl, restorePart, backupTableInfo.id,
backupTableInfo.getPartInfo(restorePart.getName()),
- allowLoad ? false : true /* if allow load, do not overwrite when commit */);
+ !allowLoad /* if allow load, do not overwrite when commit */);
}
// set restored table's new name after all 'genFileMapping'
restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName()));
@@ -689,7 +622,7 @@ public class RestoreJob extends AbstractJob {
if (batchTask.getTaskNum() > 0) {
MarkedCountDownLatch<Long, Long> latch = new MarkedCountDownLatch<Long, Long>(batchTask.getTaskNum());
for (AgentTask task : batchTask.getAllTasks()) {
- latch.addMark(((CreateReplicaTask) task).getBackendId(), ((CreateReplicaTask) task).getTabletId());
+ latch.addMark(task.getBackendId(), task.getTabletId());
((CreateReplicaTask) task).setLatch(latch);
AgentTaskQueue.addTask(task);
}
@@ -806,7 +739,54 @@ public class RestoreJob extends AbstractJob {
// No log here, PENDING state restore job will redo this method
LOG.info("finished to prepare meta and send snapshot tasks, num: {}. {}",
batchTask.getTaskNum(), this);
- return;
+ }
+
+ private boolean genFileMappingWhenBackupReplicasEqual(PartitionInfo localPartInfo, Partition localPartition, Table localTbl,
+ BackupPartitionInfo backupPartInfo, BackupTableInfo tblInfo) {
+ if (localPartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) {
+ status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name
+ + " in table " + localTbl.getName()
+ + " has different replication num '"
+ + localPartInfo.getReplicationNum(localPartition.getId())
+ + "' with partition in repository, which is " + restoreReplicationNum);
+ return true;
+ }
+
+ // No need to check range, just generate file mapping
+ OlapTable localOlapTbl = (OlapTable) localTbl;
+ genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo,
+ true /* overwrite when commit */);
+ restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(),
+ Pair.create(backupPartInfo.version,
+ backupPartInfo.versionHash));
+ return false;
+ }
+
+ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable localTbl, Partition restorePart) {
+ Set<String> bfColumns = localTbl.getCopiedBfColumns();
+ double bfFpp = localTbl.getBfFpp();
+ for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId());
+ TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(),
+ restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD);
+ for (Tablet restoreTablet : restoredIdx.getTablets()) {
+ Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
+ for (Replica restoreReplica : restoreTablet.getReplicas()) {
+ Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica);
+ CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendId(), dbId,
+ localTbl.getId(), restorePart.getId(), restoredIdx.getId(),
+ restoreTablet.getId(), indexMeta.getShortKeyColumnCount(),
+ indexMeta.getSchemaHash(), restoreReplica.getVersion(),
+ restoreReplica.getVersionHash(), indexMeta.getKeysType(), TStorageType.COLUMN,
+ TStorageMedium.HDD /* all restored replicas will be saved to HDD */,
+ indexMeta.getSchema(), bfColumns, bfFpp, null,
+ localTbl.getCopiedIndexes(),
+ localTbl.isInMemory());
+ task.setInRestoreMode(true);
+ batchTask.addTask(task);
+ }
+ }
+ }
}
// reset remote partition.
@@ -825,11 +805,11 @@ public class RestoreJob extends AbstractJob {
// indexes
Map<String, Long> localIdxNameToId = localTbl.getIndexNameToId();
- for (String localidxName : localIdxNameToId.keySet()) {
+ for (String localIdxName : localIdxNameToId.keySet()) {
// set ids of indexes in remote partition to the local index ids
- long remoteIdxId = remoteTbl.getIndexIdByName(localidxName);
+ long remoteIdxId = remoteTbl.getIndexIdByName(localIdxName);
MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId);
- long localIdxId = localIdxNameToId.get(localidxName);
+ long localIdxId = localIdxNameToId.get(localIdxName);
remoteIdx.setIdForRestore(localIdxId);
if (localIdxId != localTbl.getBaseIndexId()) {
// not base table, reset
@@ -1510,6 +1490,7 @@ public class RestoreJob extends AbstractJob {
}
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/src/main/java/org/apache/doris/catalog/Database.java b/fe/src/main/java/org/apache/doris/catalog/Database.java
index fa53e14..0d46c7a 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Database.java
@@ -113,7 +113,7 @@ public class Database extends MetaObject implements Writable {
}
this.rwLock = new ReentrantReadWriteLock(true);
this.idToTable = new ConcurrentHashMap<>();
- this.nameToTable = new HashMap<String, Table>();
+ this.nameToTable = new HashMap<>();
this.dataQuotaBytes = FeConstants.default_db_data_quota_bytes;
this.replicaQuotaSize = FeConstants.default_db_replica_quota_size;
this.dbState = DbState.NORMAL;
diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
index 6f8abd9..a921758 100644
--- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
+++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -76,8 +76,8 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
public MaterializedIndex() {
this.state = IndexState.NORMAL;
- this.idToTablets = new HashMap<Long, Tablet>();
- this.tablets = new ArrayList<Tablet>();
+ this.idToTablets = new HashMap<>();
+ this.tablets = new ArrayList<>();
}
public MaterializedIndex(long id, IndexState state) {
@@ -88,8 +88,8 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
this.state = IndexState.NORMAL;
}
- this.idToTablets = new HashMap<Long, Tablet>();
- this.tablets = new ArrayList<Tablet>();
+ this.idToTablets = new HashMap<>();
+ this.tablets = new ArrayList<>();
this.rowCount = 0;
diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
index fe252dd..f307a5f 100644
--- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -643,6 +643,7 @@ public class OlapTable extends Table {
*/
// get partition by name, not including temp partitions
+ @Override
public Partition getPartition(String partitionName) {
return getPartition(partitionName, false);
}
@@ -732,6 +733,7 @@ public class OlapTable extends Table {
return false;
}
+ @Override
public TTableDescriptor toThrift() {
TOlapTable tOlapTable = new TOlapTable(getName());
TTableDescriptor tTableDescriptor = new TTableDescriptor(id, TTableType.OLAP_TABLE,
@@ -930,6 +932,7 @@ public class OlapTable extends Table {
tempPartitions.write(out);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -947,7 +950,7 @@ public class OlapTable extends Table {
if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_75) {
// schema
int colCount = in.readInt();
- List<Column> schema = new LinkedList<Column>();
+ List<Column> schema = new LinkedList<>();
for (int j = 0; j < colCount; j++) {
Column column = Column.read(in);
schema.add(column);
@@ -1073,6 +1076,7 @@ public class OlapTable extends Table {
rebuildFullSchema();
}
+ @Override
public boolean equals(Table table) {
if (this == table) {
return true;
diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java
index 447555a..0730bed 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Partition.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java
@@ -67,13 +67,13 @@ public class Partition extends MetaObject implements Writable {
private PartitionState state;
@SerializedName(value = "baseIndex")
private MaterializedIndex baseIndex;
- /*
+ /**
* Visible rollup indexes are indexes which are visible to user.
* User can do query on them, show them in related 'show' stmt.
*/
@SerializedName(value = "idToVisibleRollupIndex")
private Map<Long, MaterializedIndex> idToVisibleRollupIndex = Maps.newHashMap();
- /*
+ /**
* Shadow indexes are indexes which are not visible to user.
* Query will not run on these shadow indexes, and user can not see them neither.
* But load process will load data into these shadow indexes.
@@ -81,7 +81,7 @@ public class Partition extends MetaObject implements Writable {
@SerializedName(value = "idToShadowIndex")
private Map<Long, MaterializedIndex> idToShadowIndex = Maps.newHashMap();
- /*
+ /**
* committed version(hash): after txn is committed, set committed version(hash)
* visible version(hash): after txn is published, set visible version
* next version(hash): next version is set after finished committing, it should equals to committed version + 1
@@ -354,6 +354,7 @@ public class Partition extends MetaObject implements Writable {
distributionInfo.write(out);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index bde4815..ee7d6f2 100644
--- a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -51,15 +51,15 @@ public class PartitionInfo implements Writable {
protected Map<Long, Boolean> idToInMemory;
public PartitionInfo() {
- this.idToDataProperty = new HashMap<Long, DataProperty>();
- this.idToReplicationNum = new HashMap<Long, Short>();
+ this.idToDataProperty = new HashMap<>();
+ this.idToReplicationNum = new HashMap<>();
this.idToInMemory = new HashMap<>();
}
public PartitionInfo(PartitionType type) {
this.type = type;
- this.idToDataProperty = new HashMap<Long, DataProperty>();
- this.idToReplicationNum = new HashMap<Long, Short>();
+ this.idToDataProperty = new HashMap<>();
+ this.idToReplicationNum = new HashMap<>();
this.idToInMemory = new HashMap<>();
}
diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java
index ad76292..c9cab33 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java
@@ -505,6 +505,7 @@ public class Replica implements Writable {
return replica;
}
+ @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/src/main/java/org/apache/doris/catalog/Tablet.java
index 1b1cb35..d8eb691 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -39,7 +39,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -80,18 +79,18 @@ public class Tablet extends MetaObject implements Writable {
private long lastStatusCheckTime = -1;
public Tablet() {
- this(0L, new ArrayList<Replica>());
+ this(0L, new ArrayList<>());
}
public Tablet(long tabletId) {
- this(tabletId, new ArrayList<Replica>());
+ this(tabletId, new ArrayList<>());
}
public Tablet(long tabletId, List<Replica> replicas) {
this.id = tabletId;
this.replicas = replicas;
if (this.replicas == null) {
- this.replicas = new ArrayList<Replica>();
+ this.replicas = new ArrayList<>();
}
checkedVersion = -1L;
@@ -300,13 +299,15 @@ public class Tablet extends MetaObject implements Writable {
public static void sortReplicaByVersionDesc(List<Replica> replicas) {
// sort replicas by version. higher version in the tops
- Collections.sort(replicas, Replica.VERSION_DESC_COMPARATOR);
+ replicas.sort(Replica.VERSION_DESC_COMPARATOR);
}
+ @Override
public String toString() {
return "tabletId=" + this.id;
}
+ @Override
public void write(DataOutput out) throws IOException {
super.write(out);
@@ -321,6 +322,7 @@ public class Tablet extends MetaObject implements Writable {
out.writeLong(checkedVersionHash);
out.writeBoolean(isConsistent);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -346,6 +348,7 @@ public class Tablet extends MetaObject implements Writable {
return tablet;
}
+ @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
@@ -392,7 +395,7 @@ public class Tablet extends MetaObject implements Writable {
return dataSize;
}
- /*
+ /**
* A replica is healthy only if
* 1. the backend is available
* 2. replica version is caught up, and last failed version is -1
@@ -502,7 +505,7 @@ public class Tablet extends MetaObject implements Writable {
return Pair.create(TabletStatus.HEALTHY, TabletSchedCtx.Priority.NORMAL);
}
- /*
+ /**
* Check colocate table's tablet health
* 1. Mismatch:
* backends set: 1,2,3
@@ -551,7 +554,7 @@ public class Tablet extends MetaObject implements Writable {
return TabletStatus.HEALTHY;
}
- /*
+ /**
* check if this tablet is ready to be repaired, based on priority.
* VERY_HIGH: repair immediately
* HIGH: delay Config.tablet_repair_delay_factor_second * 1;
diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
index 4cb2b44..0c695ca 100644
--- a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
+++ b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java
@@ -137,12 +137,12 @@ public class TabletMeta {
lock.readLock().lock();
try {
StringBuilder sb = new StringBuilder();
- sb.append("dbId=" + dbId);
- sb.append(" tableId=" + tableId);
- sb.append(" partitionId=" + partitionId);
- sb.append(" indexId=" + indexId);
- sb.append(" oldSchemaHash=" + oldSchemaHash);
- sb.append(" newSchemaHash=" + newSchemaHash);
+ sb.append("dbId=").append(dbId);
+ sb.append(" tableId=").append(tableId);
+ sb.append(" partitionId=").append(partitionId);
+ sb.append(" indexId=").append(indexId);
+ sb.append(" oldSchemaHash=").append(oldSchemaHash);
+ sb.append(" newSchemaHash=").append(newSchemaHash);
return sb.toString();
} finally {
diff --git a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
index 5479a16..5dd6e60 100644
--- a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
+++ b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
@@ -55,13 +55,7 @@ public class BackendLoadStatistic {
public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) {
double score1 = o1.getLoadScore(medium);
double score2 = o2.getLoadScore(medium);
- if (score1 > score2) {
- return 1;
- } else if (score1 == score2) {
- return 0;
- } else {
- return -1;
- }
+ return Double.compare(score1, score2);
}
}
@@ -70,14 +64,7 @@ public class BackendLoadStatistic {
public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) {
Double score1 = o1.getMixLoadScore();
Double score2 = o2.getMixLoadScore();
-
- if (score1 > score2) {
- return 1;
- } else if (score1 == score2) {
- return 0;
- } else {
- return -1;
- }
+ return score1.compareTo(score2);
}
}
@@ -180,11 +167,7 @@ public class BackendLoadStatistic {
throw new LoadBalanceException("backend " + beId + " does not exist");
}
- if (!be.isAvailable()) {
- isAvailable = false;
- } else {
- isAvailable = true;
- }
+ isAvailable = be.isAvailable();
ImmutableMap<String, DiskInfo> disks = be.getDisks();
for (DiskInfo diskInfo : disks.values()) {
@@ -333,7 +316,7 @@ public class BackendLoadStatistic {
return false;
}
- /*
+ /**
* Classify the paths into 'low', 'mid' and 'high',
* and skip offline path, and path with different storage medium
*/
diff --git a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
index 2b5fee9..3aeb406 100644
--- a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
+++ b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
@@ -115,13 +115,7 @@ public class RootPathLoadStatistic implements Comparable<RootPathLoadStatistic>
public int compareTo(RootPathLoadStatistic o) {
double myPercent = getUsedPercent();
double otherPercent = o.getUsedPercent();
- if (myPercent < otherPercent) {
- return -1;
- } else if (myPercent > otherPercent) {
- return 1;
- } else {
- return 0;
- }
+ return Double.compare(myPercent, otherPercent);
}
@Override
diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 036fe6e..a127255 100644
--- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.ColocateTableIndex.GroupId;
import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.DiskInfo.DiskState;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
@@ -60,7 +61,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
@@ -68,7 +68,7 @@ import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
-/*
+/**
* TabletScheduler saved the tablets produced by TabletChecker and try to schedule them.
* It also try to balance the cluster load.
*
@@ -170,7 +170,7 @@ public class TabletScheduler extends MasterDaemon {
if (backends.containsKey(beId)) {
List<Long> pathHashes = backends.get(beId).getDisks().values().stream()
.filter(v -> v.getState()==DiskState.ONLINE)
- .map(v -> v.getPathHash()).collect(Collectors.toList());
+ .map(DiskInfo::getPathHash).collect(Collectors.toList());
backendsWorkingSlots.get(beId).updatePaths(pathHashes);
} else {
deletedBeIds.add(beId);
@@ -186,7 +186,7 @@ public class TabletScheduler extends MasterDaemon {
// add new backends
for (Backend be : backends.values()) {
if (!backendsWorkingSlots.containsKey(be.getId())) {
- List<Long> pathHashes = be.getDisks().values().stream().map(v -> v.getPathHash()).collect(Collectors.toList());
+ List<Long> pathHashes = be.getDisks().values().stream().map(DiskInfo::getPathHash).collect(Collectors.toList());
PathSlot slot = new PathSlot(pathHashes, Config.schedule_slot_num_per_path);
backendsWorkingSlots.put(be.getId(), slot);
LOG.info("add new backend {} with slots num: {}", be.getId(), be.getDisks().size());
@@ -200,7 +200,7 @@ public class TabletScheduler extends MasterDaemon {
return backendsWorkingSlots;
}
- /*
+ /**
* add a ready-to-be-scheduled tablet to pendingTablets, if it has not being added before.
* if force is true, do not check if tablet is already added before.
*/
@@ -227,7 +227,7 @@ public class TabletScheduler extends MasterDaemon {
return allTabletIds.contains(tabletId);
}
- /*
+ /**
* Iterate current tablets, change their priority to VERY_HIGH if necessary.
*/
public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long tblId, List<Long> partitionIds) {
@@ -242,10 +242,10 @@ public class TabletScheduler extends MasterDaemon {
pendingTablets = newPendingTablets;
}
- /*
+ /**
* TabletScheduler will run as a daemon thread at a very short interval(default 5 sec)
- * Firstly, it will try to update cluster load statistic and check if priority need to be adjuested.
- * Than, it will schedule the tablets in pendingTablets.
+ * Firstly, it will try to update cluster load statistic and check if priority need to be adjusted.
+ * Then, it will schedule the tablets in pendingTablets.
* Thirdly, it will check the current running tasks.
* Finally, it try to balance the cluster if possible.
*
@@ -287,7 +287,7 @@ public class TabletScheduler extends MasterDaemon {
lastStatUpdateTime = System.currentTimeMillis();
}
- /*
+ /**
* Here is the only place we update the cluster load statistic info.
* We will not update this info dynamically along with the clone job's running.
* Although it will cause a little bit inaccurate, but is within a controllable range,
@@ -311,13 +311,13 @@ public class TabletScheduler extends MasterDaemon {
return statisticMap;
}
- /*
+ /**
* adjust priorities of all tablet infos
*/
private synchronized void adjustPriorities() {
int size = pendingTablets.size();
int changedNum = 0;
- TabletSchedCtx tabletCtx = null;
+ TabletSchedCtx tabletCtx;
for (int i = 0; i < size; i++) {
tabletCtx = pendingTablets.poll();
if (tabletCtx == null) {
@@ -333,7 +333,7 @@ public class TabletScheduler extends MasterDaemon {
LOG.info("adjust priority for all tablets. changed: {}, total: {}", changedNum, size);
}
- /*
+ /**
* get at most BATCH_NUM tablets from queue, and try to schedule them.
* After handle, the tablet info should be
* 1. in runningTablets with state RUNNING, if being scheduled success.
@@ -423,7 +423,7 @@ public class TabletScheduler extends MasterDaemon {
runningTablets.put(tabletCtx.getTabletId(), tabletCtx);
}
- /*
+ /**
* we take the tablet out of the runningTablets and than handle it,
* avoid other threads see it.
* Whoever takes this tablet, make sure to put it to the schedHistory or back to runningTablets.
@@ -432,7 +432,7 @@ public class TabletScheduler extends MasterDaemon {
return runningTablets.remove(tabletId);
}
- /*
+ /**
* Try to schedule a single tablet.
*/
private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throws SchedException {
@@ -448,7 +448,7 @@ public class TabletScheduler extends MasterDaemon {
throw new SchedException(Status.UNRECOVERABLE, "db does not exist");
}
- Pair<TabletStatus, TabletSchedCtx.Priority> statusPair = null;
+ Pair<TabletStatus, TabletSchedCtx.Priority> statusPair;
db.writeLock();
try {
OlapTable tbl = (OlapTable) db.getTable(tabletCtx.getTblId());
@@ -586,7 +586,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /*
+ /**
* Replica is missing, which means there is no enough alive replicas.
* So we need to find a destination backend to clone a new replica as possible as we can.
* 1. find an available path in a backend as destination:
@@ -615,7 +615,7 @@ public class TabletScheduler extends MasterDaemon {
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
}
- /*
+ /**
* Replica version is incomplete, which means this replica is missing some version,
* and need to be cloned from a healthy replica, in-place.
*
@@ -649,7 +649,7 @@ public class TabletScheduler extends MasterDaemon {
handleReplicaMissing(tabletCtx, batchTask);
}
- /*
+ /**
* replica is redundant, which means there are more replicas than we expected, which need to be dropped.
* we just drop one redundant replica at a time, for safety reason.
* choosing a replica to drop base on following priority:
@@ -843,7 +843,7 @@ public class TabletScheduler extends MasterDaemon {
return false;
}
- /*
+ /**
* Just delete replica which does not located in colocate backends set.
* return true if delete one replica, otherwise, return false.
*/
@@ -920,7 +920,7 @@ public class TabletScheduler extends MasterDaemon {
LOG.info("send delete replica task for tablet {} in backend {}", tabletId, backendId);
}
- /*
+ /**
* Cluster migration, which means the tablet has enough healthy replicas,
* but some replicas are not in right cluster.
* It is just same as 'replica missing'.
@@ -933,7 +933,7 @@ public class TabletScheduler extends MasterDaemon {
handleReplicaMissing(tabletCtx, batchTask);
}
- /*
+ /**
* Replicas of colocate table's tablet does not locate on right backends set.
* backends set: 1,2,3
* tablet replicas: 1,2,5
@@ -960,7 +960,7 @@ public class TabletScheduler extends MasterDaemon {
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
}
- /*
+ /**
* Try to select some alternative tablets for balance. Add them to pendingTablets with priority LOW,
* and waiting to be scheduled.
*/
@@ -984,7 +984,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /*
+ /**
* Try to create a balance task for a tablet.
*/
private void doBalance(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throws SchedException {
@@ -1005,8 +1005,7 @@ public class TabletScheduler extends MasterDaemon {
// get all available paths which this tablet can fit in.
// beStatistics is sorted by mix load score in ascend order, so select from first to last.
List<RootPathLoadStatistic> allFitPaths = Lists.newArrayList();
- for (int i = 0; i < beStatistics.size(); i++) {
- BackendLoadStatistic bes = beStatistics.get(i);
+ for (BackendLoadStatistic bes : beStatistics) {
if (!bes.isAvailable()) {
continue;
}
@@ -1071,7 +1070,7 @@ public class TabletScheduler extends MasterDaemon {
throw new SchedException(Status.SCHEDULE_FAILED, "unable to find dest path which can be fit in");
}
- /*
+ /**
* For some reason, a tablet info failed to be scheduled this time,
* So we dynamically change its priority and add back to queue, waiting for next round.
*/
@@ -1127,7 +1126,7 @@ public class TabletScheduler extends MasterDaemon {
return total;
}
- /*
+ /**
* return true if we want to remove the clone task from AgentTaskQueue
*/
public boolean finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) {
@@ -1174,7 +1173,7 @@ public class TabletScheduler extends MasterDaemon {
return true;
}
- /*
+ /**
* Gather the running statistic of the task.
* It will be evaluated for future strategy.
* This should only be called when the tablet is down with state FINISHED.
@@ -1208,7 +1207,7 @@ public class TabletScheduler extends MasterDaemon {
lastSlotAdjustTime = System.currentTimeMillis();
}
- /*
+ /**
* handle tablets which are running.
* We should finished the task if
* 1. Tablet is already healthy
@@ -1291,7 +1290,7 @@ public class TabletScheduler extends MasterDaemon {
+ runningTablets.values().stream().filter(t -> t.getType() == Type.BALANCE).count();
}
- /*
+ /**
* PathSlot keeps track of slot num per path of a Backend.
* Each path on a Backend has several slot.
* If a path's available slot num become 0, no task should be assigned to this path.
@@ -1309,13 +1308,7 @@ public class TabletScheduler extends MasterDaemon {
// update the path
public synchronized void updatePaths(List<Long> paths) {
// delete non exist path
- Iterator<Map.Entry<Long, Slot>> iter = pathSlots.entrySet().iterator();
- while (iter.hasNext()) {
- Map.Entry<Long, Slot> entry = iter.next();
- if (!paths.contains(entry.getKey())) {
- iter.remove();
- }
- }
+ pathSlots.entrySet().removeIf(entry -> !paths.contains(entry.getKey()));
// add new path
for (Long pathHash : paths) {
@@ -1339,7 +1332,7 @@ public class TabletScheduler extends MasterDaemon {
}
}
- /*
+ /**
* Update the statistic of specified path
*/
public synchronized void updateStatistic(long pathHash, long copySize, long copyTimeMs) {
@@ -1351,7 +1344,7 @@ public class TabletScheduler extends MasterDaemon {
slot.totalCopyTimeMs += copyTimeMs;
}
- /*
+ /**
* If the specified 'pathHash' has available slot, decrease the slot number and return this path hash
*/
public synchronized long takeSlot(long pathHash) throws SchedException {
@@ -1400,7 +1393,7 @@ public class TabletScheduler extends MasterDaemon {
return total;
}
- /*
+ /**
* get path whose balance slot num is larger than 0
*/
public synchronized Set<Long> getAvailPathsForBalance() {
@@ -1423,15 +1416,15 @@ public class TabletScheduler extends MasterDaemon {
public synchronized List<List<String>> getSlotInfo(long beId) {
List<List<String>> results = Lists.newArrayList();
- pathSlots.entrySet().stream().forEach(t -> {
- t.getValue().rectify();
+ pathSlots.forEach((key, value) -> {
+ value.rectify();
List<String> result = Lists.newArrayList();
result.add(String.valueOf(beId));
- result.add(String.valueOf(t.getKey()));
- result.add(String.valueOf(t.getValue().available));
- result.add(String.valueOf(t.getValue().total));
- result.add(String.valueOf(t.getValue().balanceSlot));
- result.add(String.valueOf(t.getValue().getAvgRate()));
+ result.add(String.valueOf(key));
+ result.add(String.valueOf(value.available));
+ result.add(String.valueOf(value.total));
+ result.add(String.valueOf(value.balanceSlot));
+ result.add(String.valueOf(value.getAvgRate()));
results.add(result);
});
return results;
diff --git a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
index 47121af..df0c182 100644
--- a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
+++ b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java
@@ -50,11 +50,9 @@ public class ClusterLoadStatisticProcDir implements ProcDirInterface {
statMap = Catalog.getCurrentCatalog().getTabletScheduler().getStatisticMap();
- statMap.values().stream().forEach(t -> {
+ statMap.values().forEach(t -> {
List<List<String>> statistics = t.getClusterStatistic(medium);
- statistics.stream().forEach(v -> {
- result.addRow(v);
- });
+ statistics.forEach(result::addRow);
});
return result;
diff --git a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
index 1559a4a..8b0c39c 100644
--- a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
+++ b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java
@@ -90,6 +90,7 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable {
out.writeInt(brpcPort);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
beId = in.readLong();
diff --git a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
index dda679a..e6ae49b 100644
--- a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
+++ b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java
@@ -81,6 +81,7 @@ public class BrokerHbResponse extends HeartbeatResponse implements Writable {
out.writeInt(port);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
name = Text.readString(in);
diff --git a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index 70ce4c0..b744cf2 100644
--- a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++ b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -17,484 +17,294 @@
package org.apache.doris.catalog;
-import org.apache.doris.analysis.Analyzer;
-import org.apache.doris.analysis.ColumnDef;
+import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
-import org.apache.doris.analysis.HashDistributionDesc;
-import org.apache.doris.analysis.KeysDesc;
-import org.apache.doris.analysis.PartitionKeyDesc;
-import org.apache.doris.analysis.PartitionValue;
-import org.apache.doris.analysis.RangePartitionDesc;
-import org.apache.doris.analysis.SingleRangePartitionDesc;
-import org.apache.doris.analysis.TableName;
-import org.apache.doris.analysis.TypeDef;
import org.apache.doris.common.DdlException;
-import org.apache.doris.common.UserException;
-import org.apache.doris.mysql.privilege.PaloAuth;
-import org.apache.doris.mysql.privilege.PrivPredicate;
-import org.apache.doris.persist.EditLog;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.system.SystemInfoService;
-import org.apache.doris.task.AgentBatchTask;
-import com.google.common.collect.Lists;
-
-import org.junit.Before;
+import org.apache.doris.utframe.UtFrameUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import mockit.Expectations;
-import mockit.Injectable;
-import mockit.Mock;
-import mockit.MockUp;
+import java.util.UUID;
public class DynamicPartitionTableTest {
- private TableName dbTableName;
- private String dbName = "testDb";
- private String tableName = "testTable";
- private String clusterName = "default";
- private List<Long> beIds = Lists.newArrayList();
- private List<String> columnNames = Lists.newArrayList();
- private List<ColumnDef> columnDefs = Lists.newArrayList();
-
- private Catalog catalog = Catalog.getInstance();
- private Database db = new Database();
- private Analyzer analyzer;
+ private static String runningDir = "fe/mocked/DynamicPartitionTableTest/" + UUID.randomUUID().toString() + "/";
- private Map<String, String> properties;
- private List<SingleRangePartitionDesc> singleRangePartitionDescs;
-
- @Injectable
- ConnectContext connectContext;
+ private static ConnectContext connectContext;
@Rule
- public ExpectedException expectedEx = ExpectedException.none();
-
- @Before
- public void setUp() throws Exception {
- dbTableName = new TableName(dbName, tableName);
-
- beIds.add(1L);
- beIds.add(2L);
- beIds.add(3L);
-
- columnNames.add("key1");
- columnNames.add("key2");
- columnNames.add("key3");
-
- columnDefs.add(new ColumnDef("key1", new TypeDef(ScalarType.createType(PrimitiveType.INT))));
- columnDefs.add(new ColumnDef("key2", new TypeDef(ScalarType.createType(PrimitiveType.INT))));
- columnDefs.add(new ColumnDef("key3", new TypeDef(ScalarType.createVarchar(10))));
-
- analyzer = new Analyzer(catalog, connectContext);
-
- properties = new HashMap<>();
- properties.put(DynamicPartitionProperty.ENABLE, "true");
- properties.put(DynamicPartitionProperty.PREFIX, "p");
- properties.put(DynamicPartitionProperty.TIME_UNIT, "day");
- properties.put(DynamicPartitionProperty.START, "-3");
- properties.put(DynamicPartitionProperty.END, "3");
- properties.put(DynamicPartitionProperty.BUCKETS, "30");
+ public ExpectedException expectedException = ExpectedException.none();
- singleRangePartitionDescs = new LinkedList<>();
- singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1",
- new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"))), null));
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ FeConstants.default_scheduler_interval_millisecond = 1000;
+ FeConstants.runningUnitTest = true;
- new MockUp<AgentBatchTask>() {
- @Mock
- void run() {
- return;
- }
- };
+ UtFrameUtils.createMinDorisCluster(runningDir);
- new MockUp<CountDownLatch>() {
- @Mock
- boolean await(long timeout, TimeUnit unit) {
- return true;
- }
- };
-
- new Expectations(analyzer, catalog) {{
- analyzer.getClusterName();
- minTimes = 0;
- result = clusterName;
- }};
-
- dbTableName.analyze(analyzer);
+ // create connect context
+ connectContext = UtFrameUtils.createDefaultCtx();
+ // create database
+ String createDbStmtStr = "create database test;";
+ CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
+ Catalog.getCurrentCatalog().createDb(createDbStmt);
}
- @Test
- public void testNormal(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
-
- catalog.createTable(stmt);
+ @AfterClass
+ public static void TearDown() {
+ UtFrameUtils.cleanDorisFeDir(runningDir);
}
- @Test
- public void testMissPrefix(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.PREFIX);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.prefix properties");
-
- catalog.createTable(stmt);
+ private static void createTable(String sql) throws Exception {
+ CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
}
@Test
- public void testMissTimeUnit(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.TIME_UNIT);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.time_unit properties");
-
- catalog.createTable(stmt);
+ public void testNormal() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
}
@Test
- public void testMissSTART(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.START);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
-
- catalog.createTable(stmt);
+ public void testMissPrefix() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_prefix` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.prefix properties");
+ createTable(createOlapTblStmt);
}
@Test
- public void testMissEnd(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- properties.remove(DynamicPartitionProperty.END);
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.end properties");
-
- catalog.createTable(stmt);
+ public void testMissTimeUnit() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_time_unit` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.time_unit properties");
+ createTable(createOlapTblStmt);
}
@Test
- public void testMissBuckets(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
+ public void testMissStart() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_start` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ }
- properties.remove(DynamicPartitionProperty.BUCKETS);
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
+ @Test
+ public void testMissEnd() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_end` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.end properties");
+ createTable(createOlapTblStmt);
+ }
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Must assign dynamic_partition.buckets properties");
- catalog.createTable(stmt);
+ @Test
+ public void testMissBuckets() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.buckets properties");
+ createTable(createOlapTblStmt);
}
@Test
- public void testNotAllowed(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames), null,
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Only support dynamic partition properties on range partition table");
-
- catalog.createTable(stmt);
+ public void testNotAllowed() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Only support dynamic partition properties on range partition table");
+ createTable(createOlapTblStmt);
}
@Test
- public void testNotAllowedInMultiPartitions(@Injectable SystemInfoService systemInfoService,
- @Injectable PaloAuth paloAuth,
- @Injectable EditLog editLog) throws UserException {
- new Expectations(catalog) {
- {
- catalog.getDb(dbTableName.getDb());
- minTimes = 0;
- result = db;
-
- Catalog.getCurrentSystemInfo();
- minTimes = 0;
- result = systemInfoService;
-
- systemInfoService.checkClusterCapacity(anyString);
- minTimes = 0;
- systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString);
- minTimes = 0;
- result = beIds;
-
- catalog.getAuth();
- minTimes = 0;
- result = paloAuth;
- paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE);
- minTimes = 0;
- result = true;
-
- catalog.getEditLog();
- minTimes = 0;
- result = editLog;
- }
- };
-
- List<SingleRangePartitionDesc> rangePartitionDescs = new LinkedList<>();
- rangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1",
- new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"), new PartitionValue("100"))), null));
-
- CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap",
- new KeysDesc(KeysType.AGG_KEYS, columnNames),
- new RangePartitionDesc(Lists.newArrayList("key1", "key2"), singleRangePartitionDescs),
- new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, "");
- stmt.analyze(analyzer);
-
- expectedEx.expect(DdlException.class);
- expectedEx.expectMessage("Dynamic partition only support single-column range partition");
-
- catalog.createTable(stmt);
+ public void testNotAllowedInMultiPartitions() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1, k2)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\", \"100\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\", \"200\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\", \"300\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Dynamic partition only support single-column range partition");
+ createTable(createOlapTblStmt);
}
}
diff --git a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 81c883e..2f88ecd 100644
--- a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -17,12 +17,15 @@
package org.apache.doris.common;
+import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.collect.Lists;
@@ -122,11 +125,14 @@ public class PropertyAnalyzerTest {
@Test
public void testStorageMedium() throws AnalysisException {
long tomorrowTs = System.currentTimeMillis() / 1000 + 86400;
+ String tomorrowTimeStr = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000);
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD");
- properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000));
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, tomorrowTimeStr);
DataProperty dataProperty = PropertyAnalyzer.analyzeDataProperty(properties, new DataProperty(TStorageMedium.SSD));
- Assert.assertEquals(tomorrowTs, dataProperty.getCooldownTimeMs() / 1000);
+ // avoid UT fail because time zone different
+ DateLiteral dateLiteral = new DateLiteral(tomorrowTimeStr, Type.DATETIME);
+ Assert.assertEquals(dateLiteral.unixTimestamp(TimeUtils.getTimeZone()), dataProperty.getCooldownTimeMs());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org