You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2020/02/06 07:26:44 UTC
[GitHub] [incubator-doris] kangkaisen opened a new pull request #2847: Doris
support in memory olap table
kangkaisen opened a new pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847
For https://github.com/apache/incubator-doris/issues/2846
The doc and schema change support will do in next PR.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r380171460
##########
File path: be/src/olap/tablet_meta.h
##########
@@ -155,7 +155,7 @@ class TabletMeta {
inline const bool in_restore_mode() const;
inline OLAPStatus set_in_restore_mode(bool in_restore_mode);
- inline const TabletSchema& tablet_schema() const;
+ inline TabletSchema& tablet_schema();
Review comment:
OK
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] imay commented on a change in pull request #2847:
Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
imay commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r380131087
##########
File path: fe/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
##########
@@ -18,50 +18,134 @@
package org.apache.doris.task;
import java.util.List;
-import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.tuple.Triple;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.TabletMeta;
+import org.apache.doris.common.MarkedCountDownLatch;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.Status;
+import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TTabletMetaInfo;
+import org.apache.doris.thrift.TTabletMetaType;
import org.apache.doris.thrift.TTaskType;
import org.apache.doris.thrift.TUpdateTabletMetaInfoReq;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.google.common.collect.Lists;
-import com.google.common.collect.SetMultimap;
public class UpdateTabletMetaInfoTask extends AgentTask {
- private static final Logger LOG = LogManager.getLogger(ClearTransactionTask.class);
+ private static final Logger LOG = LogManager.getLogger(UpdateTabletMetaInfoTask.class);
- private SetMultimap<Long, Integer> tabletWithoutPartitionId;
+ // used for synchronous process
+ private MarkedCountDownLatch<Long, Set<Pair<Long, Integer>>> latch;
- public UpdateTabletMetaInfoTask(long backendId, SetMultimap<Long, Integer> tabletWithoutPartitionId) {
- super(null, backendId, TTaskType.UPDATE_TABLET_META_INFO, -1L, -1L, -1L, -1L, -1L, backendId);
- this.tabletWithoutPartitionId = tabletWithoutPartitionId;
+ private Set<Pair<Long, Integer>> tableIdWithSchemaHash;
+ private boolean isInMemory;
+ private TTabletMetaType metaType;
+
+ // <tablet id, tablet schema hash, tablet in memory>
+ private List<Triple<Long, Integer, Boolean>> tabletToInMemory;
+
+ public UpdateTabletMetaInfoTask(long backendId, Set<Pair<Long, Integer>> tableIdWithSchemaHash,
+ TTabletMetaType metaType) {
+ super(null, backendId, TTaskType.UPDATE_TABLET_META_INFO,
+ -1L, -1L, -1L, -1L, -1L, tableIdWithSchemaHash.hashCode());
+ this.tableIdWithSchemaHash = tableIdWithSchemaHash;
+ this.metaType = metaType;
+ }
+
+ public UpdateTabletMetaInfoTask(long backendId,
+ Set<Pair<Long, Integer>> tableIdWithSchemaHash,
+ boolean isInMemory,
+ MarkedCountDownLatch<Long, Set<Pair<Long, Integer>>> latch) {
+ this(backendId, tableIdWithSchemaHash, TTabletMetaType.INMEMORY);
+ this.isInMemory = isInMemory;
+ this.latch = latch;
+ }
+
+ public UpdateTabletMetaInfoTask(long backendId,
+ List<Triple<Long, Integer, Boolean>> tabletToInMemory) {
+ super(null, backendId, TTaskType.UPDATE_TABLET_META_INFO,
+ -1L, -1L, -1L, -1L, -1L, tabletToInMemory.hashCode());
+ this.metaType = TTabletMetaType.INMEMORY;
+ this.tabletToInMemory = tabletToInMemory;
+ }
+
+ public void countDownLatch(long backendId, Set<Pair<Long, Integer>> tablets) {
+ if (this.latch != null) {
+ if (latch.markedCountDown(backendId, tablets)) {
+ LOG.debug("UpdateTabletMetaInfoTask current latch count: {}, backend: {}, tablets:{}",
+ latch.getCount(), backendId, tablets);
+ }
+ }
+ }
+
+ // call this always means one of tasks is failed. count down to zero to finish entire task
+ public void countDownToZero(String errMsg) {
+ if (this.latch != null) {
+ latch.countDownToZero(new Status(TStatusCode.CANCELLED, errMsg));
+ LOG.debug("UpdateTabletMetaInfoTask count down to zero. error msg: {}", errMsg);
+ }
+ }
+
+ public Set<Pair<Long, Integer>> getTablets() {
+ return tableIdWithSchemaHash;
}
public TUpdateTabletMetaInfoReq toThrift() {
TUpdateTabletMetaInfoReq updateTabletMetaInfoReq = new TUpdateTabletMetaInfoReq();
List<TTabletMetaInfo> metaInfos = Lists.newArrayList();
- int tabletEntryNum = 0;
- for (Map.Entry<Long, Integer> entry : tabletWithoutPartitionId.entries()) {
- // add at most 10000 tablet meta during one sync to avoid too large task
- if (tabletEntryNum > 10000) {
+ switch (metaType) {
+ case PARTITIONID: {
+ int tabletEntryNum = 0;
+ for (Pair<Long, Integer> pair : tableIdWithSchemaHash) {
+ // add at most 10000 tablet meta during one sync to avoid too large task
+ if (tabletEntryNum > 10000) {
+ break;
+ }
+ TTabletMetaInfo metaInfo = new TTabletMetaInfo();
+ metaInfo.setTablet_id(pair.first);
+ metaInfo.setSchema_hash(pair.second);
+ TabletMeta tabletMeta = Catalog.getInstance().getTabletInvertedIndex().getTabletMeta(pair.first);
+ if (tabletMeta == null) {
+ LOG.warn("could not find tablet [{}] in meta ignore it", pair.second);
+ continue;
+ }
+ metaInfo.setPartition_id(tabletMeta.getPartitionId());
+ metaInfo.setMeta_type(metaType);
+ metaInfos.add(metaInfo);
+ ++tabletEntryNum;
+ }
break;
}
- TTabletMetaInfo metaInfo = new TTabletMetaInfo();
- metaInfo.setTablet_id(entry.getKey());
- metaInfo.setSchema_hash(entry.getValue());
- TabletMeta tabletMeta = Catalog.getInstance().getTabletInvertedIndex().getTabletMeta(entry.getKey());
- if (tabletMeta == null) {
- LOG.warn("could not find tablet [{}] in meta ignore it", entry.getKey());
- continue;
+ case INMEMORY: {
+ if (latch != null) {
+ // for schema change
+ for (Pair<Long, Integer> pair: tableIdWithSchemaHash) {
+ TTabletMetaInfo metaInfo = new TTabletMetaInfo();
+ metaInfo.setTablet_id(pair.first);
+ metaInfo.setSchema_hash(pair.second);
+ metaInfo.setIs_in_memory(isInMemory);
+ metaInfo.setMeta_type(metaType);
+ metaInfos.add(metaInfo);
+ }
+ } else {
+ // for ReportHandler
+ for(Triple<Long, Integer, Boolean> triple: tabletToInMemory) {
Review comment:
```suggestion
for (Triple<Long, Integer, Boolean> triple: tabletToInMemory) {
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r376175012
##########
File path: fe/src/main/java/org/apache/doris/catalog/OlapTable.java
##########
@@ -1011,17 +1010,17 @@ public void readFields(DataInput in) throws IOException {
tableProperty = TableProperty.read(in);
}
}
+ // inMemory
+ if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_72) {
+ isInMemory = in.readBoolean();
Review comment:
Could we put this `isInMemory` in `tableProperty` to avoid modifying meta version?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377514349
##########
File path: fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
##########
@@ -1405,6 +1420,88 @@ private void sendClearAlterTask(Database db, OlapTable olapTable) {
LOG.info("send clear alter task for table {}, number: {}", olapTable.getName(), batchTask.getTaskNum());
}
+ public void sendUpdateTabletMetaInfoTask(Database db, OlapTable olapTable, boolean isInMemory) throws DdlException {
+ List<Partition> partitions = Lists.newArrayList();
+ db.readLock();
+ try {
+ partitions.addAll(olapTable.getPartitions());
+ } finally {
+ db.readUnlock();
+ }
+ for(Partition partition: partitions) {
+ sendUpdateTabletMetaTaskForPartition(db, olapTable, partition, isInMemory);
+ }
+ }
+
+ public void sendUpdateTabletMetaTaskForPartition(Database db,
Review comment:
OK.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangpinghuang commented on a change in pull
request #2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangpinghuang commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377976915
##########
File path: be/src/olap/page_cache.cpp
##########
@@ -42,11 +42,18 @@ bool StoragePageCache::lookup(const CacheKey& key, PageCacheHandle* handle) {
return true;
}
-void StoragePageCache::insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle) {
+void StoragePageCache::insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle,
+ bool in_memory) {
Review comment:
I think is_durable is a better name?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] imay commented on a change in pull request #2847:
Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
imay commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r380129274
##########
File path: be/src/olap/tablet_meta.h
##########
@@ -155,7 +155,7 @@ class TabletMeta {
inline const bool in_restore_mode() const;
inline OLAPStatus set_in_restore_mode(bool in_restore_mode);
- inline const TabletSchema& tablet_schema() const;
+ inline TabletSchema& tablet_schema();
Review comment:
Prefer `mutable_tablet_schema()` to return a pointer.
Normally, const modification is required when returning a reference
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377466577
##########
File path: fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
##########
@@ -1405,6 +1420,88 @@ private void sendClearAlterTask(Database db, OlapTable olapTable) {
LOG.info("send clear alter task for table {}, number: {}", olapTable.getName(), batchTask.getTaskNum());
}
+ public void sendUpdateTabletMetaInfoTask(Database db, OlapTable olapTable, boolean isInMemory) throws DdlException {
+ List<Partition> partitions = Lists.newArrayList();
+ db.readLock();
+ try {
+ partitions.addAll(olapTable.getPartitions());
+ } finally {
+ db.readUnlock();
+ }
+ for(Partition partition: partitions) {
+ sendUpdateTabletMetaTaskForPartition(db, olapTable, partition, isInMemory);
+ }
+ }
+
+ public void sendUpdateTabletMetaTaskForPartition(Database db,
Review comment:
I am more worried that if the user does not retry, there will be an inconsistent state, that is, some tablet properties on BE are `in memory`, and FE is `not in memory`. There is no mechanism to find it in this case.
We need a correction mechanism to handle this. For example, when reporting to FE, compare with this attribute of FE.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on issue #2847: Doris
support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on issue #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#issuecomment-585522781
> LGTM.
> The report process is inefficient, may be we should optimize it later.
Yes. I agree with you. I also considered this point. I think we can handle `SetTabletInMemory` once an hour even once a day, if which has a performance issue.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on issue #2847: Doris
support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on issue #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#issuecomment-586827030
**Update the PR to Fix conflict only, not change from morningman last review.**
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangpinghuang commented on a change in pull
request #2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangpinghuang commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377977089
##########
File path: be/src/olap/page_cache.h
##########
@@ -74,7 +74,7 @@ class StoragePageCache {
// Given hanlde will be set to valid reference.
// This function is thread-safe, and when two clients insert two same key
// concurrently, this function can assure that only one page is cached.
- void insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle);
+ void insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle, bool in_memory = false);
Review comment:
add some comment for this variable
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r378017282
##########
File path: be/src/olap/page_cache.cpp
##########
@@ -42,11 +42,18 @@ bool StoragePageCache::lookup(const CacheKey& key, PageCacheHandle* handle) {
return true;
}
-void StoragePageCache::insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle) {
+void StoragePageCache::insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle,
+ bool in_memory) {
Review comment:
No.
`durable` is for `LRUCache`,
`in_memory` is for storage level.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r378018109
##########
File path: gensrc/thrift/AgentService.thrift
##########
@@ -233,10 +234,17 @@ struct TRecoverTabletReq {
4: optional Types.TVersionHash version_hash // Deprecated
}
+enum TTabletMetaType {
+ PARTITIONID,
Review comment:
I think that is no ambiguous. One meta has one type, which is clear. All meta could be combine.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377108563
##########
File path: fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
##########
@@ -1405,6 +1420,88 @@ private void sendClearAlterTask(Database db, OlapTable olapTable) {
LOG.info("send clear alter task for table {}, number: {}", olapTable.getName(), batchTask.getTaskNum());
}
+ public void sendUpdateTabletMetaInfoTask(Database db, OlapTable olapTable, boolean isInMemory) throws DdlException {
+ List<Partition> partitions = Lists.newArrayList();
+ db.readLock();
+ try {
+ partitions.addAll(olapTable.getPartitions());
+ } finally {
+ db.readUnlock();
+ }
+ for(Partition partition: partitions) {
+ sendUpdateTabletMetaTaskForPartition(db, olapTable, partition, isInMemory);
+ }
+ }
+
+ public void sendUpdateTabletMetaTaskForPartition(Database db,
+ OlapTable olapTable,
+ Partition partition,
+ boolean isInMemory) throws DdlException {
+ // be id -> <tablet id,schemaHash>
+ Map<Long, Set<Pair<Long, Integer>>> beIdToTabletIdWithHash = Maps.newHashMap();
+ db.readLock();
+ try {
+ for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ int schemaHash = olapTable.getSchemaHashByIndexId(index.getId());
+ for (Tablet tablet : index.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ Set<Pair<Long, Integer>> tabletIdWithHash = beIdToTabletIdWithHash.computeIfAbsent(replica.getBackendId(), k -> Sets.newHashSet());
+ tabletIdWithHash.add(new Pair<>(tablet.getId(), schemaHash));
+ }
+ }
+ }
+ } finally {
+ db.readUnlock();
+ }
+
+ int totalTaskNum = beIdToTabletIdWithHash.keySet().size();
+ MarkedCountDownLatch<Long, Set<Pair<Long, Integer>>> countDownLatch = new MarkedCountDownLatch<>(totalTaskNum);
+ AgentBatchTask batchTask = new AgentBatchTask();
+ for(Map.Entry<Long, Set<Pair<Long, Integer>>> kv: beIdToTabletIdWithHash.entrySet()) {
+ countDownLatch.addMark(kv.getKey(), kv.getValue());
+ UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(),
+ isInMemory,countDownLatch);
+ batchTask.addTask(task);
+ }
+ if (!FeConstants.runningUnitTest) {
+ // send all tasks and wait them finished
+ AgentTaskQueue.addBatchTask(batchTask);
+ AgentTaskExecutor.submit(batchTask);
+ LOG.info("send update tablet meta task for table {}, partitions {}, number: {}",
+ olapTable.getName(), partition.getName(), batchTask.getTaskNum());
+
+ // estimate timeout
+ long timeout = Config.tablet_create_timeout_second * 1000L * totalTaskNum;
+ timeout = Math.min(timeout, Config.max_create_table_timeout_second * 1000);
+ boolean ok = false;
+ try {
+ ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ LOG.warn("InterruptedException: ", e);
+ }
+
+ String errMsg;
Review comment:
`String errMsg;` can be put in the following `if` clause
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on issue #2847: Doris
support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on issue #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#issuecomment-585084555
1. Add `in_memory` property to partition level.
2.Handle FE and BE `in_memory` meta consistency in `ReportHandler`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r378017446
##########
File path: fe/src/main/java/org/apache/doris/master/ReportHandler.java
##########
@@ -884,13 +885,14 @@ private static void handleForceCreateReplica(List<CreateReplicaTask> createRepli
}
- private static void handleSetTabletMetaInfo(long backendId, SetMultimap<Long, Integer> tabletWithoutPartitionId) {
+ private static void handleSetTabletMetaInfo(long backendId, Set<Pair<Long, Integer>>tabletWithoutPartitionId) {
Review comment:
OK
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r378017399
##########
File path: be/src/olap/page_cache.h
##########
@@ -74,7 +74,7 @@ class StoragePageCache {
// Given hanlde will be set to valid reference.
// This function is thread-safe, and when two clients insert two same key
// concurrently, this function can assure that only one page is cached.
- void insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle);
+ void insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle, bool in_memory = false);
Review comment:
OK
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] imay commented on a change in pull request #2847:
Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
imay commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r380123351
##########
File path: be/src/agent/task_worker_pool.cpp
##########
@@ -846,7 +846,19 @@ void* TaskWorkerPool::_update_tablet_meta_worker_thread_callback(void* arg_this)
continue;
}
WriteLock wrlock(tablet->get_header_lock_ptr());
- tablet->set_partition_id(tablet_meta_info.partition_id);
+ // update tablet meta
+ if (!tablet_meta_info.__isset.meta_type) {
+ tablet->set_partition_id(tablet_meta_info.partition_id);
+ } else {
+ switch(tablet_meta_info.meta_type) {
Review comment:
```suggestion
switch (tablet_meta_info.meta_type) {
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377439575
##########
File path: fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
##########
@@ -1405,6 +1420,88 @@ private void sendClearAlterTask(Database db, OlapTable olapTable) {
LOG.info("send clear alter task for table {}, number: {}", olapTable.getName(), batchTask.getTaskNum());
}
+ public void sendUpdateTabletMetaInfoTask(Database db, OlapTable olapTable, boolean isInMemory) throws DdlException {
+ List<Partition> partitions = Lists.newArrayList();
+ db.readLock();
+ try {
+ partitions.addAll(olapTable.getPartitions());
+ } finally {
+ db.readUnlock();
+ }
+ for(Partition partition: partitions) {
+ sendUpdateTabletMetaTaskForPartition(db, olapTable, partition, isInMemory);
+ }
+ }
+
+ public void sendUpdateTabletMetaTaskForPartition(Database db,
+ OlapTable olapTable,
+ Partition partition,
+ boolean isInMemory) throws DdlException {
+ // be id -> <tablet id,schemaHash>
+ Map<Long, Set<Pair<Long, Integer>>> beIdToTabletIdWithHash = Maps.newHashMap();
+ db.readLock();
+ try {
+ for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ int schemaHash = olapTable.getSchemaHashByIndexId(index.getId());
+ for (Tablet tablet : index.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ Set<Pair<Long, Integer>> tabletIdWithHash = beIdToTabletIdWithHash.computeIfAbsent(replica.getBackendId(), k -> Sets.newHashSet());
+ tabletIdWithHash.add(new Pair<>(tablet.getId(), schemaHash));
+ }
+ }
+ }
+ } finally {
+ db.readUnlock();
+ }
+
+ int totalTaskNum = beIdToTabletIdWithHash.keySet().size();
+ MarkedCountDownLatch<Long, Set<Pair<Long, Integer>>> countDownLatch = new MarkedCountDownLatch<>(totalTaskNum);
+ AgentBatchTask batchTask = new AgentBatchTask();
+ for(Map.Entry<Long, Set<Pair<Long, Integer>>> kv: beIdToTabletIdWithHash.entrySet()) {
+ countDownLatch.addMark(kv.getKey(), kv.getValue());
+ UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(),
+ isInMemory,countDownLatch);
+ batchTask.addTask(task);
+ }
+ if (!FeConstants.runningUnitTest) {
+ // send all tasks and wait them finished
+ AgentTaskQueue.addBatchTask(batchTask);
+ AgentTaskExecutor.submit(batchTask);
+ LOG.info("send update tablet meta task for table {}, partitions {}, number: {}",
+ olapTable.getName(), partition.getName(), batchTask.getTaskNum());
+
+ // estimate timeout
+ long timeout = Config.tablet_create_timeout_second * 1000L * totalTaskNum;
+ timeout = Math.min(timeout, Config.max_create_table_timeout_second * 1000);
+ boolean ok = false;
+ try {
+ ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ LOG.warn("InterruptedException: ", e);
+ }
+
+ String errMsg;
Review comment:
OK
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangpinghuang commented on a change in pull
request #2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangpinghuang commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377985339
##########
File path: gensrc/thrift/AgentService.thrift
##########
@@ -233,10 +234,17 @@ struct TRecoverTabletReq {
4: optional Types.TVersionHash version_hash // Deprecated
}
+enum TTabletMetaType {
+ PARTITIONID,
Review comment:
I think this name is ambiguous, Is INMEMORY can not has partitionid?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on issue #2847: Doris
support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on issue #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#issuecomment-583972160
Update the PR for https://github.com/apache/incubator-doris/issues/2866
Currently, I don't persist `in_memory` property in FE partition metadata, I think which is not very n necessary. If we think we should persist `in_memory` property in FE partition metadata, I could do this work in next PR. This PR is big enough.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangpinghuang commented on a change in pull
request #2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangpinghuang commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377983847
##########
File path: fe/src/main/java/org/apache/doris/master/ReportHandler.java
##########
@@ -884,13 +885,14 @@ private static void handleForceCreateReplica(List<CreateReplicaTask> createRepli
}
- private static void handleSetTabletMetaInfo(long backendId, SetMultimap<Long, Integer> tabletWithoutPartitionId) {
+ private static void handleSetTabletMetaInfo(long backendId, Set<Pair<Long, Integer>>tabletWithoutPartitionId) {
Review comment:
```suggestion
private static void handleSetTabletMetaInfo(long backendId, Set<Pair<Long, Integer>> tabletWithoutPartitionId) {
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377440342
##########
File path: fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
##########
@@ -1405,6 +1420,88 @@ private void sendClearAlterTask(Database db, OlapTable olapTable) {
LOG.info("send clear alter task for table {}, number: {}", olapTable.getName(), batchTask.getTaskNum());
}
+ public void sendUpdateTabletMetaInfoTask(Database db, OlapTable olapTable, boolean isInMemory) throws DdlException {
+ List<Partition> partitions = Lists.newArrayList();
+ db.readLock();
+ try {
+ partitions.addAll(olapTable.getPartitions());
+ } finally {
+ db.readUnlock();
+ }
+ for(Partition partition: partitions) {
+ sendUpdateTabletMetaTaskForPartition(db, olapTable, partition, isInMemory);
+ }
+ }
+
+ public void sendUpdateTabletMetaTaskForPartition(Database db,
Review comment:
If partial success, we will return error to user, and the error msg warn user to retry.
Because this operation is idempotent，So retry is OK.
If we want to avoid partial success, we need to support tablet meta rollback operation. I think which is complex and unnecessary.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen merged pull request #2847: Doris
support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen merged pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r376187046
##########
File path: fe/src/main/java/org/apache/doris/catalog/OlapTable.java
##########
@@ -1011,17 +1010,17 @@ public void readFields(DataInput in) throws IOException {
tableProperty = TableProperty.read(in);
}
}
+ // inMemory
+ if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_72) {
+ isInMemory = in.readBoolean();
Review comment:
OK.
I didn't notice the `TableProperty`, Thanks for your reminder.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] kangkaisen edited a comment on issue #2847: Doris
support in memory olap table
Posted by GitBox <gi...@apache.org>.
kangkaisen edited a comment on issue #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#issuecomment-585084555
1. Add `in_memory` property to partition level.
2. Handle FE and BE `in_memory` meta consistency in `ReportHandler`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request
#2847: Doris support in memory olap table
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #2847: Doris support in memory olap table
URL: https://github.com/apache/incubator-doris/pull/2847#discussion_r377111269
##########
File path: fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
##########
@@ -1405,6 +1420,88 @@ private void sendClearAlterTask(Database db, OlapTable olapTable) {
LOG.info("send clear alter task for table {}, number: {}", olapTable.getName(), batchTask.getTaskNum());
}
+ public void sendUpdateTabletMetaInfoTask(Database db, OlapTable olapTable, boolean isInMemory) throws DdlException {
+ List<Partition> partitions = Lists.newArrayList();
+ db.readLock();
+ try {
+ partitions.addAll(olapTable.getPartitions());
+ } finally {
+ db.readUnlock();
+ }
+ for(Partition partition: partitions) {
+ sendUpdateTabletMetaTaskForPartition(db, olapTable, partition, isInMemory);
+ }
+ }
+
+ public void sendUpdateTabletMetaTaskForPartition(Database db,
Review comment:
`sendUpdateTabletMetaTaskForPartition()`
what if this method is partial success? And some of the tablet's meta on BE changed to `in memory` and some are not?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org