You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2020/04/24 13:26:20 UTC
[GitHub] [incubator-doris] morningman opened a new pull request #3391: [Load] Add more info in SHOW LOAD result
morningman opened a new pull request #3391:
URL: https://github.com/apache/incubator-doris/pull/3391
Fix #3390
This CL add more info in `JobDetails` column of `SHOW LOAD` result for Broker Load Job.
For example:
```
{
"Unfinished backends": {
"9c3441027ff948a0-8287923329a2b6a7": [10002]
},
"All backends": {
"9c3441027ff948a0-8287923329a2b6a7": [10002]
},
"ScannedRows": 2390016,
"TaskNumber": 1,
"FileNumber": 1,
"FileSize": 1073741824
}
```
2 newly added keys:
`Unfinished backends` indicates the BE which task on them are not finished.
`All backends` indicates the BE which this job has tasks on it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3391: [Load] Add more info in SHOW LOAD result
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3391:
URL: https://github.com/apache/incubator-doris/pull/3391#discussion_r415065810
##########
File path: fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
##########
@@ -138,30 +139,41 @@
// load task id -> fragment id -> rows count
private Table<TUniqueId, TUniqueId, Long> counterTbl = HashBasedTable.create();
+ // load task id -> unfinished backend id list
+ private Map<TUniqueId, List<Long>> unfinishedBackendIds = Maps.newHashMap();
+ // load task id -> all backend id list
+ private Map<TUniqueId, List<Long>> allBackendIds = Maps.newHashMap();
+
// number of file to be loaded
public int fileNum = 0;
public long totalFileSizeB = 0;
// init the statistic of specified load task
- public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds) {
+ public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds, List<Long> relatedBackendIds) {
counterTbl.rowMap().remove(loadId);
for (TUniqueId fragId : fragmentIds) {
counterTbl.put(loadId, fragId, 0L);
}
+ allBackendIds.put(loadId, relatedBackendIds);
+ // need to get a copy of relatedBackendIds, so that when we modify the "relatedBackendIds" in
+ // allBackendIds, the list in unfinishedBackendIds will not be changed.
+ unfinishedBackendIds.put(loadId, Lists.newArrayList(relatedBackendIds));
}
public synchronized void removeLoad(TUniqueId loadId) {
counterTbl.rowMap().remove(loadId);
+ unfinishedBackendIds.remove(loadId);
+ allBackendIds.remove(loadId);
}
- public synchronized void updateLoad(TUniqueId loadId, TUniqueId fragmentId, long rows) {
+ public synchronized void updateLoadProgress(long beId, TUniqueId loadId, TUniqueId fragmentId, long rows,
+ boolean isDone) {
if (counterTbl.contains(loadId, fragmentId)) {
counterTbl.put(loadId, fragmentId, rows);
}
- }
-
- public synchronized void clearAllLoads() {
- counterTbl.clear();
+ if (isDone && unfinishedBackendIds.containsKey(loadId)) {
+ unfinishedBackendIds.get(loadId).remove(beId);
Review comment:
It won't, there is at least one BE for a load job
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3391: [Load] Add more info in SHOW LOAD result
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3391:
URL: https://github.com/apache/incubator-doris/pull/3391#discussion_r415066191
##########
File path: fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
##########
@@ -138,30 +139,41 @@
// load task id -> fragment id -> rows count
private Table<TUniqueId, TUniqueId, Long> counterTbl = HashBasedTable.create();
+ // load task id -> unfinished backend id list
+ private Map<TUniqueId, List<Long>> unfinishedBackendIds = Maps.newHashMap();
+ // load task id -> all backend id list
+ private Map<TUniqueId, List<Long>> allBackendIds = Maps.newHashMap();
+
// number of file to be loaded
public int fileNum = 0;
public long totalFileSizeB = 0;
// init the statistic of specified load task
- public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds) {
+ public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds, List<Long> relatedBackendIds) {
counterTbl.rowMap().remove(loadId);
for (TUniqueId fragId : fragmentIds) {
counterTbl.put(loadId, fragId, 0L);
}
+ allBackendIds.put(loadId, relatedBackendIds);
+ // need to get a copy of relatedBackendIds, so that when we modify the "relatedBackendIds" in
+ // allBackendIds, the list in unfinishedBackendIds will not be changed.
+ unfinishedBackendIds.put(loadId, Lists.newArrayList(relatedBackendIds));
}
public synchronized void removeLoad(TUniqueId loadId) {
counterTbl.rowMap().remove(loadId);
+ unfinishedBackendIds.remove(loadId);
+ allBackendIds.remove(loadId);
}
- public synchronized void updateLoad(TUniqueId loadId, TUniqueId fragmentId, long rows) {
+ public synchronized void updateLoadProgress(long beId, TUniqueId loadId, TUniqueId fragmentId, long rows,
Review comment:
changed
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] wutiangan commented on a change in pull request #3391: [Load] Add more info in SHOW LOAD result
Posted by GitBox <gi...@apache.org>.
wutiangan commented on a change in pull request #3391:
URL: https://github.com/apache/incubator-doris/pull/3391#discussion_r415017185
##########
File path: fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
##########
@@ -138,30 +139,41 @@
// load task id -> fragment id -> rows count
private Table<TUniqueId, TUniqueId, Long> counterTbl = HashBasedTable.create();
+ // load task id -> unfinished backend id list
+ private Map<TUniqueId, List<Long>> unfinishedBackendIds = Maps.newHashMap();
+ // load task id -> all backend id list
+ private Map<TUniqueId, List<Long>> allBackendIds = Maps.newHashMap();
+
// number of file to be loaded
public int fileNum = 0;
public long totalFileSizeB = 0;
// init the statistic of specified load task
- public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds) {
+ public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds, List<Long> relatedBackendIds) {
counterTbl.rowMap().remove(loadId);
for (TUniqueId fragId : fragmentIds) {
counterTbl.put(loadId, fragId, 0L);
}
+ allBackendIds.put(loadId, relatedBackendIds);
+ // need to get a copy of relatedBackendIds, so that when we modify the "relatedBackendIds" in
+ // allBackendIds, the list in unfinishedBackendIds will not be changed.
+ unfinishedBackendIds.put(loadId, Lists.newArrayList(relatedBackendIds));
}
public synchronized void removeLoad(TUniqueId loadId) {
counterTbl.rowMap().remove(loadId);
+ unfinishedBackendIds.remove(loadId);
+ allBackendIds.remove(loadId);
}
- public synchronized void updateLoad(TUniqueId loadId, TUniqueId fragmentId, long rows) {
+ public synchronized void updateLoadProgress(long beId, TUniqueId loadId, TUniqueId fragmentId, long rows,
Review comment:
Maybe change “beId” to ”backendId“ is better，at before you use unfinished“BackendIds” and all”BackendIds“
##########
File path: fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
##########
@@ -138,30 +139,41 @@
// load task id -> fragment id -> rows count
private Table<TUniqueId, TUniqueId, Long> counterTbl = HashBasedTable.create();
+ // load task id -> unfinished backend id list
+ private Map<TUniqueId, List<Long>> unfinishedBackendIds = Maps.newHashMap();
+ // load task id -> all backend id list
+ private Map<TUniqueId, List<Long>> allBackendIds = Maps.newHashMap();
+
// number of file to be loaded
public int fileNum = 0;
public long totalFileSizeB = 0;
// init the statistic of specified load task
- public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds) {
+ public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds, List<Long> relatedBackendIds) {
counterTbl.rowMap().remove(loadId);
for (TUniqueId fragId : fragmentIds) {
counterTbl.put(loadId, fragId, 0L);
}
+ allBackendIds.put(loadId, relatedBackendIds);
+ // need to get a copy of relatedBackendIds, so that when we modify the "relatedBackendIds" in
+ // allBackendIds, the list in unfinishedBackendIds will not be changed.
+ unfinishedBackendIds.put(loadId, Lists.newArrayList(relatedBackendIds));
}
public synchronized void removeLoad(TUniqueId loadId) {
counterTbl.rowMap().remove(loadId);
+ unfinishedBackendIds.remove(loadId);
+ allBackendIds.remove(loadId);
}
- public synchronized void updateLoad(TUniqueId loadId, TUniqueId fragmentId, long rows) {
+ public synchronized void updateLoadProgress(long beId, TUniqueId loadId, TUniqueId fragmentId, long rows,
+ boolean isDone) {
if (counterTbl.contains(loadId, fragmentId)) {
counterTbl.put(loadId, fragmentId, rows);
}
- }
-
- public synchronized void clearAllLoads() {
- counterTbl.clear();
+ if (isDone && unfinishedBackendIds.containsKey(loadId)) {
+ unfinishedBackendIds.get(loadId).remove(beId);
Review comment:
can unfinishedBackendIds.get(loadId) be empty？
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org