You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2025/10/14 19:29:16 UTC
(atlas) branch atlas-2.5 updated (74c9b98c1 -> c788d0014)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a change to branch atlas-2.5
in repository https://gitbox.apache.org/repos/asf/atlas.git
from 74c9b98c1 ATLAS-5153: React UI: Statistics component displays count as NaN when total exceeds 999 (#467) ( cherry picked from ac838ccbe00548e152909fe628c1caca3feb1f75 )
new ef49d2355 ATLAS-5136: Upgrade axios version to 1.12.0 (#442)
new c788d0014 ATLAS-5154: update notification processing metrics to include avgProcessingTime per topic-partition (#468)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/package-lock.json | 21 ++++-----
docs/package.json | 2 +-
.../store/graph/v2/tasks/ClassificationTask.java | 2 +-
.../org/apache/atlas/util/AtlasMetricsUtil.java | 52 +++++++++++++---------
.../notification/NotificationHookConsumer.java | 6 +--
5 files changed, 45 insertions(+), 38 deletions(-)
(atlas) 01/02: ATLAS-5136: Upgrade axios version to 1.12.0 (#442)
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch atlas-2.5
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit ef49d2355794f995d74652755c68695035b0e0fe
Author: Disha Talreja <37...@users.noreply.github.com>
AuthorDate: Fri Sep 26 09:51:18 2025 -0700
ATLAS-5136: Upgrade axios version to 1.12.0 (#442)
(cherry picked from commit 69cc0df422c51e94c28c8cd9d5dfbfb54af7cc65)
---
docs/package-lock.json | 21 ++++++++-------------
docs/package.json | 2 +-
.../store/graph/v2/tasks/ClassificationTask.java | 2 +-
3 files changed, 10 insertions(+), 15 deletions(-)
diff --git a/docs/package-lock.json b/docs/package-lock.json
index 734f859e7..7ef28e614 100644
--- a/docs/package-lock.json
+++ b/docs/package-lock.json
@@ -2405,25 +2405,26 @@
}
},
"axios": {
- "version": "1.8.4",
- "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz",
- "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==",
+ "version": "1.12.0",
+ "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.0.tgz",
+ "integrity": "sha512-oXTDccv8PcfjZmPGlWsPSwtOJCZ/b6W5jAMCNcfwJbCzDckwG0jrYJFaWH1yvivfCXjVzV/SPDEhMB3Q+DSurg==",
"dev": true,
"requires": {
"follow-redirects": "^1.15.6",
- "form-data": "^4.0.0",
+ "form-data": "^4.0.4",
"proxy-from-env": "^1.1.0"
},
"dependencies": {
"form-data": {
- "version": "4.0.2",
- "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz",
- "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==",
+ "version": "4.0.4",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
+ "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
"dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
+ "hasown": "^2.0.2",
"mime-types": "^2.1.12"
}
}
@@ -9130,12 +9131,6 @@
"ipaddr.js": "1.9.1"
}
},
- "proxy-from-env": {
- "version": "1.1.0",
- "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
- "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
- "dev": true
- },
"prr": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
diff --git a/docs/package.json b/docs/package.json
index 8a34ba566..77f336969 100644
--- a/docs/package.json
+++ b/docs/package.json
@@ -19,7 +19,7 @@
"@babel/core": "^7.16.7",
"@babel/plugin-syntax-dynamic-import": "^7.8.3",
"@babel/preset-react": "^7.16.7",
- "axios": "0.19.0",
+ "axios": "1.12.0",
"babel-loader": "8.0.6",
"babel-plugin-lodash": "3.3.4",
"babel-plugin-react-transform": "3.0.0",
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java
index 1822581d4..8e790d4a2 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java
@@ -130,7 +130,7 @@ public abstract class ClassificationTask extends AbstractTask {
if (obj != null) {
LOG.debug("Task: {}: Setting import progress set to: {}", getTaskGuid(), obj);
RequestContext.get().setImportInProgress((Boolean) obj);
- }else {
+ } else {
RequestContext.get().setImportInProgress(DEFAULT_IMPORT_IN_PROGRESS);
}
(atlas) 02/02: ATLAS-5154: update notification processing metrics to include avgProcessingTime per topic-partition (#468)
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch atlas-2.5
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit c788d00145ee5b7eb0b967d3e99d3fde46256c2b
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Tue Oct 14 12:02:46 2025 -0700
ATLAS-5154: update notification processing metrics to include avgProcessingTime per topic-partition (#468)
(cherry picked from commit 495110cf52846c54833bbe8ce322099099cf14e6)
---
.../org/apache/atlas/util/AtlasMetricsUtil.java | 52 +++++++++++++---------
.../notification/NotificationHookConsumer.java | 6 +--
2 files changed, 35 insertions(+), 23 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java b/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java
index f04592451..4172abe9a 100644
--- a/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java
+++ b/repository/src/main/java/org/apache/atlas/util/AtlasMetricsUtil.java
@@ -39,6 +39,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
import static org.apache.atlas.model.metrics.AtlasMetrics.STAT_NOTIFY_AVG_TIME_CURR_DAY;
import static org.apache.atlas.model.metrics.AtlasMetrics.STAT_NOTIFY_AVG_TIME_CURR_HOUR;
@@ -162,7 +163,7 @@ public class AtlasMetricsUtil {
partitionStat.incrFailedMessageCount();
}
- partitionStat.incrProcessedMessageCount();
+ partitionStat.incrProcessedMessageCount(stats.timeTakenMs);
partitionStat.setLastMessageProcessedTime(messagesProcessed.getLastIncrTime().toEpochMilli());
}
@@ -187,15 +188,18 @@ public class AtlasMetricsUtil {
for (TopicPartitionStat tpStat : tStat.partitionStats.values()) {
Map<String, Long> tpDetails = new HashMap<>();
- tpDetails.put("offsetStart", tpStat.startOffset);
- tpDetails.put("offsetCurrent", tpStat.currentOffset);
- tpDetails.put("failedMessageCount", tpStat.failedMessageCount);
- tpDetails.put("lastMessageProcessedTime", tpStat.lastMessageProcessedTime);
- tpDetails.put("processedMessageCount", tpStat.processedMessageCount);
+ tpDetails.put("offsetStart", tpStat.getStartOffset());
+ tpDetails.put("offsetCurrent", tpStat.getCurrentOffset());
+ tpDetails.put("failedMessageCount", tpStat.getFailedMessageCount());
+ tpDetails.put("lastMessageProcessedTime", tpStat.getLastMessageProcessedTime());
+ tpDetails.put("processedMessageCount", tpStat.getProcessedMessageCount());
+ tpDetails.put("avgProcessingTime", tpStat.getAvgProcessingTime());
- LOG.debug("Setting failedMessageCount : {} and lastMessageProcessedTime : {} for topic {}-{}", tpStat.failedMessageCount, tpStat.lastMessageProcessedTime, tpStat.topicName, tpStat.partition);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting failedMessageCount : {} and lastMessageProcessedTime : {} for topic {}-{}", tpStat.getFailedMessageCount(), tpStat.getLastMessageProcessedTime(), tpStat.getTopicName(), tpStat.getPartition());
+ }
- topicDetails.put(tpStat.topicName + "-" + tpStat.partition, tpDetails);
+ topicDetails.put(tpStat.getTopicName() + "-" + tpStat.getPartition(), tpDetails);
}
}
@@ -418,13 +422,14 @@ public class AtlasMetricsUtil {
}
static class TopicPartitionStat {
- private final String topicName;
- private final int partition;
- private final long startOffset;
- private long currentOffset;
- private long lastMessageProcessedTime;
- private long failedMessageCount;
- private long processedMessageCount;
+ private final String topicName;
+ private final int partition;
+ private final long startOffset;
+ private long currentOffset;
+ private long lastMessageProcessedTime;
+ private final AtomicLong failedMessageCount = new AtomicLong();
+ private final AtomicLong processedMessageCount = new AtomicLong();
+ private final AtomicLong totalProcessingTimeMs = new AtomicLong();
public TopicPartitionStat(String topicName, int partition, long startOffset, long currentOffset) {
this.topicName = topicName;
@@ -462,19 +467,26 @@ public class AtlasMetricsUtil {
}
public long getFailedMessageCount() {
- return failedMessageCount;
+ return failedMessageCount.get();
}
public void incrFailedMessageCount() {
- this.failedMessageCount++;
+ this.failedMessageCount.incrementAndGet();
}
public long getProcessedMessageCount() {
- return processedMessageCount;
+ return processedMessageCount.get();
}
- public void incrProcessedMessageCount() {
- this.processedMessageCount++;
+ public void incrProcessedMessageCount(long timeTakenMs) {
+ this.processedMessageCount.incrementAndGet();
+ this.totalProcessingTimeMs.addAndGet(timeTakenMs);
+ }
+
+ public long getAvgProcessingTime() {
+ long processedMessageCount = this.processedMessageCount.get();
+
+ return processedMessageCount == 0 ? 0 : (totalProcessingTimeMs.get() / processedMessageCount);
}
}
}
diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
index 001e57579..642bb0bf1 100644
--- a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
+++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
@@ -701,7 +701,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
}
if (entities.size() - count > 0) {
- LOG.info("preprocess: moved {} hive_process/hive_column_lineage entities to end of list (listSize={}). topic-offset={}, partition={}", entities.size() - count, entities.size(), kafkaMsg.getOffset(), kafkaMsg.getPartition());
+ LOG.info("preprocess: moved {} hive_process/hive_column_lineage entities to end of list (listSize={}). topic={}, partition={}, offset={}", entities.size() - count, entities.size(), kafkaMsg.getTopic(), kafkaMsg.getPartition(), kafkaMsg.getOffset());
}
}
}
@@ -1533,9 +1533,9 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
try {
String strMessage = AbstractNotification.getMessageJson(message);
- LOG.warn("msgProcessingTime={}, msgSize={}, topicOffset={}}", stats.timeTakenMs, strMessage.length(), kafkaMsg.getOffset());
+ LOG.warn("msgProcessingTime={}, msgSize={}, topic={}, partition={}, offset={}}", stats.timeTakenMs, strMessage.length(), kafkaMsg.getTopic(), kafkaMsg.getPartition(), kafkaMsg.getOffset());
- LARGE_MESSAGES_LOG.warn("{\"msgProcessingTime\":{},\"msgSize\":{},\"topicOffset\":{},\"data\":{}}", stats.timeTakenMs, strMessage.length(), kafkaMsg.getOffset(), strMessage);
+ LARGE_MESSAGES_LOG.warn("{\"msgProcessingTime\":{},\"msgSize\":{},\"topic\":{},\"partition\":{},\"topicOffset\":{},\"data\":{}}", stats.timeTakenMs, strMessage.length(), kafkaMsg.getTopic(), kafkaMsg.getPartition(), kafkaMsg.getOffset(), strMessage);
} catch (Throwable t) {
LOG.warn("error while recording large message: msgProcessingTime={}, type={}, topic={}, partition={}, offset={}", stats.timeTakenMs, message.getType(), kafkaMsg.getTopic(), kafkaMsg.getPartition(), kafkaMsg.getOffset(), t);
}