You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2024/06/15 17:21:51 UTC
(asterixdb) 05/14: [NO ISSUE][FUN] Skip job capacity assignment for completed/active requests
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 14c153f1e7d4dd93df1edcda8ff83efcd70ecbc4
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Wed Dec 20 19:13:49 2023 -0800
[NO ISSUE][FUN] Skip job capacity assignment for completed/active requests
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: I87c2fd9c864b29e8ec905df9d22e1e5cfa99147c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18021
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Wail Alkowaileet <wa...@gmail.com>
(cherry picked from commit d46e8b69468d1f01ba50595d1304ee8e28aa7785)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18113
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../asterix/optimizer/base/AsterixOptimizationContext.java | 13 +++++++++----
.../asterix/app/function/ActiveRequestsDatasource.java | 5 +++++
.../asterix/app/function/CompletedRequestsDatasource.java | 5 +++++
.../results/misc/active_requests/active_requests.4.regex | 2 +-
.../misc/completed_requests/completed_requests.3.regex | 2 +-
.../asterix/metadata/declared/FunctionDataSource.java | 4 ++++
6 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java
index 05e7f7b8a6..733da2dcef 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java
@@ -24,6 +24,7 @@ import java.util.Set;
import java.util.function.Predicate;
import org.apache.asterix.metadata.declared.DataSource;
+import org.apache.asterix.metadata.declared.FunctionDataSource;
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver;
@@ -83,10 +84,10 @@ public final class AsterixOptimizationContext extends AlgebricksOptimizationCont
}
for (Int2ObjectMap.Entry<Set<DataSource>> me : dataSourceMap.int2ObjectEntrySet()) {
int dataSourceType = me.getIntKey();
- if (dataSourceType != DataSource.Type.INTERNAL_DATASET) {
+ if (dataSourceType != DataSource.Type.INTERNAL_DATASET && dataSourceType != DataSource.Type.FUNCTION) {
return false;
}
- Predicate<DataSource> dataSourceTest = AsterixOptimizationContext::isMetadata;
+ Predicate<DataSource> dataSourceTest = AsterixOptimizationContext::skipJobCapacityAssignment;
if (!me.getValue().stream().allMatch(dataSourceTest)) {
return false;
}
@@ -94,7 +95,11 @@ public final class AsterixOptimizationContext extends AlgebricksOptimizationCont
return true;
}
- private static boolean isMetadata(DataSource ds) {
- return MetadataConstants.METADATA_DATAVERSE_NAME.equals(ds.getId().getDataverseName());
+ private static boolean skipJobCapacityAssignment(DataSource ds) {
+ return MetadataConstants.METADATA_DATAVERSE_NAME.equals(ds.getId().getDataverseName()) || isSkipping(ds);
+ }
+
+ private static boolean isSkipping(DataSource ds) {
+ return (ds instanceof FunctionDataSource) && ((FunctionDataSource) ds).skipJobCapacityAssignment();
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
index 24a127ae8b..5239d0146f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
@@ -43,4 +43,9 @@ public class ActiveRequestsDatasource extends FunctionDataSource {
AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations());
return new ClientRequestsFunction(randomLocation, ClientRequestsRequest.RequestType.RUNNING);
}
+
+ @Override
+ public boolean skipJobCapacityAssignment() {
+ return true;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
index 0bfbff1bf1..f02af21955 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
@@ -43,4 +43,9 @@ public class CompletedRequestsDatasource extends FunctionDataSource {
AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations());
return new ClientRequestsFunction(randomLocation, ClientRequestsRequest.RequestType.COMPLETED);
}
+
+ @Override
+ public boolean skipJobCapacityAssignment() {
+ return true;
+ }
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
index e2d166b9ad..695472c47f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/active_requests/active_requests.4.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+0/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
index e2d166b9ad..695472c47f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/completed_requests/completed_requests.3.regex
@@ -1 +1 @@
-/memory\D+229376/
\ No newline at end of file
+/memory\D+0/
\ No newline at end of file
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 97c4c5d2b2..3b6bd9d55a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -106,6 +106,10 @@ public abstract class FunctionDataSource extends DataSource {
tupleFilterFactory, outputLimit);
}
+ public boolean skipJobCapacityAssignment() {
+ return false;
+ }
+
protected abstract IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations);