You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2023/12/13 22:49:15 UTC

(asterixdb) 06/09: [NO ISSUE][API] Add query request param to indicate source

This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 9079019423f96566997d933ef789058c1e28baba
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Fri Dec 8 09:00:59 2023 -0800

    [NO ISSUE][API] Add query request param to indicate source
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    Add parameter 'source' to QueryServiceRequestParameters
    to indicate the source of the request.
    
    Change-Id: I04e2e2336d6bca82b4fa44a43cb188cfa9e807f7
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18006
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Michael Blow <mb...@apache.org>
---
 .../http/server/QueryServiceRequestParameters.java | 22 +++++++++++++++++++---
 .../api/http/server/QueryServiceServlet.java       | 20 +++++++++-----------
 .../message/ExecuteStatementRequestMessage.java    |  5 +++--
 .../asterix/app/translator/QueryTranslator.java    |  6 +++---
 .../org/apache/asterix/utils/RedactionUtil.java    |  3 +++
 5 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
index b6913e448b..20d79d5e7e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
@@ -19,6 +19,8 @@
 
 package org.apache.asterix.api.http.server;
 
+import static org.apache.asterix.utils.RedactionUtil.REDACTED_SENSITIVE_ENTRY_VALUE;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -43,6 +45,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.server.utils.HttpUtil;
 import org.apache.hyracks.util.JSONUtil;
+import org.apache.hyracks.util.LogRedactionUtil;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -80,7 +83,8 @@ public class QueryServiceRequestParameters {
         SIGNATURE("signature"),
         MULTI_STATEMENT("multi-statement"),
         MAX_WARNINGS("max-warnings"),
-        SQL_COMPAT("sql-compat");
+        SQL_COMPAT("sql-compat"),
+        SOURCE("source");
 
         private final String str;
 
@@ -124,6 +128,7 @@ public class QueryServiceRequestParameters {
     private String statement;
     private String clientContextID;
     private String dataverse;
+    private String source;
     private ClientType clientType = ClientType.ASTERIX;
     private OutputFormat format = OutputFormat.CLEAN_JSON;
     private ResultDelivery mode = ResultDelivery.IMMEDIATE;
@@ -172,6 +177,14 @@ public class QueryServiceRequestParameters {
         this.statement = statement;
     }
 
+    public String getSource() {
+        return source;
+    }
+
+    public void setSource(String source) {
+        this.source = source;
+    }
+
     public OutputFormat getFormat() {
         return format;
     }
@@ -380,7 +393,8 @@ public class QueryServiceRequestParameters {
         ObjectNode object = OBJECT_MAPPER.createObjectNode();
         object.put("host", host);
         object.put("path", path);
-        object.put("statement", statement != null ? JSONUtil.escape(new StringBuilder(), statement).toString() : null);
+        object.put("statement", statement != null
+                ? LogRedactionUtil.statement(JSONUtil.escape(new StringBuilder(), statement).toString()) : null);
         object.put("pretty", pretty);
         object.put("mode", mode.getName());
         object.put("clientContextID", clientContextID);
@@ -402,9 +416,10 @@ public class QueryServiceRequestParameters {
         object.put("readOnly", readOnly);
         object.put("maxWarnings", maxWarnings);
         object.put("sqlCompat", sqlCompatMode);
+        object.put("source", source);
         if (statementParams != null) {
             for (Map.Entry<String, JsonNode> statementParam : statementParams.entrySet()) {
-                object.set('$' + statementParam.getKey(), statementParam.getValue());
+                object.set('$' + statementParam.getKey(), REDACTED_SENSITIVE_ENTRY_VALUE);
             }
         }
         return object;
@@ -486,6 +501,7 @@ public class QueryServiceRequestParameters {
         setSignature(parseBoolean(req, Parameter.SIGNATURE.str(), valGetter, isSignature()));
         setClientType(parseIfExists(req, Parameter.CLIENT_TYPE.str(), valGetter, getClientType(), clientTypes::get));
         setSQLCompatMode(parseBoolean(req, Parameter.SQL_COMPAT.str(), valGetter, isSQLCompatMode()));
+        setSource(valGetter.apply(req, Parameter.SOURCE.str()));
     }
 
     protected void setExtraParams(JsonNode jsonRequest) throws HyracksDataException {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 1966a88ba6..9a8c0d5b23 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -90,7 +90,6 @@ import org.apache.hyracks.control.common.controllers.CCConfig;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
 import org.apache.hyracks.http.server.utils.HttpUtil;
-import org.apache.hyracks.util.LogRedactionUtil;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -278,7 +277,10 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
             if (forceReadOnly) {
                 param.setReadOnly(true);
             }
-            LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.statement(param.toString()));
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info("handleRequest: uuid={}, clientContextID={}, {}", requestRef.getUuid(),
+                        param.getClientContextID(), param.toString());
+            }
             delivery = param.getMode();
             setSessionConfig(sessionOutput, param, delivery);
             final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads());
@@ -429,14 +431,12 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
                     executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
                     return true;
                 case REQUEST_TIMEOUT:
-                    LOGGER.info(() -> "handleException: request execution timed out: "
-                            + LogRedactionUtil.userData(param.toString()));
+                    LOGGER.info(() -> "handleException: request execution timed out: " + param.toString());
                     executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK);
                     return true;
                 case REJECT_NODE_UNREGISTERED:
                 case REJECT_BAD_CLUSTER_STATE:
-                    LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": "
-                            + LogRedactionUtil.userData(param.toString()));
+                    LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": " + param.toString());
                     executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE);
                     return true;
                 default:
@@ -456,11 +456,9 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
             QueryServiceRequestParameters param, IServletResponse response) {
         if (t instanceof org.apache.asterix.lang.sqlpp.parser.TokenMgrError || t instanceof AlgebricksException) {
             if (LOGGER.isDebugEnabled()) {
-                LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.statement(param.toString()),
-                        t);
+                LOGGER.debug("handleException: {}: {}", t.getMessage(), param.toString(), t);
             } else {
-                LOGGER.info(() -> "handleException: " + t.getMessage() + ": "
-                        + LogRedactionUtil.statement(param.toString()));
+                LOGGER.info(() -> "handleException: " + t.getMessage() + ": " + param.toString());
             }
             executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
             return;
@@ -472,7 +470,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
                 return;
             }
         }
-        LOGGER.warn(() -> "handleException: unexpected exception: " + LogRedactionUtil.userData(param.toString()), t);
+        LOGGER.warn(() -> "handleException: unexpected exception: " + param.toString(), t);
         executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
     }
 
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 05bc87b2f5..e31417798b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -243,7 +243,8 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
 
     @Override
     public String toString() {
-        return String.format("%s(id=%s, from=%s, uuid=%s): %s", getClass().getSimpleName(), requestMessageId,
-                requestNodeId, requestReference.getUuid(), LogRedactionUtil.statement(statementsText));
+        return String.format("%s(id=%s, from=%s, uuid=%s, clientContextID=%s): %s", getClass().getSimpleName(),
+                requestMessageId, requestNodeId, requestReference.getUuid(), clientContextID,
+                LogRedactionUtil.statement(statementsText));
     }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index afdd0359a4..6b40cbfffd 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -4837,9 +4837,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
             // ensure request not cancelled before running job
             ensureNotCancelled(clientRequest);
             final JobId jobId = JobUtils.runJob(hcc, jobSpec, jobFlags, false);
-            if (LOGGER.isDebugEnabled()) {
-                LOGGER.debug("createAndRunJob jobId:{}, uuid:{}", jobId,
-                        requestParameters.getRequestReference().getUuid());
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info("Created job {} for query uuid:{}, clientContextID:{}", jobId,
+                        requestParameters.getRequestReference().getUuid(), requestParameters.getClientContextId());
             }
             clientRequest.setJobId(jobId);
             if (jId != null) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
index 156b78ae93..d8356643a3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
@@ -26,6 +26,8 @@ import java.util.regex.Pattern;
 
 import org.apache.hyracks.util.ILogRedactor;
 
+import com.fasterxml.jackson.databind.node.TextNode;
+
 public class RedactionUtil {
     private RedactionUtil() {
         throw new AssertionError("do not instantiate");
@@ -34,6 +36,7 @@ public class RedactionUtil {
     private static final Pattern STATEMENT_PATTERN =
             Pattern.compile("(" + SECRET_ACCESS_KEY_FIELD_NAME + ").*", CASE_INSENSITIVE | DOTALL);
     private static final String STATEMENT_REPLACEMENT = "$1...<redacted sensitive data>";
+    public static final TextNode REDACTED_SENSITIVE_ENTRY_VALUE = new TextNode("<redacted sensitive entry>");
 
     public static final ILogRedactor LOG_REDACTOR = new ILogRedactor() {
         @Override