You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/07/19 22:10:47 UTC
[asterixdb] 04/07: [NO ISSUE] Redact sensitive data from statement
logging
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b44f5c3cd4802cd215f51dcd7698370a8e45ea78
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Thu Jul 9 17:01:03 2020 -0400
[NO ISSUE] Redact sensitive data from statement logging
Change-Id: Ibd63ca9167c769eea4d03982dbf7fa543913dc67
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/7165
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../api/http/server/QueryServiceServlet.java | 7 ++--
.../message/ExecuteStatementRequestMessage.java | 2 +-
.../asterix/hyracks/bootstrap/CCApplication.java | 3 ++
.../asterix/hyracks/bootstrap/NCApplication.java | 3 ++
.../org/apache/asterix/utils/RedactionUtil.java | 41 ++++++++++++----------
.../java/org/apache/hyracks/util/ILogRedactor.java | 8 +++++
.../org/apache/hyracks/util/LogRedactionUtil.java | 9 +++++
7 files changed, 50 insertions(+), 23 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index cb1d6cf..440b351 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -279,7 +279,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
if (forceReadOnly) {
param.setReadOnly(true);
}
- LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.userData(param.toString()));
+ LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.statement(param.toString()));
delivery = param.getMode();
setSessionConfig(sessionOutput, param, delivery);
final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads());
@@ -427,10 +427,11 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
if (t instanceof org.apache.asterix.aqlplus.parser.TokenMgrError || t instanceof TokenMgrError
|| t instanceof AlgebricksException) {
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.userData(param.toString()), t);
+ LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.statement(param.toString()),
+ t);
} else {
LOGGER.info(() -> "handleException: " + t.getMessage() + ": "
- + LogRedactionUtil.userData(param.toString()));
+ + LogRedactionUtil.statement(param.toString()));
}
executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
} else if (t instanceof HyracksException) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 149ed33..2eced12 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -204,6 +204,6 @@ public final class ExecuteStatementRequestMessage implements ICcAddressedMessage
@Override
public String toString() {
return String.format("%s(id=%s, from=%s): %s", getClass().getSimpleName(), requestMessageId, requestNodeId,
- LogRedactionUtil.userData(statementsText));
+ LogRedactionUtil.statement(statementsText));
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index fc912b0..0f0620c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -86,6 +86,7 @@ import org.apache.asterix.runtime.utils.CcApplicationContext;
import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.asterix.translator.Receptionist;
import org.apache.asterix.util.MetadataBuiltinFunctions;
+import org.apache.asterix.utils.RedactionUtil;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.client.IHyracksClientConnection;
@@ -104,6 +105,7 @@ import org.apache.hyracks.http.server.HttpServerConfig;
import org.apache.hyracks.http.server.HttpServerConfigBuilder;
import org.apache.hyracks.http.server.WebManager;
import org.apache.hyracks.ipc.impl.HyracksConnection;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.LoggingConfigUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -204,6 +206,7 @@ public class CCApplication extends BaseCCApplication {
public void configureLoggingLevel(Level level) {
super.configureLoggingLevel(level);
LoggingConfigUtil.defaultIfMissing(GlobalConfig.ASTERIX_LOGGER_NAME, level);
+ LogRedactionUtil.setRedactor(RedactionUtil.LOG_REDACTOR);
}
protected List<AsterixExtension> getExtensions() throws Exception {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 2e5c09c..1036fb2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -72,6 +72,7 @@ import org.apache.asterix.messaging.NCMessageBroker;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.asterix.translator.Receptionist;
import org.apache.asterix.util.MetadataBuiltinFunctions;
+import org.apache.asterix.utils.RedactionUtil;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.client.NodeStatus;
import org.apache.hyracks.api.config.IConfigManager;
@@ -86,6 +87,7 @@ import org.apache.hyracks.http.server.HttpServer;
import org.apache.hyracks.http.server.HttpServerConfig;
import org.apache.hyracks.http.server.HttpServerConfigBuilder;
import org.apache.hyracks.http.server.WebManager;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.LoggingConfigUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -184,6 +186,7 @@ public class NCApplication extends BaseNCApplication {
public void configureLoggingLevel(Level level) {
super.configureLoggingLevel(level);
LoggingConfigUtil.defaultIfMissing(GlobalConfig.ASTERIX_LOGGER_NAME, level);
+ LogRedactionUtil.setRedactor(RedactionUtil.LOG_REDACTOR);
}
protected void configureServers() throws Exception {
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
similarity index 53%
copy from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java
copy to asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
index 89c957e..156b78a 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
@@ -16,36 +16,39 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.asterix.utils;
-package org.apache.hyracks.util;
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
+import static java.util.regex.Pattern.DOTALL;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
-public class LogRedactionUtil {
+import java.util.regex.Pattern;
- private static final ILogRedactor DEFAULT_LOG_REDACTOR = new ILogRedactor() {
+import org.apache.hyracks.util.ILogRedactor;
+
+public class RedactionUtil {
+ private RedactionUtil() {
+ throw new AssertionError("do not instantiate");
+ }
+
+ private static final Pattern STATEMENT_PATTERN =
+ Pattern.compile("(" + SECRET_ACCESS_KEY_FIELD_NAME + ").*", CASE_INSENSITIVE | DOTALL);
+ private static final String STATEMENT_REPLACEMENT = "$1...<redacted sensitive data>";
+
+ public static final ILogRedactor LOG_REDACTOR = new ILogRedactor() {
@Override
public String userData(String text) {
return text;
}
@Override
+ public String statement(String text) {
+ return STATEMENT_PATTERN.matcher(text).replaceFirst(STATEMENT_REPLACEMENT);
+ }
+
+ @Override
public String unredactUserData(String text) {
return text;
}
};
- private static ILogRedactor redactor = DEFAULT_LOG_REDACTOR;
-
- private LogRedactionUtil() {
- }
-
- public static void setRedactor(ILogRedactor redactor) {
- LogRedactionUtil.redactor = redactor;
- }
-
- public static String userData(String text) {
- return redactor.userData(text);
- }
-
- public static String unredactUserData(String text) {
- return redactor.unredactUserData(text);
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java
index b133894..d36c77b 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java
@@ -29,6 +29,14 @@ public interface ILogRedactor {
String userData(String text);
/**
+ * Redacts statement argument.
+ *
+ * @param text statement to redact.
+ * @return redacted statement.
+ */
+ String statement(String text);
+
+ /**
* Unredacts user data found in the argument.
*
* @param text text that contains some redacted user data.
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java
index 89c957e..fdce8c8 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java
@@ -28,6 +28,11 @@ public class LogRedactionUtil {
}
@Override
+ public String statement(String text) {
+ return text;
+ }
+
+ @Override
public String unredactUserData(String text) {
return text;
}
@@ -45,6 +50,10 @@ public class LogRedactionUtil {
return redactor.userData(text);
}
+ public static String statement(String text) {
+ return redactor.statement(text);
+ }
+
public static String unredactUserData(String text) {
return redactor.unredactUserData(text);
}