You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by re...@apache.org on 2015/11/23 12:57:14 UTC
[1/2] git commit: updated refs/heads/master to a2a7288
Repository: cloudstack
Updated Branches:
refs/heads/master 36d0bfdec -> a2a72887d
CLOUDSTACK-8485: listAPIs are taking too long to return results
- Removed regex. based search/replace of sensitive data on API response introduced as part of commit b0c6d4734724358df97b6fa4d8c5beb0f447745e
- Added new response serializer to skip sensitive data from getting logged based on annotation present in resposne object fields
- Added new parameter 'isSensitive' to @Param for marking a field as sensitive in response objects
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/e13df963
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/e13df963
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/e13df963
Branch: refs/heads/master
Commit: e13df96348c60694849f91d0c025cb349aea124e
Parents: acce645
Author: Koushik Das <ko...@apache.org>
Authored: Mon Nov 2 14:59:00 2015 +0530
Committer: Koushik Das <ko...@apache.org>
Committed: Fri Nov 20 16:40:53 2015 +0530
----------------------------------------------------------------------
api/src/com/cloud/serializer/Param.java | 2 +
.../api/response/CreateSSHKeyPairResponse.java | 3 +-
.../api/response/GetVMPasswordResponse.java | 2 +-
.../api/response/LoginCmdResponse.java | 2 +-
.../api/response/RegisterResponse.java | 4 +-
.../api/response/RemoteAccessVpnResponse.java | 2 +-
.../Site2SiteCustomerGatewayResponse.java | 2 +-
.../Site2SiteVpnConnectionResponse.java | 2 +-
.../cloudstack/api/response/UserResponse.java | 4 +-
.../cloudstack/api/response/UserVmResponse.java | 2 +-
.../response/BigSwitchBcfDeviceResponse.java | 2 +-
.../api/response/LDAPConfigResponse.java | 2 +-
.../com/cloud/api/ApiResponseGsonHelper.java | 33 ++++-
server/src/com/cloud/api/ApiServer.java | 15 +-
server/src/com/cloud/api/ApiServlet.java | 5 +-
.../api/response/ApiResponseSerializer.java | 148 +++++++++++++------
16 files changed, 153 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/com/cloud/serializer/Param.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/serializer/Param.java b/api/src/com/cloud/serializer/Param.java
index 3e6f852..26a848a 100644
--- a/api/src/com/cloud/serializer/Param.java
+++ b/api/src/com/cloud/serializer/Param.java
@@ -37,4 +37,6 @@ public @interface Param {
String since() default "";
RoleType[] authorized() default {};
+
+ boolean isSensitive() default false;
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/CreateSSHKeyPairResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/CreateSSHKeyPairResponse.java b/api/src/org/apache/cloudstack/api/response/CreateSSHKeyPairResponse.java
index 4ed1b53..410719f 100644
--- a/api/src/org/apache/cloudstack/api/response/CreateSSHKeyPairResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/CreateSSHKeyPairResponse.java
@@ -17,13 +17,12 @@
package org.apache.cloudstack.api.response;
import com.google.gson.annotations.SerializedName;
-
import com.cloud.serializer.Param;
public class CreateSSHKeyPairResponse extends SSHKeyPairResponse {
@SerializedName("privatekey")
- @Param(description = "Private key")
+ @Param(description = "Private key", isSensitive = true)
private String privateKey;
public CreateSSHKeyPairResponse() {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/GetVMPasswordResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/GetVMPasswordResponse.java b/api/src/org/apache/cloudstack/api/response/GetVMPasswordResponse.java
index 7a4543d..4960321 100644
--- a/api/src/org/apache/cloudstack/api/response/GetVMPasswordResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/GetVMPasswordResponse.java
@@ -25,7 +25,7 @@ import com.cloud.serializer.Param;
public class GetVMPasswordResponse extends BaseResponse {
@SerializedName("encryptedpassword")
- @Param(description = "The base64 encoded encrypted password of the VM")
+ @Param(description = "The base64 encoded encrypted password of the VM", isSensitive = true)
private String encryptedPassword;
public GetVMPasswordResponse() {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/LoginCmdResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/LoginCmdResponse.java b/api/src/org/apache/cloudstack/api/response/LoginCmdResponse.java
index 3139b6f..55eb2c4 100644
--- a/api/src/org/apache/cloudstack/api/response/LoginCmdResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/LoginCmdResponse.java
@@ -63,7 +63,7 @@ public class LoginCmdResponse extends AuthenticationCmdResponse {
private String registered;
@SerializedName(value = ApiConstants.SESSIONKEY)
- @Param(description = "Session key that can be passed in subsequent Query command calls")
+ @Param(description = "Session key that can be passed in subsequent Query command calls", isSensitive = true)
private String sessionKey;
public String getUsername() {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/RegisterResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/RegisterResponse.java b/api/src/org/apache/cloudstack/api/response/RegisterResponse.java
index fd944b0..5faedab 100644
--- a/api/src/org/apache/cloudstack/api/response/RegisterResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/RegisterResponse.java
@@ -24,11 +24,11 @@ import com.cloud.serializer.Param;
public class RegisterResponse extends BaseResponse {
@SerializedName("apikey")
- @Param(description = "the api key of the registered user")
+ @Param(description = "the api key of the registered user", isSensitive = true)
private String apiKey;
@SerializedName("secretkey")
- @Param(description = "the secret key of the registered user")
+ @Param(description = "the secret key of the registered user", isSensitive = true)
private String secretKey;
public String getApiKey() {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/RemoteAccessVpnResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/RemoteAccessVpnResponse.java b/api/src/org/apache/cloudstack/api/response/RemoteAccessVpnResponse.java
index 28d788b..0e078be 100644
--- a/api/src/org/apache/cloudstack/api/response/RemoteAccessVpnResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/RemoteAccessVpnResponse.java
@@ -42,7 +42,7 @@ public class RemoteAccessVpnResponse extends BaseResponse implements ControlledE
private String ipRange;
@SerializedName("presharedkey")
- @Param(description = "the ipsec preshared key")
+ @Param(description = "the ipsec preshared key", isSensitive = true)
private String presharedKey;
@SerializedName(ApiConstants.ACCOUNT)
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/Site2SiteCustomerGatewayResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/Site2SiteCustomerGatewayResponse.java b/api/src/org/apache/cloudstack/api/response/Site2SiteCustomerGatewayResponse.java
index 8a6ccd0..2bda8f9 100644
--- a/api/src/org/apache/cloudstack/api/response/Site2SiteCustomerGatewayResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/Site2SiteCustomerGatewayResponse.java
@@ -51,7 +51,7 @@ public class Site2SiteCustomerGatewayResponse extends BaseResponse implements Co
private String guestCidrList;
@SerializedName(ApiConstants.IPSEC_PSK)
- @Param(description = "IPsec preshared-key of customer gateway")
+ @Param(description = "IPsec preshared-key of customer gateway", isSensitive = true)
private String ipsecPsk;
@SerializedName(ApiConstants.IKE_POLICY)
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/Site2SiteVpnConnectionResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/Site2SiteVpnConnectionResponse.java b/api/src/org/apache/cloudstack/api/response/Site2SiteVpnConnectionResponse.java
index 4c57256..c00a4d4 100644
--- a/api/src/org/apache/cloudstack/api/response/Site2SiteVpnConnectionResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/Site2SiteVpnConnectionResponse.java
@@ -58,7 +58,7 @@ public class Site2SiteVpnConnectionResponse extends BaseResponse implements Cont
private String guestCidrList;
@SerializedName(ApiConstants.IPSEC_PSK)
- @Param(description = "IPsec Preshared-Key of the customer gateway")
+ @Param(description = "IPsec Preshared-Key of the customer gateway", isSensitive = true)
//from CustomerGateway
private String ipsecPsk;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/UserResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/UserResponse.java b/api/src/org/apache/cloudstack/api/response/UserResponse.java
index 40e1561..36611ae 100644
--- a/api/src/org/apache/cloudstack/api/response/UserResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/UserResponse.java
@@ -78,11 +78,11 @@ public class UserResponse extends BaseResponse {
private String timezone;
@SerializedName("apikey")
- @Param(description = "the api key of the user")
+ @Param(description = "the api key of the user", isSensitive = true)
private String apiKey;
@SerializedName("secretkey")
- @Param(description = "the secret key of the user")
+ @Param(description = "the secret key of the user", isSensitive = true)
private String secretKey;
@SerializedName("accountid")
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/api/src/org/apache/cloudstack/api/response/UserVmResponse.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/org/apache/cloudstack/api/response/UserVmResponse.java
index 36d82de..e8d1a9e 100644
--- a/api/src/org/apache/cloudstack/api/response/UserVmResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/UserVmResponse.java
@@ -221,7 +221,7 @@ public class UserVmResponse extends BaseResponse implements ControlledEntityResp
private Set<SecurityGroupResponse> securityGroupList;
@SerializedName(ApiConstants.PASSWORD)
- @Param(description = "the password (if exists) of the virtual machine")
+ @Param(description = "the password (if exists) of the virtual machine", isSensitive = true)
private String password;
@SerializedName("nic")
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/plugins/network-elements/bigswitch/src/com/cloud/api/response/BigSwitchBcfDeviceResponse.java
----------------------------------------------------------------------
diff --git a/plugins/network-elements/bigswitch/src/com/cloud/api/response/BigSwitchBcfDeviceResponse.java b/plugins/network-elements/bigswitch/src/com/cloud/api/response/BigSwitchBcfDeviceResponse.java
index 1d1fe44..e93f99d 100644
--- a/plugins/network-elements/bigswitch/src/com/cloud/api/response/BigSwitchBcfDeviceResponse.java
+++ b/plugins/network-elements/bigswitch/src/com/cloud/api/response/BigSwitchBcfDeviceResponse.java
@@ -54,7 +54,7 @@ public class BigSwitchBcfDeviceResponse extends BaseResponse {
@SerializedName(ApiConstants.USERNAME) @Param(description="the controller username")
private String username;
- @SerializedName(ApiConstants.PASSWORD) @Param(description="the controller password")
+ @SerializedName(ApiConstants.PASSWORD) @Param(description="the controller password", isSensitive = true)
private String password;
@SerializedName(BcfConstants.BIGSWITCH_BCF_DEVICE_NAT)
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/response/LDAPConfigResponse.java
----------------------------------------------------------------------
diff --git a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/response/LDAPConfigResponse.java b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/response/LDAPConfigResponse.java
index 8570bac..0f06e78 100644
--- a/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/response/LDAPConfigResponse.java
+++ b/plugins/user-authenticators/ldap/src/org/apache/cloudstack/api/response/LDAPConfigResponse.java
@@ -53,7 +53,7 @@ public class LDAPConfigResponse extends BaseResponse {
private String bindDN;
@SerializedName(ApiConstants.BIND_PASSWORD)
- @Param(description = "DN password")
+ @Param(description = "DN password", isSensitive = true)
private String bindPassword;
public String getHostname() {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/server/src/com/cloud/api/ApiResponseGsonHelper.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/ApiResponseGsonHelper.java b/server/src/com/cloud/api/ApiResponseGsonHelper.java
index c2cc9d9..1708fa7 100644
--- a/server/src/com/cloud/api/ApiResponseGsonHelper.java
+++ b/server/src/com/cloud/api/ApiResponseGsonHelper.java
@@ -27,30 +27,40 @@ import com.google.gson.FieldAttributes;
import com.google.gson.GsonBuilder;
/**
- * The ApiResonseGsonHelper is different from ApiGsonHelper - it registeres one more adapter for String type required for api response encoding
+ * The ApiResonseGsonHelper is different from ApiGsonHelper - it registers one more adapter for String type required for api response encoding
*/
public class ApiResponseGsonHelper {
private static final GsonBuilder s_gBuilder;
+ private static final GsonBuilder s_gLogBuilder;
static {
s_gBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
s_gBuilder.setVersion(1.3);
s_gBuilder.registerTypeAdapter(ResponseObject.class, new ResponseObjectTypeAdapter());
s_gBuilder.registerTypeAdapter(String.class, new EncodedStringTypeAdapter());
- s_gBuilder.setExclusionStrategies(new ExclStrat());
+ s_gBuilder.setExclusionStrategies(new ApiResponseExclusionStrategy());
+
+ s_gLogBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
+ s_gLogBuilder.setVersion(1.3);
+ s_gLogBuilder.registerTypeAdapter(ResponseObject.class, new ResponseObjectTypeAdapter());
+ s_gLogBuilder.registerTypeAdapter(String.class, new EncodedStringTypeAdapter());
+ s_gLogBuilder.setExclusionStrategies(new LogExclusionStrategy());
}
public static GsonBuilder getBuilder() {
return s_gBuilder;
}
- private static class ExclStrat implements ExclusionStrategy {
+ public static GsonBuilder getLogBuilder() {
+ return s_gLogBuilder;
+ }
+ private static class ApiResponseExclusionStrategy implements ExclusionStrategy {
public boolean shouldSkipClass(Class<?> arg0) {
return false;
}
- public boolean shouldSkipField(FieldAttributes f) {
+ public boolean shouldSkipField(FieldAttributes f) {
Param param = f.getAnnotation(Param.class);
if (param != null) {
RoleType[] allowedRoles = param.authorized();
@@ -71,4 +81,19 @@ public class ApiResponseGsonHelper {
return false;
}
}
+
+ private static class LogExclusionStrategy extends ApiResponseExclusionStrategy implements ExclusionStrategy {
+ public boolean shouldSkipClass(Class<?> arg0) {
+ return false;
+ }
+
+ public boolean shouldSkipField(FieldAttributes f) {
+ Param param = f.getAnnotation(Param.class);
+ boolean skip = (param != null && param.isSensitive());
+ if (!skip) {
+ skip = super.shouldSkipField(f);
+ }
+ return skip;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/server/src/com/cloud/api/ApiServer.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java
index 1459dc2..e5ae097 100644
--- a/server/src/com/cloud/api/ApiServer.java
+++ b/server/src/com/cloud/api/ApiServer.java
@@ -525,14 +525,9 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
cmdObj.setHttpMethod(paramMap.get(ApiConstants.HTTPMETHOD).toString());
// This is where the command is either serialized, or directly dispatched
- response = queueCommand(cmdObj, paramMap);
- if (annotation.responseHasSensitiveInfo())
- {
- buildAuditTrail(auditTrailSb, command[0],
- StringUtils.cleanString(response));
- }
- else
- buildAuditTrail(auditTrailSb, command[0], response);
+ StringBuilder log = new StringBuilder();
+ response = queueCommand(cmdObj, paramMap, log);
+ buildAuditTrail(auditTrailSb, command[0], log.toString());
} else {
final String errorString = "Unknown API command: " + command[0];
s_logger.warn(errorString);
@@ -617,7 +612,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
return ApiResponseSerializer.toSerializedString(response, cmd.getResponseType());
}
- private String queueCommand(final BaseCmd cmdObj, final Map<String, String> params) throws Exception {
+ private String queueCommand(final BaseCmd cmdObj, final Map<String, String> params, StringBuilder log) throws Exception {
final CallContext ctx = CallContext.current();
final Long callerUserId = ctx.getCallingUserId();
final Account caller = ctx.getCallingAccount();
@@ -717,7 +712,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
}
SerializationContext.current().setUuidTranslation(true);
- return ApiResponseSerializer.toSerializedString((ResponseObject)cmdObj.getResponseObject(), cmdObj.getResponseType());
+ return ApiResponseSerializer.toSerializedStringWithSecureLogs((ResponseObject)cmdObj.getResponseObject(), cmdObj.getResponseType(), log);
}
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/server/src/com/cloud/api/ApiServlet.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/ApiServlet.java b/server/src/com/cloud/api/ApiServlet.java
index 4322154..51827da 100644
--- a/server/src/com/cloud/api/ApiServlet.java
+++ b/server/src/com/cloud/api/ApiServlet.java
@@ -147,8 +147,9 @@ public class ApiServlet extends HttpServlet {
// logging the request start and end in management log for easy debugging
String reqStr = "";
+ String cleanQueryString = StringUtils.cleanString(req.getQueryString());
if (s_logger.isDebugEnabled()) {
- reqStr = auditTrailSb.toString() + " " + StringUtils.cleanString(req.getQueryString());
+ reqStr = auditTrailSb.toString() + " " + cleanQueryString;
s_logger.debug("===START=== " + reqStr);
}
@@ -233,7 +234,7 @@ public class ApiServlet extends HttpServlet {
}
}
- auditTrailSb.append(StringUtils.cleanString(req.getQueryString()));
+ auditTrailSb.append(cleanQueryString);
final boolean isNew = ((session == null) ? true : session.isNew());
// Initialize an empty context and we will update it after we have verified the request below,
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e13df963/server/src/com/cloud/api/response/ApiResponseSerializer.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/api/response/ApiResponseSerializer.java b/server/src/com/cloud/api/response/ApiResponseSerializer.java
index 19d2671..59426e6 100644
--- a/server/src/com/cloud/api/response/ApiResponseSerializer.java
+++ b/server/src/com/cloud/api/response/ApiResponseSerializer.java
@@ -27,6 +27,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.exception.ExceptionProxyObject;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
+
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseCmd;
@@ -56,9 +57,18 @@ public class ApiResponseSerializer {
public static String toSerializedString(ResponseObject result, String responseType) {
s_logger.trace("===Serializing Response===");
if (HttpUtils.RESPONSE_TYPE_JSON.equalsIgnoreCase(responseType)) {
- return toJSONSerializedString(result);
+ return toJSONSerializedString(result, new StringBuilder());
+ } else {
+ return toXMLSerializedString(result, new StringBuilder());
+ }
+ }
+
+ public static String toSerializedStringWithSecureLogs(ResponseObject result, String responseType, StringBuilder log) {
+ s_logger.trace("===Serializing Response===");
+ if (HttpUtils.RESPONSE_TYPE_JSON.equalsIgnoreCase(responseType)) {
+ return toJSONSerializedString(result, log);
} else {
- return toXMLSerializedString(result);
+ return toXMLSerializedString(result, log);
}
}
@@ -73,51 +83,65 @@ public class ApiResponseSerializer {
return str;
}
- public static String toJSONSerializedString(ResponseObject result) {
- if (result != null) {
- Gson gson = ApiResponseGsonHelper.getBuilder().excludeFieldsWithModifiers(Modifier.TRANSIENT).create();
+ public static String toJSONSerializedString(ResponseObject result, StringBuilder log) {
+ if (result != null && log != null) {
+ Gson responseBuilder = ApiResponseGsonHelper.getBuilder().excludeFieldsWithModifiers(Modifier.TRANSIENT).create();
+ Gson logBuilder = ApiResponseGsonHelper.getLogBuilder().excludeFieldsWithModifiers(Modifier.TRANSIENT).create();
StringBuilder sb = new StringBuilder();
sb.append("{\"").append(result.getResponseName()).append("\":");
+ log.append("{\"").append(result.getResponseName()).append("\":");
if (result instanceof ListResponse) {
List<? extends ResponseObject> responses = ((ListResponse)result).getResponses();
Integer count = ((ListResponse)result).getCount();
boolean nonZeroCount = (count != null && count.longValue() != 0);
if (nonZeroCount) {
sb.append("{\"").append(ApiConstants.COUNT).append("\":").append(count);
+ log.append("{\"").append(ApiConstants.COUNT).append("\":").append(count);
}
if ((responses != null) && !responses.isEmpty()) {
- String jsonStr = gson.toJson(responses.get(0));
+ String jsonStr = responseBuilder.toJson(responses.get(0));
jsonStr = unescape(jsonStr);
+ String logStr = logBuilder.toJson(responses.get(0));
+ logStr = unescape(logStr);
if (nonZeroCount) {
sb.append(",\"").append(responses.get(0).getObjectName()).append("\":[").append(jsonStr);
+ log.append(",\"").append(responses.get(0).getObjectName()).append("\":[").append(logStr);
}
for (int i = 1; i < ((ListResponse)result).getResponses().size(); i++) {
- jsonStr = gson.toJson(responses.get(i));
+ jsonStr = responseBuilder.toJson(responses.get(i));
jsonStr = unescape(jsonStr);
+ logStr = logBuilder.toJson(responses.get(i));
+ logStr = unescape(logStr);
sb.append(",").append(jsonStr);
+ log.append(",").append(logStr);
}
sb.append("]}");
+ log.append("]}");
} else {
if (!nonZeroCount) {
sb.append("{");
+ log.append("{");
}
sb.append("}");
+ log.append("}");
}
} else if (result instanceof SuccessResponse) {
sb.append("{\"success\":\"").append(((SuccessResponse)result).getSuccess()).append("\"}");
+ log.append("{\"success\":\"").append(((SuccessResponse)result).getSuccess()).append("\"}");
} else if (result instanceof ExceptionResponse) {
- String jsonErrorText = gson.toJson(result);
+ String jsonErrorText = responseBuilder.toJson(result);
jsonErrorText = unescape(jsonErrorText);
sb.append(jsonErrorText);
+ log.append(jsonErrorText);
} else {
- String jsonStr = gson.toJson(result);
- if ((jsonStr != null) && !"".equals(jsonStr)) {
+ String jsonStr = responseBuilder.toJson(result);
+ if (jsonStr != null && !jsonStr.isEmpty()) {
jsonStr = unescape(jsonStr);
if (result instanceof AsyncJobResponse || result instanceof CreateCmdResponse || result instanceof AuthenticationCmdResponse) {
sb.append(jsonStr);
@@ -127,53 +151,74 @@ public class ApiResponseSerializer {
} else {
sb.append("{}");
}
+ String logStr = logBuilder.toJson(result);
+ if (logStr != null && !logStr.isEmpty()) {
+ logStr = unescape(logStr);
+ if (result instanceof AsyncJobResponse || result instanceof CreateCmdResponse || result instanceof AuthenticationCmdResponse) {
+ log.append(logStr);
+ } else {
+ log.append("{\"").append(result.getObjectName()).append("\":").append(logStr).append("}");
+ }
+ } else {
+ log.append("{}");
+ }
}
sb.append("}");
+ log.append("}");
return sb.toString();
}
return null;
}
- private static String toXMLSerializedString(ResponseObject result) {
- StringBuilder sb = new StringBuilder();
- sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
- sb.append("<").append(result.getResponseName()).append(" cloud-stack-version=\"").append(ApiDBUtils.getVersion()).append("\">");
+ private static String toXMLSerializedString(ResponseObject result, StringBuilder log) {
+ if (result != null && log != null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+ sb.append("<").append(result.getResponseName()).append(" cloud-stack-version=\"").append(ApiDBUtils.getVersion()).append("\">");
+ log.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+ log.append("<").append(result.getResponseName()).append(" cloud-stack-version=\"").append(ApiDBUtils.getVersion()).append("\">");
- if (result instanceof ListResponse) {
- Integer count = ((ListResponse)result).getCount();
+ if (result instanceof ListResponse) {
+ Integer count = ((ListResponse)result).getCount();
- if (count != null && count != 0) {
- sb.append("<").append(ApiConstants.COUNT).append(">").append(((ListResponse)result).getCount()).append("</").append(ApiConstants.COUNT).append(">");
- }
- List<? extends ResponseObject> responses = ((ListResponse)result).getResponses();
- if ((responses != null) && !responses.isEmpty()) {
- for (ResponseObject obj : responses) {
- serializeResponseObjXML(sb, obj);
+ if (count != null && count != 0) {
+ sb.append("<").append(ApiConstants.COUNT).append(">").append(((ListResponse)result).getCount()).append("</").append(ApiConstants.COUNT).append(">");
+ log.append("<").append(ApiConstants.COUNT).append(">").append(((ListResponse)result).getCount()).append("</").append(ApiConstants.COUNT).append(">");
+ }
+ List<? extends ResponseObject> responses = ((ListResponse)result).getResponses();
+ if ((responses != null) && !responses.isEmpty()) {
+ for (ResponseObject obj : responses) {
+ serializeResponseObjXML(sb, log, obj);
+ }
}
- }
- } else {
- if (result instanceof CreateCmdResponse || result instanceof AsyncJobResponse || result instanceof AuthenticationCmdResponse) {
- serializeResponseObjFieldsXML(sb, result);
} else {
- serializeResponseObjXML(sb, result);
+ if (result instanceof CreateCmdResponse || result instanceof AsyncJobResponse || result instanceof AuthenticationCmdResponse) {
+ serializeResponseObjFieldsXML(sb, log, result);
+ } else {
+ serializeResponseObjXML(sb, log, result);
+ }
}
- }
- sb.append("</").append(result.getResponseName()).append(">");
- return sb.toString();
+ sb.append("</").append(result.getResponseName()).append(">");
+ log.append("</").append(result.getResponseName()).append(">");
+ return sb.toString();
+ }
+ return null;
}
- private static void serializeResponseObjXML(StringBuilder sb, ResponseObject obj) {
+ private static void serializeResponseObjXML(StringBuilder sb, StringBuilder log, ResponseObject obj) {
if (!(obj instanceof SuccessResponse) && !(obj instanceof ExceptionResponse)) {
sb.append("<").append(obj.getObjectName()).append(">");
+ log.append("<").append(obj.getObjectName()).append(">");
}
- serializeResponseObjFieldsXML(sb, obj);
+ serializeResponseObjFieldsXML(sb, log, obj);
if (!(obj instanceof SuccessResponse) && !(obj instanceof ExceptionResponse)) {
sb.append("</").append(obj.getObjectName()).append(">");
+ log.append("</").append(obj.getObjectName()).append(">");
}
}
- public static Field[] getFlattenFields(Class<?> clz) {
+ private static Field[] getFlattenFields(Class<?> clz) {
List<Field> fields = new ArrayList<Field>();
fields.addAll(Arrays.asList(clz.getDeclaredFields()));
if (clz.getSuperclass() != null) {
@@ -182,24 +227,23 @@ public class ApiResponseSerializer {
return fields.toArray(new Field[] {});
}
- private static void serializeResponseObjFieldsXML(StringBuilder sb, ResponseObject obj) {
+ private static void serializeResponseObjFieldsXML(StringBuilder sb, StringBuilder log, ResponseObject obj) {
boolean isAsync = false;
if (obj instanceof AsyncJobResponse)
isAsync = true;
- //Field[] fields = obj.getClass().getDeclaredFields();
Field[] fields = getFlattenFields(obj.getClass());
for (Field field : fields) {
if ((field.getModifiers() & Modifier.TRANSIENT) != 0) {
continue; // skip transient fields
}
-
SerializedName serializedName = field.getAnnotation(SerializedName.class);
if (serializedName == null) {
continue; // skip fields w/o serialized name
}
+ boolean logField = true;
Param param = field.getAnnotation(Param.class);
if (param != null) {
RoleType[] allowedRoles = param.authorized();
@@ -213,10 +257,13 @@ public class ApiResponseSerializer {
}
}
if (!permittedParameter) {
- s_logger.trace("Ignoring paremeter " + param.name() + " as the caller is not authorized to see it");
+ s_logger.trace("Ignoring parameter " + param.name() + " as the caller is not authorized to see it");
continue;
}
}
+ if (param.isSensitive()) {
+ logField = false;
+ }
}
field.setAccessible(true);
@@ -233,10 +280,12 @@ public class ApiResponseSerializer {
ResponseObject subObj = (ResponseObject)fieldValue;
if (isAsync) {
sb.append("<jobresult>");
+ log.append("<jobresult>");
}
- serializeResponseObjXML(sb, subObj);
+ serializeResponseObjXML(sb, log, subObj);
if (isAsync) {
sb.append("</jobresult>");
+ log.append("</jobresult>");
}
} else if (fieldValue instanceof Collection<?>) {
Collection<?> subResponseList = (Collection<?>)fieldValue;
@@ -247,7 +296,7 @@ public class ApiResponseSerializer {
if (serializedName != null) {
subObj.setObjectName(serializedName.value());
}
- serializeResponseObjXML(sb, subObj);
+ serializeResponseObjXML(sb, log, subObj);
} else if (value instanceof ExceptionProxyObject) {
// Only exception reponses carry a list of
// ExceptionProxyObject objects.
@@ -256,30 +305,32 @@ public class ApiResponseSerializer {
// encountered, put in a uuidList tag.
if (!usedUuidList) {
sb.append("<" + serializedName.value() + ">");
+ log.append("<" + serializedName.value() + ">");
usedUuidList = true;
}
sb.append("<" + "uuid" + ">" + idProxy.getUuid() + "</" + "uuid" + ">");
+ log.append("<" + "uuid" + ">" + idProxy.getUuid() + "</" + "uuid" + ">");
// Append the new descriptive property also.
String idFieldName = idProxy.getDescription();
if (idFieldName != null) {
sb.append("<" + "uuidProperty" + ">" + idFieldName + "</" + "uuidProperty" + ">");
+ log.append("<" + "uuidProperty" + ">" + idFieldName + "</" + "uuidProperty" + ">");
}
} else if (value instanceof String) {
sb.append("<").append(serializedName.value()).append(">").append(value).append("</").append(serializedName.value()).append(">");
+ if (logField) {
+ log.append("<").append(serializedName.value()).append(">").append(value).append("</").append(serializedName.value()).append(">");
+ }
}
}
if (usedUuidList) {
// close the uuidList.
sb.append("</").append(serializedName.value()).append(">");
+ log.append("</").append(serializedName.value()).append(">");
}
} else if (fieldValue instanceof Date) {
- sb.append("<")
- .append(serializedName.value())
- .append(">")
- .append(BaseCmd.getDateString((Date)fieldValue))
- .append("</")
- .append(serializedName.value())
- .append(">");
+ sb.append("<").append(serializedName.value()).append(">").append(BaseCmd.getDateString((Date)fieldValue)).append("</").append(serializedName.value()).append(">");
+ log.append("<").append(serializedName.value()).append(">").append(BaseCmd.getDateString((Date)fieldValue)).append("</").append(serializedName.value()).append(">");
} else {
String resultString = escapeSpecialXmlChars(fieldValue.toString());
if (!(obj instanceof ExceptionResponse)) {
@@ -287,6 +338,9 @@ public class ApiResponseSerializer {
}
sb.append("<").append(serializedName.value()).append(">").append(resultString).append("</").append(serializedName.value()).append(">");
+ if (logField) {
+ log.append("<").append(serializedName.value()).append(">").append(resultString).append("</").append(serializedName.value()).append(">");
+ }
}
}
}
[2/2] git commit: updated refs/heads/master to a2a7288
Posted by re...@apache.org.
Merge pull request #1021 from koushik-das/CLOUDSTACK-8485
CLOUDSTACK-8485: listAPIs are taking too long to return results- Removed regex. based search/replace of sensitive data on API response introduced as part of commit b0c6d4734724358df97b6fa4d8c5beb0f447745e
- Added new response serializer to skip sensitive data from getting logged based on annotation present in resposne object fields
- Added annotation (@LogLevel(Log4jLevel.Off)) to sensitive response object fields
Ran the following tests on simulator:
test_vm_life_cycle.py
Test advanced zone virtual router ... === TestName: test_advZoneVirtualRouter | Status : SUCCESS ===
ok
Test Deploy Virtual Machine ... === TestName: test_deploy_vm | Status : SUCCESS ===
ok
Test Multiple Deploy Virtual Machine ... === TestName: test_deploy_vm_multiple | Status : SUCCESS ===
ok
Test Stop Virtual Machine ... === TestName: test_01_stop_vm | Status : SUCCESS ===
ok
Test Start Virtual Machine ... === TestName: test_02_start_vm | Status : SUCCESS ===
ok
Test Reboot Virtual Machine ... === TestName: test_03_reboot_vm | Status : SUCCESS ===
ok
Test destroy Virtual Machine ... === TestName: test_06_destroy_vm | Status : SUCCESS ===
ok
Test recover Virtual Machine ... === TestName: test_07_restore_vm | Status : SUCCESS ===
ok
Test migrate VM ... === TestName: test_08_migrate_vm | Status : SUCCESS ===
ok
Test destroy(expunge) Virtual Machine ... === TestName: test_09_expunge_vm | Status : SUCCESS ===
ok
----------------------------------------------------------------------
Ran 10 tests in 306.429s
OK
test_volumes.py
Download a Volume attached to a VM ... === TestName: test_03_download_attached_volume | Status : SUCCESS ===
ok
Delete a Volume attached to a VM ... === TestName: test_04_delete_attached_volume | Status : SUCCESS ===
ok
Detach a Volume attached to a VM ... === TestName: test_05_detach_volume | Status : SUCCESS ===
ok
Delete a Volume unattached to an VM ... === TestName: test_09_delete_detached_volume | Status : SUCCESS ===
ok
----------------------------------------------------------------------
Ran 4 tests in 184.132s
OK
test_network.py
Test for delete account ... === TestName: test_delete_account | Status : SUCCESS ===
ok
Test for Associate/Disassociate public IP address for admin account ... === TestName: test_public_ip_admin_account | Status : SUCCESS ===
ok
Test for Associate/Disassociate public IP address for user account ... === TestName: test_public_ip_user_account | Status : SUCCESS ===
ok
Test for release public IP address ... === TestName: test_releaseIP | Status : SUCCESS ===
ok
----------------------------------------------------------------------
Ran 4 tests in 783.726s
OK
test_routers.py
Test router internal advanced zone ... SKIP: Marvin configuration has no host credentials to check router services
Test restart network ... === TestName: test_03_restart_network_cleanup | Status : SUCCESS ===
ok
Test router basic setup ... === TestName: test_05_router_basic | Status : SUCCESS ===
ok
Test router advanced setup ... === TestName: test_06_router_advanced | Status : SUCCESS ===
ok
Test stop router ... === TestName: test_07_stop_router | Status : SUCCESS ===
ok
Test start router ... === TestName: test_08_start_router | Status : SUCCESS ===
ok
Test reboot router ... === TestName: test_09_reboot_router | Status : SUCCESS ===
ok
----------------------------------------------------------------------
Ran 7 tests in 42.958s
OK (SKIP=1)
test_global_settings.py
test update configuration setting at zone level scope ... === TestName: test_UpdateConfigParamWithScope | Status : SUCCESS ===
ok
----------------------------------------------------------------------
Ran 1 test in 0.127s
OK
test_resource_detail.py
Test volume detail ... === TestName: test_01_updatevolumedetail | Status : SUCCESS ===
ok
----------------------------------------------------------------------
Ran 1 test in 11.492s
OK
* pr/1021:
CLOUDSTACK-8485: listAPIs are taking too long to return results - Removed regex. based search/replace of sensitive data on API response introduced as part of commit b0c6d4734724358df97b6fa4d8c5beb0f447745e - Added new response serializer to skip sensitive data from getting logged based on annotation present in resposne object fields - Added new parameter 'isSensitive' to @Param for marking a field as sensitive in response objects
Signed-off-by: Remi Bergsma <gi...@remi.nl>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/a2a72887
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/a2a72887
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/a2a72887
Branch: refs/heads/master
Commit: a2a72887d205f9ed51c0b31014ab3bd6e571ea4d
Parents: 36d0bfd e13df96
Author: Remi Bergsma <gi...@remi.nl>
Authored: Mon Nov 23 12:56:24 2015 +0100
Committer: Remi Bergsma <gi...@remi.nl>
Committed: Mon Nov 23 12:56:25 2015 +0100
----------------------------------------------------------------------
api/src/com/cloud/serializer/Param.java | 2 +
.../api/response/CreateSSHKeyPairResponse.java | 3 +-
.../api/response/GetVMPasswordResponse.java | 2 +-
.../api/response/LoginCmdResponse.java | 2 +-
.../api/response/RegisterResponse.java | 4 +-
.../api/response/RemoteAccessVpnResponse.java | 2 +-
.../Site2SiteCustomerGatewayResponse.java | 2 +-
.../Site2SiteVpnConnectionResponse.java | 2 +-
.../cloudstack/api/response/UserResponse.java | 4 +-
.../cloudstack/api/response/UserVmResponse.java | 2 +-
.../response/BigSwitchBcfDeviceResponse.java | 2 +-
.../api/response/LDAPConfigResponse.java | 2 +-
.../com/cloud/api/ApiResponseGsonHelper.java | 33 ++++-
server/src/com/cloud/api/ApiServer.java | 15 +-
server/src/com/cloud/api/ApiServlet.java | 5 +-
.../api/response/ApiResponseSerializer.java | 148 +++++++++++++------
16 files changed, 153 insertions(+), 77 deletions(-)
----------------------------------------------------------------------