You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pi...@apache.org on 2015/08/11 12:13:51 UTC
falcon git commit: FALCON-1384 AbstractEntityHelper.java cleanup.
Contributed by Paul Isaychuk
Repository: falcon
Updated Branches:
refs/heads/master cbb38ce18 -> 815e47695
FALCON-1384 AbstractEntityHelper.java cleanup. Contributed by Paul Isaychuk
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/815e4769
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/815e4769
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/815e4769
Branch: refs/heads/master
Commit: 815e476951f3f1e62744c4ae9449ed8af513f11e
Parents: cbb38ce
Author: Paul Isaychuk <pi...@apache.org>
Authored: Tue Aug 11 13:07:00 2015 +0300
Committer: Paul Isaychuk <pi...@apache.org>
Committed: Tue Aug 11 13:07:51 2015 +0300
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 2 +
.../helpers/entity/AbstractEntityHelper.java | 250 +++++++++----------
.../org/apache/falcon/request/BaseRequest.java | 13 +-
3 files changed, 129 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/815e4769/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index d6fa935..f382066 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -98,6 +98,8 @@ Trunk (Unreleased)
via Samarth Gupta)
IMPROVEMENTS
+ FALCON-1384 AbstractEntityHelper.java cleanup (Paul Isaychuk)
+
FALCON-1306 Custom window dimensions for UI tests (Ruslan Ostafiychuk)
FALCON-1284 Fix entity cleanup when is_depracate=true (Ruslan Ostafiychuk via Paul Isaychuk)
http://git-wip-us.apache.org/repos/asf/falcon/blob/815e4769/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
index 2baa35f..0bd8a32 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java
@@ -55,14 +55,85 @@ import java.util.List;
/** Abstract class for helper classes. */
public abstract class AbstractEntityHelper {
- public static final boolean AUTHENTICATE = setAuthenticate();
-
private static final Logger LOGGER = Logger.getLogger(AbstractEntityHelper.class);
- private static boolean setAuthenticate() {
- String value = Config.getProperty("isAuthenticationSet");
- value = (null == value) ? "true" : value;
- return !value.equalsIgnoreCase("false");
+ //basic properties
+ private String qaHost;
+ private String hostname = "";
+ private String username = "";
+ private String password = "";
+ private String hadoopLocation = "";
+ private String hadoopURL = "";
+ private String clusterReadonly = "";
+ private String clusterWrite = "";
+ private String oozieURL = "";
+ private String activeMQ = "";
+ private String storeLocation = "";
+ private String colo;
+ private String allColo;
+ private String coloName;
+ //hive jdbc
+ private String hiveJdbcUrl = "";
+ private String hiveJdbcUser = "";
+ private String hiveJdbcPassword = "";
+ private Connection hiveJdbcConnection;
+ //clients
+ private OozieClient oozieClient;
+ private String hcatEndpoint = "";
+ private HCatClient hCatClient;
+ private FileSystem hadoopFS;
+ //other properties
+ private String namenodePrincipal;
+ private String hiveMetaStorePrincipal;
+ private String identityFile;
+ private String serviceUser;
+ private String serviceStartCmd;
+ private String serviceStopCmd;
+ private String serviceStatusMsg;
+ private String serviceStatusCmd;
+
+ public AbstractEntityHelper(String prefix) {
+ if ((null == prefix) || prefix.isEmpty()) {
+ prefix = "";
+ } else {
+ prefix += ".";
+ }
+ this.qaHost = Config.getProperty(prefix + "qa_host");
+ this.hostname = Config.getProperty(prefix + "hostname");
+ this.username = Config.getProperty(prefix + "username", System.getProperty("user.name"));
+ this.password = Config.getProperty(prefix + "password", "");
+ this.hadoopLocation = Config.getProperty(prefix + "hadoop_location");
+ this.hadoopURL = Config.getProperty(prefix + "hadoop_url");
+ this.hcatEndpoint = Config.getProperty(prefix + "hcat_endpoint");
+ this.clusterReadonly = Config.getProperty(prefix + "cluster_readonly");
+ this.clusterWrite = Config.getProperty(prefix + "cluster_write");
+ this.oozieURL = Config.getProperty(prefix + "oozie_url");
+ this.activeMQ = Config.getProperty(prefix + "activemq_url");
+ this.storeLocation = Config.getProperty(prefix + "storeLocation");
+ this.allColo = "?colo=" + Config.getProperty(prefix + "colo", "*");
+ this.colo = (!Config.getProperty(prefix + "colo", "").isEmpty()) ? "?colo=" + Config
+ .getProperty(prefix + "colo") : "";
+ this.coloName = this.colo.contains("=") ? this.colo.split("=")[1] : "";
+ this.serviceStartCmd =
+ Config.getProperty(prefix + "service_start_cmd", "/etc/init.d/tomcat6 start");
+ this.serviceStopCmd = Config.getProperty(prefix + "service_stop_cmd",
+ "/etc/init.d/tomcat6 stop");
+ this.serviceUser = Config.getProperty(prefix + "service_user", null);
+ this.serviceStatusMsg = Config.getProperty(prefix + "service_status_msg",
+ "Tomcat servlet engine is running with pid");
+ this.serviceStatusCmd =
+ Config.getProperty(prefix + "service_status_cmd", "/etc/init.d/tomcat6 status");
+ this.identityFile = Config.getProperty(prefix + "identityFile",
+ System.getProperty("user.home") + "/.ssh/id_rsa");
+ this.hadoopFS = null;
+ this.oozieClient = null;
+ this.namenodePrincipal = Config.getProperty(prefix + "namenode.kerberos.principal", "none");
+ this.hiveMetaStorePrincipal = Config.getProperty(
+ prefix + "hive.metastore.kerberos.principal", "none");
+ this.hiveJdbcUrl = Config.getProperty(prefix + "hive.jdbc.url", "none");
+ this.hiveJdbcUser =
+ Config.getProperty(prefix + "hive.jdbc.user", System.getProperty("user.name"));
+ this.hiveJdbcPassword = Config.getProperty(prefix + "hive.jdbc.password", "");
}
public String getActiveMQ() {
@@ -105,7 +176,45 @@ public abstract class AbstractEntityHelper {
return hcatEndpoint;
}
- protected HCatClient hCatClient;
+ public String getQaHost() {
+ return qaHost;
+ }
+
+ public String getIdentityFile() {
+ return identityFile;
+ }
+
+ public String getServiceUser() {
+ return serviceUser;
+ }
+
+ public String getServiceStopCmd() {
+ return serviceStopCmd;
+ }
+
+ public String getServiceStartCmd() {
+ return serviceStartCmd;
+ }
+
+ public String getColo() {
+ return colo;
+ }
+
+ public String getColoName() {
+ return coloName;
+ }
+
+ public abstract String getEntityType();
+
+ public abstract String getEntityName(String entity);
+
+ public String getNamenodePrincipal() {
+ return namenodePrincipal;
+ }
+
+ public String getHiveMetaStorePrincipal() {
+ return hiveMetaStorePrincipal;
+ }
public HCatClient getHCatClient() {
if (null == this.hCatClient) {
@@ -119,8 +228,6 @@ public abstract class AbstractEntityHelper {
return this.hCatClient;
}
- protected Connection hiveJdbcConnection;
-
public Connection getHiveJdbcConnection() {
if (null == hiveJdbcConnection) {
try {
@@ -134,45 +241,6 @@ public abstract class AbstractEntityHelper {
return hiveJdbcConnection;
}
- //basic properties
- protected String qaHost;
-
- public String getQaHost() {
- return qaHost;
- }
-
- protected String hostname = "";
- protected String username = "";
- protected String password = "";
- protected String hadoopLocation = "";
- protected String hadoopURL = "";
- protected String clusterReadonly = "";
- protected String clusterWrite = "";
- private String oozieURL = "";
- protected String activeMQ = "";
- protected String storeLocation = "";
- protected String colo;
- protected String allColo;
- protected String coloName;
- protected String serviceStartCmd;
- protected String serviceStopCmd;
- protected String serviceStatusCmd;
- protected String hcatEndpoint = "";
- protected String hiveJdbcUrl = "";
- protected String hiveJdbcUser = "";
- protected String hiveJdbcPassword = "";
-
- public String getNamenodePrincipal() {
- return namenodePrincipal;
- }
-
- public String getHiveMetaStorePrincipal() {
- return hiveMetaStorePrincipal;
- }
-
- protected String namenodePrincipal;
- protected String hiveMetaStorePrincipal;
-
public OozieClient getOozieClient() {
if (null == this.oozieClient) {
this.oozieClient = OozieUtil.getClient(this.oozieURL);
@@ -180,8 +248,6 @@ public abstract class AbstractEntityHelper {
return this.oozieClient;
}
- protected OozieClient oozieClient;
-
public FileSystem getHadoopFS() throws IOException {
if (null == this.hadoopFS) {
Configuration conf = new Configuration();
@@ -192,87 +258,7 @@ public abstract class AbstractEntityHelper {
return this.hadoopFS;
}
- protected FileSystem hadoopFS;
-
- public String getIdentityFile() {
- return identityFile;
- }
-
- protected String identityFile;
-
- protected String serviceStatusMsg;
-
- public String getServiceUser() {
- return serviceUser;
- }
-
- public String getServiceStopCmd() {
- return serviceStopCmd;
- }
-
- public String getServiceStartCmd() {
- return serviceStartCmd;
- }
-
- protected String serviceUser;
-
- public String getColo() {
- return colo;
- }
-
- public String getColoName() {
- return coloName;
- }
-
- public AbstractEntityHelper(String prefix) {
- if ((null == prefix) || prefix.isEmpty()) {
- prefix = "";
- } else {
- prefix += ".";
- }
- this.qaHost = Config.getProperty(prefix + "qa_host");
- this.hostname = Config.getProperty(prefix + "hostname");
- this.username = Config.getProperty(prefix + "username", System.getProperty("user.name"));
- this.password = Config.getProperty(prefix + "password", "");
- this.hadoopLocation = Config.getProperty(prefix + "hadoop_location");
- this.hadoopURL = Config.getProperty(prefix + "hadoop_url");
- this.hcatEndpoint = Config.getProperty(prefix + "hcat_endpoint");
- this.clusterReadonly = Config.getProperty(prefix + "cluster_readonly");
- this.clusterWrite = Config.getProperty(prefix + "cluster_write");
- this.oozieURL = Config.getProperty(prefix + "oozie_url");
- this.activeMQ = Config.getProperty(prefix + "activemq_url");
- this.storeLocation = Config.getProperty(prefix + "storeLocation");
- this.allColo = "?colo=" + Config.getProperty(prefix + "colo", "*");
- this.colo = (!Config.getProperty(prefix + "colo", "").isEmpty()) ? "?colo=" + Config
- .getProperty(prefix + "colo") : "";
- this.coloName = this.colo.contains("=") ? this.colo.split("=")[1] : "";
- this.serviceStartCmd =
- Config.getProperty(prefix + "service_start_cmd", "/etc/init.d/tomcat6 start");
- this.serviceStopCmd = Config.getProperty(prefix + "service_stop_cmd",
- "/etc/init.d/tomcat6 stop");
- this.serviceUser = Config.getProperty(prefix + "service_user", null);
- this.serviceStatusMsg = Config.getProperty(prefix + "service_status_msg",
- "Tomcat servlet engine is running with pid");
- this.serviceStatusCmd =
- Config.getProperty(prefix + "service_status_cmd", "/etc/init.d/tomcat6 status");
- this.identityFile = Config.getProperty(prefix + "identityFile",
- System.getProperty("user.home") + "/.ssh/id_rsa");
- this.hadoopFS = null;
- this.oozieClient = null;
- this.namenodePrincipal = Config.getProperty(prefix + "namenode.kerberos.principal", "none");
- this.hiveMetaStorePrincipal = Config.getProperty(
- prefix + "hive.metastore.kerberos.principal", "none");
- this.hiveJdbcUrl = Config.getProperty(prefix + "hive.jdbc.url", "none");
- this.hiveJdbcUser =
- Config.getProperty(prefix + "hive.jdbc.user", System.getProperty("user.name"));
- this.hiveJdbcPassword = Config.getProperty(prefix + "hive.jdbc.password", "");
- }
-
- public abstract String getEntityType();
-
- public abstract String getEntityName(String entity);
-
- protected String createUrl(String... parts) {
+ private String createUrl(String... parts) {
return StringUtils.join(parts, "/");
}
@@ -608,7 +594,6 @@ public abstract class AbstractEntityHelper {
return ExecUtil.executeCommand(commandLine);
}
-
/**
* Retrieves entities summary.
* @param clusterName compulsory parameter for request
@@ -670,7 +655,6 @@ public abstract class AbstractEntityHelper {
return Util.sendRequest(url, "post", data, user);
}
-
/**
* Retrieves entities lineage.
* @param params list of optional parameters
http://git-wip-us.apache.org/repos/asf/falcon/blob/815e4769/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
index 7157111..e5430eb 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java
@@ -20,7 +20,7 @@ package org.apache.falcon.request;
import org.apache.commons.net.util.TrustManagerUtils;
import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants;
-import org.apache.falcon.regression.core.helpers.entity.AbstractEntityHelper;
+import org.apache.falcon.regression.core.util.Config;
import org.apache.falcon.security.FalconAuthorizationToken;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -59,6 +59,7 @@ import java.util.List;
/** Class for making rest requests. */
public class BaseRequest {
+ private static final boolean AUTHENTICATE = setAuthenticate();
private static final Logger LOGGER = Logger.getLogger(BaseRequest.class);
private String method;
@@ -104,6 +105,12 @@ public class BaseRequest {
this.requestData = data;
}
+ private static boolean setAuthenticate() {
+ String value = Config.getProperty("isAuthenticationSet");
+ value = (null == value) ? "true" : value;
+ return !value.equalsIgnoreCase("false");
+ }
+
public void addHeader(String name, String value) {
headers.add(new BasicHeader(name, value));
}
@@ -149,7 +156,7 @@ public class BaseRequest {
/*get the token and add it to the header.
works in secure and un secure mode.*/
AuthenticatedURL.Token token;
- if (AbstractEntityHelper.AUTHENTICATE) {
+ if (AUTHENTICATE) {
token = FalconAuthorizationToken.getToken(user, uri.getScheme(),
uri.getHost(), uri.getPort());
request.addHeader(RequestKeys.COOKIE, RequestKeys.AUTH_COOKIE_EQ + token);
@@ -179,7 +186,7 @@ public class BaseRequest {
Header[] wwwAuthHeaders = response.getHeaders(RequestKeys.WWW_AUTHENTICATE);
if (wwwAuthHeaders != null && wwwAuthHeaders.length != 0
&& wwwAuthHeaders[0].getValue().trim().startsWith(RequestKeys.NEGOTIATE)) {
- if (AbstractEntityHelper.AUTHENTICATE) {
+ if (AUTHENTICATE) {
token = FalconAuthorizationToken.getToken(user, uri.getScheme(),
uri.getHost(), uri.getPort(), true);
request.removeHeaders(RequestKeys.COOKIE);