You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/27 00:34:52 UTC
[28/50] [abbrv] kylin git commit: KYLIN-2043 rollback httpclient to
4.2.5 to align with Hadoop 2.6/2.7
KYLIN-2043 rollback httpclient to 4.2.5 to align with Hadoop 2.6/2.7
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6cdc0d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6cdc0d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6cdc0d1
Branch: refs/heads/1.5.x-CDH5.7
Commit: b6cdc0d1b20683feb7255a71dfa8d5b98c4e636e
Parents: 65afee7
Author: Yang Li <li...@apache.org>
Authored: Sat Sep 24 01:31:22 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Sep 24 01:31:22 2016 +0800
----------------------------------------------------------------------
.../kylin/common/restclient/RestClient.java | 35 +++++++--------
.../engine/mr/common/HadoopStatusGetter.java | 27 ++++++------
.../java/org/apache/kylin/jdbc/KylinClient.java | 46 ++++++++++----------
pom.xml | 2 +-
4 files changed, 52 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 46a9e9b..363cc54 100644
--- a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -23,16 +23,15 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.util.EntityUtils;
import org.apache.kylin.common.util.JsonUtil;
/**
@@ -45,7 +44,7 @@ public class RestClient {
protected String baseUrl;
protected String userName;
protected String password;
- protected CloseableHttpClient client;
+ protected DefaultHttpClient client;
protected static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
@@ -79,13 +78,13 @@ public class RestClient {
this.password = password;
this.baseUrl = "http://" + host + ":" + port + "/kylin/api";
- client = HttpClients.createDefault();
+ client = new DefaultHttpClient();
if (userName != null && password != null) {
CredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userName, password);
- provider.setCredentials(AuthScope.ANY, credentials);
- client = HttpClients.custom().setDefaultCredentialsProvider(provider).build();
+ provider.setCredentials(AuthScope.ANY, credentials);
+ client.setCredentialsProvider(provider);
}
}
@@ -94,12 +93,11 @@ public class RestClient {
HttpPut request = new HttpPut(url);
try {
- CloseableHttpResponse response = client.execute(request);
+ HttpResponse response = client.execute(request);
String msg = EntityUtils.toString(response.getEntity());
if (response.getStatusLine().getStatusCode() != 200)
- throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg);
- response.close();
+ throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg);
} catch (Exception ex) {
throw new IOException(ex);
} finally {
@@ -111,14 +109,13 @@ public class RestClient {
String url = baseUrl + "/admin/config";
HttpGet request = new HttpGet(url);
try {
- CloseableHttpResponse response = client.execute(request);
+ HttpResponse response = client.execute(request);
String msg = EntityUtils.toString(response.getEntity());
Map<String, String> map = JsonUtil.readValueAsMap(msg);
msg = map.get("config");
if (response.getStatusLine().getStatusCode() != 200)
throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg);
- response.close();
return msg;
} finally {
request.releaseConnection();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 14f7235..619de90 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -34,19 +34,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
-import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthSchemeRegistry;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
-import org.apache.http.client.config.AuthSchemes;
-import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.config.Lookup;
-import org.apache.http.config.RegistryBuilder;
+import org.apache.http.client.params.AuthPolicy;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.DefaultHttpClient;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
@@ -88,9 +84,12 @@ public class HadoopStatusGetter {
System.setProperty("java.security.krb5.conf", krb5ConfigPath);
System.setProperty("sun.security.krb5.debug", "true");
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
- Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider> create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(skipPortAtKerberosDatabaseLookup)).build();
- CloseableHttpClient client = HttpClients.custom().setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
- HttpClientContext context = HttpClientContext.create();
+
+ DefaultHttpClient client = new DefaultHttpClient();
+ AuthSchemeRegistry authSchemeRegistry = new AuthSchemeRegistry();
+ authSchemeRegistry.register(AuthPolicy.SPNEGO, new SPNegoSchemeFactory(skipPortAtKerberosDatabaseLookup));
+ client.setAuthSchemes(authSchemeRegistry);
+
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
Credentials useJaasCreds = new Credentials() {
public String getPassword() {
@@ -101,9 +100,9 @@ public class HadoopStatusGetter {
return null;
}
};
-
credentialsProvider.setCredentials(new AuthScope(null, -1, null), useJaasCreds);
- context.setCredentialsProvider(credentialsProvider);
+ client.setCredentialsProvider(credentialsProvider);
+
String response = null;
while (response == null) {
if (url.startsWith("https://")) {
@@ -116,7 +115,7 @@ public class HadoopStatusGetter {
HttpGet httpget = new HttpGet(url);
httpget.addHeader("accept", "application/json");
try {
- CloseableHttpResponse httpResponse = client.execute(httpget, context);
+ HttpResponse httpResponse = client.execute(httpget);
String redirect = null;
org.apache.http.Header h = httpResponse.getFirstHeader("Location");
if (h != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 8890575..2d06a92 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -20,6 +20,7 @@ package org.apache.kylin.jdbc;
import java.io.IOException;
import java.math.BigDecimal;
+import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.sql.Date;
import java.sql.Time;
@@ -31,7 +32,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import javax.net.ssl.SSLContext;
import javax.xml.bind.DatatypeConverter;
import org.apache.calcite.avatica.AvaticaParameter;
@@ -39,17 +39,15 @@ import org.apache.calcite.avatica.ColumnMetaData;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.calcite.avatica.ColumnMetaData.ScalarType;
import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.ssl.SSLContexts;
+import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.kylin.jdbc.KylinMeta.KMetaCatalog;
import org.apache.kylin.jdbc.KylinMeta.KMetaColumn;
@@ -74,25 +72,25 @@ public class KylinClient implements IRemoteClient {
private final KylinConnection conn;
private final Properties connProps;
- private CloseableHttpClient httpClient;
+ private DefaultHttpClient httpClient;
private final ObjectMapper jsonMapper;
public KylinClient(KylinConnection conn) {
this.conn = conn;
this.connProps = conn.getConnectionProperties();
- this.httpClient = HttpClients.createDefault();
+ this.httpClient = new DefaultHttpClient();
this.jsonMapper = new ObjectMapper();
// trust all certificates
if (isSSL()) {
try {
- TrustStrategy acceptingTrustStrategy = new TrustStrategy() {
- public boolean isTrusted(X509Certificate[] certificate, String type) {
+ SSLSocketFactory sslsf = new SSLSocketFactory(new TrustStrategy() {
+ public boolean isTrusted(final X509Certificate[] chain, String authType) throws CertificateException {
+ // Oh, I am easy...
return true;
}
- };
- SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build();
- httpClient = HttpClients.custom().setSSLHostnameVerifier(new NoopHostnameVerifier()).setSSLContext(sslContext).build();
+ });
+ httpClient.getConnectionManager().getSchemeRegistry().register(new Scheme("https", 443, sslsf));
} catch (Exception e) {
throw new RuntimeException("Initialize HTTPS client failed", e);
}
@@ -227,12 +225,15 @@ public class KylinClient implements IRemoteClient {
StringEntity requestEntity = new StringEntity("{}", ContentType.create("application/json", "UTF-8"));
post.setEntity(requestEntity);
- CloseableHttpResponse response = httpClient.execute(post);
+ try {
+ HttpResponse response = httpClient.execute(post);
- if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) {
- throw asIOException(post, response);
+ if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) {
+ throw asIOException(post, response);
+ }
+ } finally {
+ post.releaseConnection();
}
- response.close();
}
@Override
@@ -243,7 +244,7 @@ public class KylinClient implements IRemoteClient {
HttpGet get = new HttpGet(url);
addHttpHeaders(get);
- CloseableHttpResponse response = httpClient.execute(get);
+ HttpResponse response = httpClient.execute(get);
if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) {
throw asIOException(get, response);
@@ -255,7 +256,7 @@ public class KylinClient implements IRemoteClient {
List<KMetaTable> tables = convertMetaTables(tableMetaStubs);
List<KMetaSchema> schemas = convertMetaSchemas(tables);
List<KMetaCatalog> catalogs = convertMetaCatalogs(schemas);
- response.close();
+ get.releaseConnection();
return new KMetaProject(project, catalogs);
}
@@ -368,14 +369,14 @@ public class KylinClient implements IRemoteClient {
StringEntity requestEntity = new StringEntity(postBody, ContentType.create("application/json", "UTF-8"));
post.setEntity(requestEntity);
- CloseableHttpResponse response = httpClient.execute(post);
+ HttpResponse response = httpClient.execute(post);
if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 201) {
throw asIOException(post, response);
}
SQLResponseStub stub = jsonMapper.readValue(response.getEntity().getContent(), SQLResponseStub.class);
- response.close();
+ post.releaseConnection();
return stub;
}
@@ -416,8 +417,5 @@ public class KylinClient implements IRemoteClient {
@Override
public void close() throws IOException {
- if (httpClient != null) {
- httpClient.close();
- }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b6cdc0d1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1abc4eb..ecb2724 100644
--- a/pom.xml
+++ b/pom.xml
@@ -96,7 +96,7 @@
<extendedset.version>1.3.4</extendedset.version>
<kryo.version>4.0.0</kryo.version>
<ehcache.version>2.10.2.2.21</ehcache.version>
- <apache-httpclient.version>4.5.2</apache-httpclient.version>
+ <apache-httpclient.version>4.2.5</apache-httpclient.version>
<roaring.version>0.6.18</roaring.version>
<cglib.version>3.2.4</cglib.version>
<supercsv.version>2.4.0</supercsv.version>