You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2023/02/27 20:29:03 UTC
[asterixdb] 03/16: [NO ISSUE] Make UDF http client extensible
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ee12055797925c22fbeceb60244e3ad98c5dd570
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Tue Oct 11 14:32:11 2022 -0700
[NO ISSUE] Make UDF http client extensible
(cherry pick of 0ed28248c3471079ae226a66c267976e271a1d4a)
Change-Id: I783ce8c1d888188f8c6d894a5aa435cbe318ec0e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17239
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Wail Alkowaileet <wa...@gmail.com>
Change-Id: Ib924f177f8ee50b536953667bbe1ba1ee4dc3701
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17365
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../asterix/common/library/ILibraryManager.java | 7 +++
.../external/library/ExternalLibraryManager.java | 59 ++++++++++++++--------
2 files changed, 44 insertions(+), 22 deletions(-)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
index 93fe92dcaa..acbce6d4d6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
@@ -24,13 +24,16 @@ import java.io.InputStream;
import java.net.URI;
import java.security.MessageDigest;
import java.util.List;
+import java.util.function.Function;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.ipc.impl.IPCSystem;
public interface ILibraryManager {
@@ -59,9 +62,13 @@ public interface ILibraryManager {
IPCSystem getIPCI();
+ NodeControllerService getNcs();
+
MessageDigest download(FileReference targetFile, String authToken, URI libLocation) throws HyracksException;
void unzip(FileReference sourceFile, FileReference outputDir) throws IOException;
void writeAndForce(FileReference outputFile, InputStream dataStream, byte[] copyBuf) throws IOException;
+
+ void setUploadClient(Function<ILibraryManager, CloseableHttpClient> f);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index f71150a7c6..050d9f0071 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -49,6 +49,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@@ -77,7 +78,6 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
-import org.apache.http.ssl.SSLContexts;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -92,6 +92,7 @@ import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.control.common.work.AbstractWork;
import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.ipc.impl.IPCSystem;
+import org.apache.hyracks.ipc.security.NetworkSecurityManager;
import org.apache.hyracks.ipc.sockets.PlainSocketChannelFactory;
import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
@@ -102,7 +103,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
-public final class ExternalLibraryManager implements ILibraryManager, ILifeCycleComponent {
+public class ExternalLibraryManager implements ILibraryManager, ILifeCycleComponent {
public static final String LIBRARY_MANAGER_BASE_DIR_NAME = "library";
@@ -140,6 +141,7 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle
private final ExternalFunctionResultRouter router;
private final IIOManager ioManager;
private boolean sslEnabled;
+ private Function<ILibraryManager, CloseableHttpClient> uploadClientSupp;
public ExternalLibraryManager(NodeControllerService ncs, IPersistedResourceRegistry reg, FileReference appDir,
IIOManager ioManager) {
@@ -155,6 +157,7 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle
router = new ExternalFunctionResultRouter();
this.sslEnabled = ncs.getConfiguration().isSslEnabled();
this.ioManager = ioManager;
+ uploadClientSupp = ExternalLibraryManager::defaultHttpClient;
}
public void initialize(boolean resetStorageData) throws HyracksDataException {
@@ -486,6 +489,11 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle
return pythonIPC;
}
+ @Override
+ public NodeControllerService getNcs() {
+ return ncs;
+ }
+
private static final class DeleteDirectoryWork extends AbstractWork {
private final Path path;
@@ -630,30 +638,37 @@ public final class ExternalLibraryManager implements ILibraryManager, ILifeCycle
}
}
- //TODO: this should probably be static so it could be reused somewhere else, or made such that the trust store is not
- // reloaded from disk on every client intialization?
- private CloseableHttpClient newClient() {
+ public CloseableHttpClient newClient() {
if (sslEnabled) {
- try {
- final INetworkSecurityManager networkSecurityManager = ncs.getNetworkSecurityManager();
- final INetworkSecurityConfig configuration = networkSecurityManager.getConfiguration();
- KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
- try (FileInputStream trustStoreFile = new FileInputStream(configuration.getTrustStoreFile())) {
- String ksPassword = configuration.getKeyStorePassword();
- trustStore.load(trustStoreFile,
- ksPassword == null || ksPassword.isEmpty() ? null : ksPassword.toCharArray());
- }
- SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(trustStore, null).build();
- SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext,
- new String[] { "TLSv1.2" }, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier());
- return HttpClients.custom().setSSLSocketFactory(sslsf).build();
-
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
+ return uploadClientSupp.apply(this);
} else {
return HttpClients.createDefault();
}
}
+ @Override
+ public void setUploadClient(Function<ILibraryManager, CloseableHttpClient> f) {
+ uploadClientSupp = f;
+ }
+
+ private static CloseableHttpClient defaultHttpClient(ILibraryManager extLib) {
+ try {
+ final INetworkSecurityManager networkSecurityManager = extLib.getNcs().getNetworkSecurityManager();
+ final INetworkSecurityConfig configuration = networkSecurityManager.getConfiguration();
+ KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+ try (FileInputStream trustStoreFile = new FileInputStream(configuration.getTrustStoreFile())) {
+ String ksPassword = configuration.getKeyStorePassword();
+ trustStore.load(trustStoreFile,
+ ksPassword == null || ksPassword.isEmpty() ? null : ksPassword.toCharArray());
+ }
+ SSLContext sslcontext = NetworkSecurityManager.newSSLContext(configuration);
+ SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, new String[] { "TLSv1.2" },
+ null, SSLConnectionSocketFactory.getDefaultHostnameVerifier());
+ return HttpClients.custom().setSSLSocketFactory(sslsf).build();
+
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
}