You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/10/11 06:47:49 UTC
[airavata-sandbox] branch master updated: Jupyter container magic
service
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-sandbox.git
The following commit(s) were added to refs/heads/master by this push:
new 1a8d2e4 Jupyter container magic service
1a8d2e4 is described below
commit 1a8d2e485f5a10c3e68ba4875839390c738d9309
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Mon Oct 11 02:47:06 2021 -0400
Jupyter container magic service
---
jupyter-container-magic/.gitignore | 3 +
.../jupyter/container/magic/api/ApiClient.java | 15 ++++
.../jupyter/container/magic/api/Application.java | 20 +++++
.../jupyter/container/magic/api/Configuration.java | 4 +
.../api/handler/ContainerMagicApiHandler.java | 95 ++++++++++++++++++++++
.../magic/api/handler/FileUploadHandler.java | 32 ++++++++
.../magic/api/service/StorageService.java | 43 ++++++++++
.../src/main/resources/application.properties | 2 +
.../magic-stubs/src/main/proto/api.proto | 48 +++++++++++
9 files changed, 262 insertions(+)
diff --git a/jupyter-container-magic/.gitignore b/jupyter-container-magic/.gitignore
new file mode 100644
index 0000000..89d30e8
--- /dev/null
+++ b/jupyter-container-magic/.gitignore
@@ -0,0 +1,3 @@
+jupyter-container-magic.iml
+.idea
+target/*
\ No newline at end of file
diff --git a/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/ApiClient.java b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/ApiClient.java
new file mode 100644
index 0000000..ad8c343
--- /dev/null
+++ b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/ApiClient.java
@@ -0,0 +1,15 @@
+package org.apache.airavata.jupyter.container.magic.api;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+
+public class ApiClient {
+ public static void main(String args[]) {
+ ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 9999).usePlaintext().build();
+ ContainerMagicApiGrpc.ContainerMagicApiBlockingStub containerMagicApiBlockingStub = ContainerMagicApiGrpc.newBlockingStub(channel);
+ PythonCellExecutionResponse response = containerMagicApiBlockingStub
+ .executePythonCell(PythonCellExecutionRequest.newBuilder().setCellContent("some content").build());
+
+ System.out.println(response);
+ }
+}
diff --git a/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/Application.java b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/Application.java
new file mode 100644
index 0000000..4bd34c4
--- /dev/null
+++ b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/Application.java
@@ -0,0 +1,20 @@
+package org.apache.airavata.jupyter.container.magic.api;
+
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+
+@ComponentScan(basePackages = {"org.apache.airavata.jupyter"})
+@SpringBootApplication()
+public class Application implements CommandLineRunner {
+
+ @Override
+ public void run(String... args) throws Exception {
+
+ }
+
+ public static void main(String[] args) {
+ SpringApplication.run(Application.class, args);
+ }
+}
diff --git a/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/Configuration.java b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/Configuration.java
new file mode 100644
index 0000000..2d048dc
--- /dev/null
+++ b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/Configuration.java
@@ -0,0 +1,4 @@
+package org.apache.airavata.jupyter.container.magic.api;
+
+public class Configuration {
+}
diff --git a/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/handler/ContainerMagicApiHandler.java b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/handler/ContainerMagicApiHandler.java
new file mode 100644
index 0000000..218ea28
--- /dev/null
+++ b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/handler/ContainerMagicApiHandler.java
@@ -0,0 +1,95 @@
+package org.apache.airavata.jupyter.container.magic.api.handler;
+
+import com.google.protobuf.ByteString;
+import io.grpc.stub.StreamObserver;
+import org.apache.airavata.jupyter.container.magic.api.ContainerMagicApiGrpc;
+import org.apache.airavata.jupyter.container.magic.api.PythonCellExecutionRequest;
+import org.apache.airavata.jupyter.container.magic.api.PythonCellExecutionResponse;
+import org.apache.commons.text.StringSubstitutor;
+import org.lognet.springboot.grpc.GRpcService;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+@GRpcService
+public class ContainerMagicApiHandler extends ContainerMagicApiGrpc.ContainerMagicApiImplBase {
+
+ private String tempDir = "C:\\Users\\dimut\\temp_shuttle";
+ private String codeTemplate = "import dill as pickle\n" +
+ "\n" +
+ "with open('${baseDir}${separator}context.pickle', 'rb') as f:\n" +
+ " context = pickle.load(f)\n" +
+ "\n" +
+ "exec(\"${code}\", None, context)" +
+ "\n" +
+ "with open('${baseDir}${separator}final-context.pickle', 'wb') as f:\n" +
+ " pickle.dump(context, f)\n";
+
+ @Override
+ public void executePythonCell(PythonCellExecutionRequest request, StreamObserver<PythonCellExecutionResponse> responseObserver) {
+
+
+ try {
+ Path directory = Files.createDirectory(Path.of(tempDir, UUID.randomUUID().toString()));
+ System.out.println("Directory " + directory.toAbsolutePath().toString());
+
+ Path scriptPath = Path.of(directory.toAbsolutePath().toString(), "script.py");
+
+ Map<String, String> parameters = new HashMap<>();
+ parameters.put("code", request.getCellContent().trim());
+ parameters.put("baseDir", directory.toAbsolutePath().toString().replace("\\", "\\\\"));
+ parameters.put("separator", File.separator.replace("\\", "\\\\"));
+
+
+ StringSubstitutor sub = new StringSubstitutor(parameters);
+ Files.write(scriptPath, sub.replace(codeTemplate).getBytes());
+ Files.write(Path.of(directory.toAbsolutePath().toString(), "context.pickle"), request.getLocalScope().toByteArray());
+
+ Runtime rt = Runtime.getRuntime();
+ String[] commands = {"python3", scriptPath.toAbsolutePath().toString()};
+ Process proc = rt.exec(commands);
+
+ BufferedReader stdInput = new BufferedReader(new
+ InputStreamReader(proc.getInputStream()));
+
+ BufferedReader stdError = new BufferedReader(new
+ InputStreamReader(proc.getErrorStream()));
+
+ System.out.println("Here is the standard output of the command:\n");
+ String s = null;
+ StringBuilder stdOut = new StringBuilder();
+ StringBuilder stdErr = new StringBuilder();
+ while ((s = stdInput.readLine()) != null) {
+ System.out.println(s);
+ stdOut.append(s).append("\n");
+ }
+
+ System.out.println("Here is the standard error of the command (if any):\n");
+ while ((s = stdError.readLine()) != null) {
+ System.out.println(s);
+ stdErr.append(s).append("\n");
+ }
+
+ byte[] finalContext = Files.readAllBytes(Path.of(directory.toAbsolutePath().toString(), "final-context.pickle"));
+ PythonCellExecutionResponse executionResponse = PythonCellExecutionResponse.newBuilder()
+ .setStdOut(stdOut.toString())
+ .setStdErr(stdErr.toString())
+ .setLocalScope(ByteString.copyFrom(finalContext)).build();
+
+ responseObserver.onNext(executionResponse);
+ responseObserver.onCompleted();
+
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ responseObserver.onError(e);
+ }
+
+
+ }
+}
diff --git a/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/handler/FileUploadHandler.java b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/handler/FileUploadHandler.java
new file mode 100644
index 0000000..bdb6a08
--- /dev/null
+++ b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/handler/FileUploadHandler.java
@@ -0,0 +1,32 @@
+package org.apache.airavata.jupyter.container.magic.api.handler;
+
+import org.apache.airavata.jupyter.container.magic.api.service.StorageService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.servlet.mvc.support.RedirectAttributes;
+
+@Controller
+public class FileUploadHandler {
+
+ @Autowired
+ private StorageService storageService;
+
+ @PostMapping("/upload")
+ public ResponseEntity<String> handleFileUpload(@RequestParam("file") MultipartFile file,
+ RedirectAttributes redirectAttributes) {
+
+ try {
+ String uploadPath = storageService.store(file);
+ System.out.println("Uploaded to " + uploadPath);
+ return new ResponseEntity<String>(uploadPath, HttpStatus.OK);
+ } catch (Exception e) {
+ return new ResponseEntity<String>("Failed", HttpStatus.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+}
diff --git a/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/service/StorageService.java b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/service/StorageService.java
new file mode 100644
index 0000000..c039427
--- /dev/null
+++ b/jupyter-container-magic/magic-api/src/main/java/org/apache/airavata/jupyter/container/magic/api/service/StorageService.java
@@ -0,0 +1,43 @@
+package org.apache.airavata.jupyter.container.magic.api.service;
+
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+@Service
+public class StorageService {
+
+ private final Path rootLocation = Files.createTempDirectory("temproot");
+
+ public StorageService() throws IOException {
+ }
+
+ public String store(MultipartFile file) throws Exception {
+ try {
+ if (file.isEmpty()) {
+ throw new Exception("Failed to store empty file.");
+ }
+ Path destinationFile = this.rootLocation.resolve(
+ Paths.get(file.getOriginalFilename()))
+ .normalize().toAbsolutePath();
+ if (!destinationFile.getParent().equals(this.rootLocation.toAbsolutePath())) {
+ // This is a security check
+ throw new Exception(
+ "Cannot store file outside current directory.");
+ }
+ try (InputStream inputStream = file.getInputStream()) {
+ Files.copy(inputStream, destinationFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ return destinationFile.toAbsolutePath().toString();
+ }
+ catch (IOException e) {
+ throw new Exception("Failed to store file.", e);
+ }
+ }
+}
diff --git a/jupyter-container-magic/magic-api/src/main/resources/application.properties b/jupyter-container-magic/magic-api/src/main/resources/application.properties
new file mode 100644
index 0000000..89ce7ff
--- /dev/null
+++ b/jupyter-container-magic/magic-api/src/main/resources/application.properties
@@ -0,0 +1,2 @@
+grpc.port=9999
+server.address=0.0.0.0
\ No newline at end of file
diff --git a/jupyter-container-magic/magic-stubs/src/main/proto/api.proto b/jupyter-container-magic/magic-stubs/src/main/proto/api.proto
new file mode 100644
index 0000000..91835ea
--- /dev/null
+++ b/jupyter-container-magic/magic-stubs/src/main/proto/api.proto
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+package org.apache.airavata.jupyter.container.magic.api;
+
+import "google/api/annotations.proto";
+import "google/protobuf/empty.proto";
+
+message DataUploadEntry {
+ string filePath = 1;
+ string uploadedPath = 2;
+}
+
+message PythonCellExecutionRequest {
+ string cellContent = 1;
+ bytes localScope = 2;
+ repeated DataUploadEntry uploadEntries = 3;
+}
+
+message PythonCellExecutionResponse {
+ string stdOut = 1;
+ string stdErr = 2;
+ bytes localScope = 3;
+}
+
+service ContainerMagicApi {
+ rpc executePythonCell (PythonCellExecutionRequest) returns (PythonCellExecutionResponse) {
+
+ }
+}
+