You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@teaclave.apache.org by ms...@apache.org on 2020/03/25 02:52:16 UTC
[incubator-teaclave] branch develop updated: [types] Simplify
StagedFiles structures (#247)
This is an automated email from the ASF dual-hosted git repository.
mssun pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-teaclave.git
The following commit(s) were added to refs/heads/develop by this push:
new 9838d5b [types] Simplify StagedFiles structures (#247)
9838d5b is described below
commit 9838d5be05ae61dc888777dced5bb9d43d24456a
Author: Zhaofeng Chen <zf...@apache.org>
AuthorDate: Tue Mar 24 19:52:11 2020 -0700
[types] Simplify StagedFiles structures (#247)
---
services/execution/enclave/src/service.rs | 6 +-
tests/integration/enclave/src/teaclave_worker.rs | 8 +-
types/src/staged_file.rs | 114 +++++------------------
types/src/staged_function.rs | 10 +-
worker/src/function/context.rs | 7 +-
worker/src/function/gbdt_prediction.rs | 9 +-
worker/src/function/gbdt_training.rs | 7 +-
worker/src/function/mesapy.rs | 7 +-
worker/src/runtime/default.rs | 11 +--
worker/src/runtime/raw_io.rs | 11 +--
worker/src/worker.rs | 17 +---
11 files changed, 61 insertions(+), 146 deletions(-)
diff --git a/services/execution/enclave/src/service.rs b/services/execution/enclave/src/service.rs
index 1a60a34..86fa19b 100644
--- a/services/execution/enclave/src/service.rs
+++ b/services/execution/enclave/src/service.rs
@@ -197,7 +197,7 @@ fn prepare_task(task: &StagedTask) -> StagedFunction {
let request = FileAgentRequest::new(HandleFileCommand::Download, file_request_info);
handle_file_request(request).unwrap();
- let mut converted_input_file_map: HashMap<String, StagedInputFile> = HashMap::new();
+ let mut converted_input_file_map: HashMap<String, StagedFileInfo> = HashMap::new();
for (key, value) in input_file_map.iter() {
let (from, crypto_info) = value;
let mut dest = from.clone();
@@ -209,7 +209,7 @@ fn prepare_task(task: &StagedTask) -> StagedFunction {
}
let input_files = StagedFiles::new(converted_input_file_map);
- let mut output_file_map: HashMap<String, StagedOutputFile> = HashMap::new();
+ let mut output_file_map: HashMap<String, StagedFileInfo> = HashMap::new();
for (key, value) in task.output_data.iter() {
let mut dest = agent_dir_path.to_path_buf();
dest.push(&format!("{}.out", key));
@@ -217,7 +217,7 @@ fn prepare_task(task: &StagedTask) -> StagedFunction {
FileCrypto::TeaclaveFile128(crypto) => crypto,
_ => unimplemented!(),
};
- let output_info = StagedOutputFile::new(dest.to_string_lossy().to_string(), crypto);
+ let output_info = StagedFileInfo::new(&dest, crypto);
output_file_map.insert(key.to_string(), output_info);
}
let output_files = StagedFiles::new(output_file_map);
diff --git a/tests/integration/enclave/src/teaclave_worker.rs b/tests/integration/enclave/src/teaclave_worker.rs
index 3ed204d..6d763bc 100644
--- a/tests/integration/enclave/src/teaclave_worker.rs
+++ b/tests/integration/enclave/src/teaclave_worker.rs
@@ -18,8 +18,8 @@
use std::prelude::v1::*;
use teaclave_types::{
- hashmap, read_all_bytes, ExecutorType, FunctionArguments, StagedFiles, StagedFunction,
- StagedInputFile, StagedOutputFile, TeaclaveFile128Key,
+ hashmap, read_all_bytes, ExecutorType, FunctionArguments, StagedFileInfo, StagedFiles,
+ StagedFunction, TeaclaveFile128Key,
};
use teaclave_worker::Worker;
@@ -40,12 +40,12 @@ fn test_start_worker() {
let enc_output = "fixtures/functions/gbdt_training/model.enc.out";
let expected_output = "fixtures/functions/gbdt_training/expected_model.txt";
- let input_info = StagedInputFile::create_with_plaintext_file(plain_input).unwrap();
+ let input_info = StagedFileInfo::create_with_plaintext_file(plain_input).unwrap();
let input_files = StagedFiles::new(hashmap!(
"training_data" => input_info));
- let output_info = StagedOutputFile::new(enc_output, TeaclaveFile128Key::random());
+ let output_info = StagedFileInfo::new(enc_output, TeaclaveFile128Key::random());
let output_files = StagedFiles::new(hashmap!(
"trained_model" => output_info.clone()));
diff --git a/types/src/staged_file.rs b/types/src/staged_file.rs
index 001fdc6..b98c3c3 100644
--- a/types/src/staged_file.rs
+++ b/types/src/staged_file.rs
@@ -28,57 +28,53 @@ use std::untrusted::fs::File;
use protected_fs::ProtectedFile;
#[derive(Clone, Debug, Default)]
-pub struct StagedInputFile {
+pub struct StagedFileInfo {
pub path: std::path::PathBuf,
pub crypto_info: TeaclaveFile128Key,
}
-#[derive(Clone, Debug, Default)]
-pub struct StagedOutputFile {
- pub path: std::path::PathBuf,
- pub crypto_info: TeaclaveFile128Key,
-}
-
-impl std::convert::From<StagedOutputFile> for StagedInputFile {
- fn from(info: StagedOutputFile) -> Self {
- StagedInputFile {
- path: info.path,
- crypto_info: info.crypto_info,
- }
- }
-}
-
-impl StagedInputFile {
+impl StagedFileInfo {
pub fn new(
path: impl std::convert::Into<std::path::PathBuf>,
crypto_info: TeaclaveFile128Key,
) -> Self {
- StagedInputFile {
+ StagedFileInfo {
path: path.into(),
crypto_info,
}
}
pub fn get_readable_io(&self) -> anyhow::Result<Box<dyn io::Read>> {
- log::debug!("path: {:?}", self.path);
- log::debug!("key: {:?}", self.crypto_info.key);
let f = ProtectedFile::open_ex(&self.path, &self.crypto_info.key)?;
Ok(Box::new(f))
}
+ pub fn get_writable_io(&self) -> anyhow::Result<Box<dyn io::Write>> {
+ let f = ProtectedFile::create_ex(&self.path, &self.crypto_info.key)?;
+ Ok(Box::new(f))
+ }
+
#[cfg(test_mode)]
pub fn create_with_plaintext_file(
path: impl AsRef<std::path::Path>,
- ) -> anyhow::Result<StagedInputFile> {
+ ) -> anyhow::Result<StagedFileInfo> {
let bytes = read_all_bytes(path.as_ref())?;
let dst = path.as_ref().with_extension("enc");
Self::create_with_bytes(dst, &bytes)
}
+ #[cfg(test_mode)]
+ pub fn get_plaintext(&self) -> anyhow::Result<Vec<u8>> {
+ let mut content = Vec::new();
+ let mut f = ProtectedFile::open_ex(&self.path, &self.crypto_info.key)?;
+ f.read_to_end(&mut content)?;
+ Ok(content)
+ }
+
pub fn create_with_bytes(
path: impl AsRef<std::path::Path>,
bytes: &[u8],
- ) -> anyhow::Result<StagedInputFile> {
+ ) -> anyhow::Result<StagedFileInfo> {
let crypto = TeaclaveFile128Key::random();
let mut f = ProtectedFile::create_ex(&path, &crypto.key)?;
f.write_all(bytes)?;
@@ -86,33 +82,6 @@ impl StagedInputFile {
}
}
-impl StagedOutputFile {
- pub fn new(
- path: impl std::convert::Into<std::path::PathBuf>,
- crypto_info: TeaclaveFile128Key,
- ) -> Self {
- StagedOutputFile {
- path: path.into(),
- crypto_info,
- }
- }
-
- pub fn get_writable_io(&self) -> anyhow::Result<Box<dyn io::Write>> {
- log::debug!("path: {:?}", self.path);
- log::debug!("key: {:?}", self.crypto_info.key);
- let f = ProtectedFile::create_ex(&self.path, &self.crypto_info.key)?;
- Ok(Box::new(f))
- }
-
- #[cfg(test_mode)]
- pub fn get_plaintext(&self) -> anyhow::Result<Vec<u8>> {
- let mut content = Vec::new();
- let mut f = ProtectedFile::open_ex(&self.path, &self.crypto_info.key)?;
- f.read_to_end(&mut content)?;
- Ok(content)
- }
-}
-
pub fn read_all_bytes(path: impl AsRef<std::path::Path>) -> anyhow::Result<Vec<u8>> {
let mut content = Vec::new();
let mut file = File::open(path)?;
@@ -124,7 +93,7 @@ pub fn convert_encrypted_input_file(
path: impl AsRef<std::path::Path>,
crypto_info: FileCrypto,
dst: impl AsRef<std::path::Path>,
-) -> anyhow::Result<StagedInputFile> {
+) -> anyhow::Result<StagedFileInfo> {
log::debug!("from: {:?}, to: {:?}", path.as_ref(), dst.as_ref());
#[cfg(not(feature = "mesalock_sgx"))]
use std::fs;
@@ -144,53 +113,20 @@ pub fn convert_encrypted_input_file(
FileCrypto::TeaclaveFile128(crypto) => {
fs::copy(path, dst.as_ref())?;
let dst = dst.as_ref().to_owned();
- return Ok(StagedInputFile::new(dst, crypto));
+ return Ok(StagedFileInfo::new(dst, crypto));
}
FileCrypto::Raw => read_all_bytes(path)?,
};
- StagedInputFile::create_with_bytes(dst.as_ref(), &plain_text)
+ StagedFileInfo::create_with_bytes(dst.as_ref(), &plain_text)
}
#[derive(Debug, Default)]
-pub struct StagedFiles<T> {
- pub entries: HashMap<String, T>,
+pub struct StagedFiles {
+ pub entries: HashMap<String, StagedFileInfo>,
}
-impl<T> StagedFiles<T> {
- pub fn new(entries: HashMap<String, T>) -> Self {
+impl StagedFiles {
+ pub fn new(entries: HashMap<String, StagedFileInfo>) -> Self {
StagedFiles { entries }
}
}
-
-impl<U, V> std::convert::TryFrom<HashMap<String, U>> for StagedFiles<V>
-where
- U: std::convert::TryInto<V, Error = anyhow::Error>,
-{
- type Error = anyhow::Error;
- fn try_from(entries: HashMap<String, U>) -> anyhow::Result<Self> {
- let mut out_info: HashMap<String, V> = HashMap::new();
- entries
- .into_iter()
- .try_for_each(|(fid, finfo): (String, U)| -> anyhow::Result<()> {
- out_info.insert(fid, finfo.try_into()?);
- Ok(())
- })?;
- Ok(StagedFiles { entries: out_info })
- }
-}
-
-impl<U, V, S> std::convert::From<StagedFiles<U>> for HashMap<String, V, S>
-where
- V: std::convert::From<U>,
- S: std::hash::BuildHasher + Default,
-{
- fn from(reg: StagedFiles<U>) -> Self {
- let mut out_info: HashMap<String, V, S> = HashMap::default();
- reg.entries
- .into_iter()
- .for_each(|(fid, finfo): (String, U)| {
- out_info.insert(fid, finfo.into());
- });
- out_info
- }
-}
diff --git a/types/src/staged_function.rs b/types/src/staged_function.rs
index 40d0605..756cfee 100644
--- a/types/src/staged_function.rs
+++ b/types/src/staged_function.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{ExecutorType, StagedFiles, StagedInputFile, StagedOutputFile};
+use crate::{ExecutorType, StagedFiles};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -158,8 +158,8 @@ pub struct StagedFunction {
pub name: String,
pub payload: String,
pub arguments: FunctionArguments,
- pub input_files: StagedFiles<StagedInputFile>,
- pub output_files: StagedFiles<StagedOutputFile>,
+ pub input_files: StagedFiles,
+ pub output_files: StagedFiles,
pub runtime_name: String,
pub executor_type: ExecutorType,
}
@@ -187,14 +187,14 @@ impl StagedFunction {
Self { arguments, ..self }
}
- pub fn input_files(self, input_files: StagedFiles<StagedInputFile>) -> Self {
+ pub fn input_files(self, input_files: StagedFiles) -> Self {
Self {
input_files,
..self
}
}
- pub fn output_files(self, output_files: StagedFiles<StagedOutputFile>) -> Self {
+ pub fn output_files(self, output_files: StagedFiles) -> Self {
Self {
output_files,
..self
diff --git a/worker/src/function/context.rs b/worker/src/function/context.rs
index 5aa5ac2..0a23338 100644
--- a/worker/src/function/context.rs
+++ b/worker/src/function/context.rs
@@ -241,9 +241,8 @@ pub mod tests {
use std::str::FromStr;
use teaclave_test_utils::*;
use teaclave_types::hashmap;
+ use teaclave_types::StagedFileInfo;
use teaclave_types::StagedFiles;
- use teaclave_types::StagedInputFile;
- use teaclave_types::StagedOutputFile;
use teaclave_types::TeaclaveFile128Key;
pub fn run_tests() -> bool {
@@ -263,9 +262,9 @@ pub mod tests {
let input = PathBuf::from_str("fixtures/functions/mesapy/input.txt").unwrap();
let output = PathBuf::from_str("fixtures/functions/mesapy/output.txt.out").unwrap();
- let input_info = StagedInputFile::new(input, TeaclaveFile128Key::random());
+ let input_info = StagedFileInfo::new(input, TeaclaveFile128Key::random());
- let output_info = StagedOutputFile::new(output, TeaclaveFile128Key::random());
+ let output_info = StagedFileInfo::new(output, TeaclaveFile128Key::random());
let in_fid = "in_f1";
let out_fid = "out_f1";
diff --git a/worker/src/function/gbdt_prediction.rs b/worker/src/function/gbdt_prediction.rs
index d20a9e7..1bcc5bc 100644
--- a/worker/src/function/gbdt_prediction.rs
+++ b/worker/src/function/gbdt_prediction.rs
@@ -102,9 +102,8 @@ pub mod tests {
use teaclave_types::hashmap;
use teaclave_types::FunctionArguments;
+ use teaclave_types::StagedFileInfo;
use teaclave_types::StagedFiles;
- use teaclave_types::StagedInputFile;
- use teaclave_types::StagedOutputFile;
use teaclave_types::TeaclaveFile128Key;
use crate::function::TeaclaveFunction;
@@ -124,14 +123,14 @@ pub mod tests {
let input_files = StagedFiles::new(hashmap!(
IN_MODEL.to_string() =>
- StagedInputFile::new(plain_if_model, TeaclaveFile128Key::random()),
+ StagedFileInfo::new(plain_if_model, TeaclaveFile128Key::random()),
IN_DATA.to_string() =>
- StagedInputFile::new(plain_if_data, TeaclaveFile128Key::random())
+ StagedFileInfo::new(plain_if_data, TeaclaveFile128Key::random())
));
let output_files = StagedFiles::new(hashmap!(
OUT_RESULT.to_string() =>
- StagedOutputFile::new(plain_output, TeaclaveFile128Key::random())
+ StagedFileInfo::new(plain_output, TeaclaveFile128Key::random())
));
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
diff --git a/worker/src/function/gbdt_training.rs b/worker/src/function/gbdt_training.rs
index f0913e0..046001c 100644
--- a/worker/src/function/gbdt_training.rs
+++ b/worker/src/function/gbdt_training.rs
@@ -137,9 +137,8 @@ pub mod tests {
use teaclave_types::hashmap;
use teaclave_types::FunctionArguments;
+ use teaclave_types::StagedFileInfo;
use teaclave_types::StagedFiles;
- use teaclave_types::StagedInputFile;
- use teaclave_types::StagedOutputFile;
use teaclave_types::TeaclaveFile128Key;
use crate::function::TeaclaveFunction;
@@ -168,12 +167,12 @@ pub mod tests {
let input_files = StagedFiles::new(hashmap!(
IN_DATA.to_string() =>
- StagedInputFile::new(plain_input, TeaclaveFile128Key::random())
+ StagedFileInfo::new(plain_input, TeaclaveFile128Key::random())
));
let output_files = StagedFiles::new(hashmap!(
OUT_MODEL.to_string() =>
- StagedOutputFile::new(plain_output, TeaclaveFile128Key::random())
+ StagedFileInfo::new(plain_output, TeaclaveFile128Key::random())
));
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
diff --git a/worker/src/function/mesapy.rs b/worker/src/function/mesapy.rs
index 4f0de89..fb7c13b 100644
--- a/worker/src/function/mesapy.rs
+++ b/worker/src/function/mesapy.rs
@@ -109,9 +109,8 @@ pub mod tests {
use crate::runtime::RawIoRuntime;
use teaclave_types::hashmap;
use teaclave_types::FunctionArguments;
+ use teaclave_types::StagedFileInfo;
use teaclave_types::StagedFiles;
- use teaclave_types::StagedInputFile;
- use teaclave_types::StagedOutputFile;
use teaclave_types::TeaclaveFile128Key;
pub fn run_tests() -> bool {
@@ -168,9 +167,9 @@ def entrypoint(argv):
let input = "fixtures/functions/mesapy/input.txt";
let output = "fixtures/functions/mesapy/output.txt";
- let input_info = StagedInputFile::new(input, TeaclaveFile128Key::random());
+ let input_info = StagedFileInfo::new(input, TeaclaveFile128Key::random());
- let output_info = StagedOutputFile::new(output, TeaclaveFile128Key::random());
+ let output_info = StagedFileInfo::new(output, TeaclaveFile128Key::random());
let input_files = StagedFiles {
entries: hashmap!("in_f1".to_string() => input_info),
diff --git a/worker/src/runtime/default.rs b/worker/src/runtime/default.rs
index cf36793..0e47acc 100644
--- a/worker/src/runtime/default.rs
+++ b/worker/src/runtime/default.rs
@@ -23,19 +23,14 @@ use std::io;
use super::TeaclaveRuntime;
use teaclave_types::StagedFiles;
-use teaclave_types::StagedInputFile;
-use teaclave_types::StagedOutputFile;
pub struct DefaultRuntime {
- input_files: StagedFiles<StagedInputFile>,
- output_files: StagedFiles<StagedOutputFile>,
+ input_files: StagedFiles,
+ output_files: StagedFiles,
}
impl DefaultRuntime {
- pub fn new(
- input_files: StagedFiles<StagedInputFile>,
- output_files: StagedFiles<StagedOutputFile>,
- ) -> DefaultRuntime {
+ pub fn new(input_files: StagedFiles, output_files: StagedFiles) -> DefaultRuntime {
DefaultRuntime {
input_files,
output_files,
diff --git a/worker/src/runtime/raw_io.rs b/worker/src/runtime/raw_io.rs
index 8ceb457..4a58742 100644
--- a/worker/src/runtime/raw_io.rs
+++ b/worker/src/runtime/raw_io.rs
@@ -25,19 +25,14 @@ use anyhow;
use super::TeaclaveRuntime;
use teaclave_types::StagedFiles;
-use teaclave_types::StagedInputFile;
-use teaclave_types::StagedOutputFile;
pub struct RawIoRuntime {
- input_files: StagedFiles<StagedInputFile>,
- output_files: StagedFiles<StagedOutputFile>,
+ input_files: StagedFiles,
+ output_files: StagedFiles,
}
impl RawIoRuntime {
- pub fn new(
- input_files: StagedFiles<StagedInputFile>,
- output_files: StagedFiles<StagedOutputFile>,
- ) -> RawIoRuntime {
+ pub fn new(input_files: StagedFiles, output_files: StagedFiles) -> RawIoRuntime {
RawIoRuntime {
input_files,
output_files,
diff --git a/worker/src/worker.rs b/worker/src/worker.rs
index 6166778..60aec18 100644
--- a/worker/src/worker.rs
+++ b/worker/src/worker.rs
@@ -25,8 +25,7 @@ use anyhow;
use serde_json;
use teaclave_types::{
- hashmap, ExecutorType, FunctionArguments, StagedFiles, StagedFunction, StagedInputFile,
- StagedOutputFile, WorkerCapability,
+ hashmap, ExecutorType, FunctionArguments, StagedFiles, StagedFunction, WorkerCapability,
};
use crate::function::{self, TeaclaveFunction};
@@ -88,8 +87,8 @@ impl Worker {
fn get_runtime(
&self,
name: &str,
- input_files: StagedFiles<StagedInputFile>,
- output_files: StagedFiles<StagedOutputFile>,
+ input_files: StagedFiles,
+ output_files: StagedFiles,
) -> anyhow::Result<Box<dyn TeaclaveRuntime + Send + Sync>> {
let build_runtime = self
.runtimes
@@ -175,11 +174,5 @@ fn prepare_arguments(
}
type FunctionBuilder = Box<dyn Fn() -> Box<dyn TeaclaveFunction + Send + Sync> + Send + Sync>;
-type RuntimeBuilder = Box<
- dyn Fn(
- StagedFiles<StagedInputFile>,
- StagedFiles<StagedOutputFile>,
- ) -> Box<dyn TeaclaveRuntime + Send + Sync>
- + Send
- + Sync,
->;
+type RuntimeBuilder =
+ Box<dyn Fn(StagedFiles, StagedFiles) -> Box<dyn TeaclaveRuntime + Send + Sync> + Send + Sync>;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@teaclave.apache.org
For additional commands, e-mail: commits-help@teaclave.apache.org