You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ho...@apache.org on 2024/03/28 05:00:21 UTC
(opendal) 01/01: feat(services/github): make access_token optional
This is an automated email from the ASF dual-hosted git repository.
hoslo pushed a commit to branch github-token
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit d64837c632b3fd8400095bc0de2aee0b77911203
Author: Jun Ouyang <ju...@apache.org>
AuthorDate: Wed Mar 27 19:00:37 2024 +0800
feat(services/github): make access_token optional
---
bin/ofs/src/fuse.rs | 171 ++++++++++++++++++++++++++++++++++--
core/src/services/github/backend.rs | 15 ++--
core/src/services/github/core.rs | 25 ++++--
3 files changed, 187 insertions(+), 24 deletions(-)
diff --git a/bin/ofs/src/fuse.rs b/bin/ofs/src/fuse.rs
index 40df1b041e..c8bb50dbc6 100644
--- a/bin/ofs/src/fuse.rs
+++ b/bin/ofs/src/fuse.rs
@@ -20,10 +20,12 @@ use std::ffi::OsString;
use std::num::NonZeroU32;
use std::ops::Deref;
use std::path::PathBuf;
+use std::sync::RwLock;
use std::time::Duration;
use std::time::SystemTime;
use bytes::Bytes;
+
use fuse3::path::prelude::*;
use fuse3::Errno;
use fuse3::Result;
@@ -46,6 +48,7 @@ struct OpenedFile {
is_read: bool,
is_write: bool,
is_append: bool,
+ offset: u64,
}
#[derive(Debug, Clone, Copy)]
@@ -72,7 +75,7 @@ pub(super) struct Fuse {
op: Operator,
gid: u32,
uid: u32,
- opened_files: Slab<OpenedFile>,
+ opened_files: Slab<RwLock<OpenedFile>>,
}
impl Fuse {
@@ -107,13 +110,18 @@ impl Fuse {
// Get opened file and check given path
fn get_opened_file(&self, key: FileKey, path: Option<&OsStr>) -> Result<OpenedFile> {
- let file = self
+ let file = match self
.opened_files
.get(key.0)
.as_ref()
.ok_or(Errno::from(libc::ENOENT))?
.deref()
- .clone();
+ .read()
+ {
+ Ok(file) => file.clone(),
+ Err(_) => Err(Errno::from(libc::EBADF))?,
+ };
+
if matches!(path, Some(path) if path != file.path) {
log::trace!(
"get_opened_file: path not match: path={:?}, file={:?}",
@@ -125,6 +133,37 @@ impl Fuse {
Ok(file)
}
+
+ // Set opened file offset
+ fn set_opened_file_offset(
+ &self,
+ key: FileKey,
+ path: Option<&OsStr>,
+ offset: u64,
+ ) -> Result<()> {
+ let binding = self.opened_files.get(key.0);
+ let mut file = match binding
+ .as_ref()
+ .ok_or(Errno::from(libc::ENOENT))?
+ .deref()
+ .write()
+ {
+ Ok(file) => file,
+ Err(_) => Err(Errno::from(libc::EBADF))?,
+ };
+ if matches!(path, Some(path) if path != file.path) {
+ log::trace!(
+ "set_opened_file: path not match: path={:?}, file={:?}",
+ path,
+ file.path
+ );
+ Err(Errno::from(libc::EBADF))?;
+ }
+
+ file.offset = offset;
+
+ Ok(())
+ }
}
impl PathFilesystem for Fuse {
@@ -170,6 +209,7 @@ impl PathFilesystem for Fuse {
self.opened_files
.get(FileKey::try_from(fh).ok()?.0)
.as_ref()
+ .and_then(|f| f.read().ok())
.map(|f| f.path.clone())
});
@@ -208,7 +248,8 @@ impl PathFilesystem for Fuse {
fh,
set_attr
);
- Err(libc::EOPNOTSUPP.into())
+
+ self.getattr(_req, path, fh, 0).await
}
async fn symlink(
@@ -313,6 +354,10 @@ impl PathFilesystem for Fuse {
name
);
+ if !self.op.info().full_capability().rename {
+ return Err(Errno::from(libc::ENOTSUP))?;
+ }
+
let origin_path = PathBuf::from(origin_parent).join(origin_name);
let path = PathBuf::from(parent).join(name);
@@ -369,12 +414,13 @@ impl PathFilesystem for Fuse {
let key = self
.opened_files
- .insert(OpenedFile {
+ .insert(RwLock::new(OpenedFile {
path: path.into(),
is_read,
is_write,
is_append: flags & libc::O_APPEND as u32 != 0,
- })
+ offset: 0,
+ }))
.ok_or(Errno::from(libc::EBUSY))?;
Ok(ReplyCreated {
@@ -407,6 +453,7 @@ impl PathFilesystem for Fuse {
let file = self
.opened_files
.take(FileKey::try_from(fh)?.0)
+ .map(|f| f.read().unwrap().clone())
.ok_or(Errno::from(libc::EBADF))?;
if matches!(path, Some(ref p) if p != &file.path) {
Err(Errno::from(libc::EBADF))?;
@@ -422,12 +469,13 @@ impl PathFilesystem for Fuse {
let key = self
.opened_files
- .insert(OpenedFile {
+ .insert(RwLock::new(OpenedFile {
path: path.into(),
is_read,
is_write,
is_append: flags & libc::O_APPEND as u32 != 0,
- })
+ offset: 0,
+ }))
.ok_or(Errno::from(libc::EBUSY))?;
Ok(ReplyOpen {
@@ -465,6 +513,8 @@ impl PathFilesystem for Fuse {
.await
.map_err(opendal_error2errno)?;
+ self.set_opened_file_offset(FileKey::try_from(fh)?, path, offset + data.len() as u64)?;
+
Ok(ReplyData { data: data.into() })
}
@@ -505,6 +555,8 @@ impl PathFilesystem for Fuse {
.await
.map_err(opendal_error2errno)?;
+ self.set_opened_file_offset(FileKey::try_from(fh)?, path, offset + data.len() as u64)?;
+
Ok(ReplyWrite {
written: data.len() as _,
})
@@ -638,6 +690,109 @@ impl PathFilesystem for Fuse {
entries: relative_paths.chain(children).skip(offset as usize).boxed(),
})
}
+ async fn rename2(
+ &self,
+ req: Request,
+ origin_parent: &OsStr,
+ origin_name: &OsStr,
+ parent: &OsStr,
+ name: &OsStr,
+ _flags: u32,
+ ) -> Result<()> {
+ log::debug!(
+ "rename2(origin_parent={:?}, origin_name={:?}, parent={:?}, name={:?})",
+ origin_parent,
+ origin_name,
+ parent,
+ name
+ );
+ self.rename(req, origin_parent, origin_name, parent, name)
+ .await
+ }
+
+ async fn lseek(
+ &self,
+ _req: Request,
+ path: Option<&OsStr>,
+ fh: u64,
+ offset: u64,
+ whence: u32,
+ ) -> Result<ReplyLSeek> {
+ log::debug!(
+ "lseek(path={:?}, fh={}, offset={}, whence={})",
+ path,
+ fh,
+ offset,
+ whence
+ );
+
+ let whence = whence as i32;
+
+ let file = self.get_opened_file(FileKey::try_from(fh)?, path)?;
+
+ if !file.is_read && !file.is_write {
+ Err(Errno::from(libc::EACCES))?;
+ }
+
+ let offset = if whence == libc::SEEK_SET {
+ offset
+ } else if whence == libc::SEEK_CUR {
+ file.offset + offset
+ } else if whence == libc::SEEK_END {
+ let metadata = self
+ .op
+ .stat(&path.unwrap().to_string_lossy())
+ .await
+ .map_err(opendal_error2errno)?;
+ let content_size = metadata.content_length();
+
+ if content_size >= offset as _ {
+ content_size as u64 - offset
+ } else {
+ 0
+ }
+ } else {
+ return Err(libc::ENOSYS.into());
+ };
+
+ Ok(ReplyLSeek { offset })
+ }
+
+ async fn copy_file_range(
+ &self,
+ req: Request,
+ from_path: Option<&OsStr>,
+ fh_in: u64,
+ offset_in: u64,
+ to_path: Option<&OsStr>,
+ fh_out: u64,
+ offset_out: u64,
+ length: u64,
+ flags: u64,
+ ) -> Result<ReplyCopyFileRange> {
+ log::debug!(
+ "copy_file_range(from_path={:?}, fh_in={}, offset_in={}, to_path={:?}, fh_out={}, offset_out={}, length={}, flags={})",
+ from_path,
+ fh_in,
+ offset_in,
+ to_path,
+ fh_out,
+ offset_out,
+ length,
+ flags
+ );
+ let data = self
+ .read(req, from_path, fh_in, offset_in, length as _)
+ .await?;
+
+ let ReplyWrite { written } = self
+ .write(req, to_path, fh_out, offset_out, &data.data, 0, flags as _)
+ .await?;
+
+ Ok(ReplyCopyFileRange {
+ copied: u64::from(written),
+ })
+ }
}
const fn entry_mode2file_type(mode: EntryMode) -> FileType {
diff --git a/core/src/services/github/backend.rs b/core/src/services/github/backend.rs
index b9a3d6e5c3..d004288d2c 100644
--- a/core/src/services/github/backend.rs
+++ b/core/src/services/github/backend.rs
@@ -45,8 +45,10 @@ pub struct GithubConfig {
pub root: Option<String>,
/// Github access_token.
///
- /// required.
- pub token: String,
+ /// optional.
+ /// If not provided, the backend will only support read operations for public repositories.
+ /// And has
+ pub token: Option<String>,
/// Github repo owner.
///
/// required.
@@ -105,7 +107,7 @@ impl GithubBuilder {
///
/// required.
pub fn token(&mut self, token: &str) -> &mut Self {
- self.config.token = token.to_string();
+ self.config.token = Some(token.to_string());
self
}
@@ -168,13 +170,6 @@ impl Builder for GithubBuilder {
let root = normalize_root(&self.config.root.clone().unwrap_or_default());
debug!("backend use root {}", &root);
- // Handle token.
- if self.config.token.is_empty() {
- return Err(Error::new(ErrorKind::ConfigInvalid, "token is empty")
- .with_operation("Builder::build")
- .with_context("service", Scheme::Github));
- }
-
// Handle owner.
if self.config.owner.is_empty() {
return Err(Error::new(ErrorKind::ConfigInvalid, "owner is empty")
diff --git a/core/src/services/github/core.rs b/core/src/services/github/core.rs
index 0c93c83aa5..799cbe4fce 100644
--- a/core/src/services/github/core.rs
+++ b/core/src/services/github/core.rs
@@ -39,7 +39,7 @@ pub struct GithubCore {
/// The root of this core.
pub root: String,
/// Github access_token.
- pub token: String,
+ pub token: Option<String>,
/// Github repo owner.
pub owner: String,
/// Github repo name.
@@ -65,19 +65,32 @@ impl GithubCore {
}
pub fn sign(&self, req: request::Builder) -> Result<request::Builder> {
- let req = req
+ let mut req = req
.header(header::USER_AGENT, format!("opendal-{}", VERSION))
.header("X-GitHub-Api-Version", "2022-11-28");
- Ok(req.header(
- header::AUTHORIZATION,
- format_authorization_by_bearer(&self.token)?,
- ))
+ // Github access_token is optional.
+ if let Some(token) = &self.token {
+ req = req.header(
+ header::AUTHORIZATION,
+ format_authorization_by_bearer(token)?,
+ )
+ }
+
+ Ok(req)
}
}
impl GithubCore {
pub async fn get_file_sha(&self, path: &str) -> Result<Option<String>> {
+ // if the token is not set, we shhould not try to get the sha of the file.
+ if self.token.is_none() {
+ return Err(Error::new(
+ ErrorKind::PermissionDenied,
+ "Github access_token is not set",
+ ));
+ }
+
let resp = self.stat(path).await?;
match resp.status() {