[I] ObjectStore::get_opts Incorrectly Returns Response Size not Object Size [arrow-rs-object-store]

["CarlKCarlK (via GitHub)" <gi...@apache.org>]

CarlKCarlK opened a new issue, #99:
URL: https://github.com/apache/arrow-rs-object-store/issues/99

   get_opts is documented to return the metadata of the object. When used on InMemory it does the right thing. However, when used on an S3 store, it can return the metadata of a region rather than the object.
   
   Here is repro showing the issue:
   
   ```rust
   use std::collections::HashMap;
   
   use bytes::Bytes;
   use object_store::{memory::InMemory, path::Path as StorePath, GetOptions, ObjectStore};
   use tokio::runtime;
   use url::Url;
   
   fn main() {
       let rt = runtime::Runtime::new().unwrap();
       rt.block_on(async {
           // Create an in-memory object store
           let object_store = InMemory::new();
           let store_path = StorePath::from("data/file1");
           let bytes = Bytes::from_static(b"hello");
           object_store.put(&store_path, bytes).await.unwrap();
   
           // Ask for the object's metadata
           let object_meta = object_store.head(&store_path).await.unwrap();
           assert_eq!(object_meta.size, 5);
   
           // We read a region and get the metadata in one go?
           let get_options = GetOptions {
               range: Some(0..2),
               ..Default::default()
           };
           let get_result = object_store
               .get_opts(&store_path, get_options)
               .await
               .unwrap();
   
           // ============================================================
           // NOTE: The metadata is the same as the one we got from `head`
           // ============================================================
           assert_eq!(get_result.meta.size, 5);
           let bytes = get_result.bytes().await.unwrap();
           assert_eq!(bytes, Bytes::from_static(b"he"));
   
           // Next, we try to read from a public S3 bucket
   
           // arn:aws:s3:::ubc-sunflower-genome
           // AWS Region
           // us-west-2
           // AWS CLI Access (No AWS account required)
           // aws s3 ls --no-sign-request s3://ubc-sunflower-genome/
           // >aws s3 ls --no-sign-request s3://ubc-sunflower-genome/sequence/sra/PRJNA322345/SRR3648257.sra
           // 2020-04-05 16:22:18 7064209461 SRR3648257.sra
   
           // Any AWS credentials will do
           use rusoto_credential::{ProfileProvider, ProvideAwsCredentials};
           let credentials = ProfileProvider::new().unwrap().credentials().await.unwrap();
           let url = "s3://ubc-sunflower-genome/sequence/sra/PRJNA322345/SRR3648257.sra";
           let url = Url::parse(url).unwrap();
           let options: HashMap<&str, &str> = [
               ("aws_access_key_id", credentials.aws_access_key_id()),
               ("aws_secret_access_key", credentials.aws_secret_access_key()),
               ("aws_region", "us-west-2"),
           ]
           .iter()
           .cloned()
           .collect();
           let (object_store, store_path) = object_store::parse_url_opts(&url, options).unwrap();
   
           // Get the metadata
           let object_meta = object_store.head(&store_path).await.unwrap();
           assert_eq!(object_meta.size, 7064209461);
   
           // Can read a region and get the metadata in one go?
           let get_options = GetOptions {
               range: Some(0..2),
               ..Default::default()
           };
           let get_result = object_store
               .get_opts(&store_path, get_options)
               .await
               .unwrap();
   
           // ============================================================
           // NOTE: The metadata is NOT the same as the one we got from `head`.
           // `meta` is documented as "The [`ObjectMeta`] for this object".
           // However, it is not the metadata for the object, but the metadata
           // for the region we read.
           // ============================================================
           assert_eq!(get_result.meta.size, 2);
           let bytes = get_result.bytes().await.unwrap();
           assert_eq!(bytes, Bytes::from_static(b"NC"));
       })
   }
   ```
   ```
   [package]
   name = "object-store-issue"
   version = "0.1.0"
   edition = "2021"
   
   [dependencies]
   tokio = { version = "1.35.0", features = ["full"] }
   object_store = { version = "0.8.0", features = ["aws"] }
   bytes = "1.5.0"
   url = "2.5.0"
   rusoto_credential = "0.48.0"
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org