You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@arrow.apache.org by 1057445597 <10...@qq.com> on 2022/07/01 07:39:28 UTC
why set use thread doesn't work?
Setting set_use_thread(true) is useful when reading local files, but not when reading S3, That's a 10-fold difference, is there any way to improve the speed of reading S3?
// EnsureS3Initialized();
// S3Options s3Options = S3Options::FromAccessKey(K_ACCESS_KEY1, K_SECRET_KEY1);
// s3Options.endpoint_override = K_ENDPOINT_OVERRIDE1;
// std::shared_ptr<S3FileSystem> s3fs = S3FileSystem::Make(s3Options).ValueOrDie();
// auto infile = s3fs->OpenInputFile(path).ValueOrDie();
std::shared_ptr<arrow::io::ReadableFile> infile;
PARQUET_ASSIGN_OR_THROW(infile,
arrow::io::ReadableFile::Open(path,
arrow::default_memory_pool()));
std::unique_ptr<parquet::arrow::FileReader> reader;
PARQUET_THROW_NOT_OK(
parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader));
reader->set_use_threads(false);
// reader->set_use_threads(true);
std::shared_ptr<arrow::Table> table;
1057445597
1057445597@qq.com