You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/04/08 18:03:33 UTC
orc git commit: ORC-43. FileMemory.cc throws uncaught exception.
(omalley reviewed by asandryh)
Repository: orc
Updated Branches:
refs/heads/master 73248622e -> da9b62f3e
ORC-43. FileMemory.cc throws uncaught exception. (omalley reviewed by
asandryh)
This fixes #19
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/da9b62f3
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/da9b62f3
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/da9b62f3
Branch: refs/heads/master
Commit: da9b62f3e2c04365e9ba158bb1afe3452c414485
Parents: 7324862
Author: Owen O'Malley <om...@apache.org>
Authored: Tue Feb 23 15:14:29 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Apr 8 09:02:35 2016 -0700
----------------------------------------------------------------------
tools/src/FileMemory.cc | 72 ++++++++++++++++++++++++--------------------
1 file changed, 39 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/da9b62f3/tools/src/FileMemory.cc
----------------------------------------------------------------------
diff --git a/tools/src/FileMemory.cc b/tools/src/FileMemory.cc
index de834fc..696000f 100644
--- a/tools/src/FileMemory.cc
+++ b/tools/src/FileMemory.cc
@@ -59,6 +59,37 @@ public:
TestMemoryPool::~TestMemoryPool() {}
+void processFile(const char* filename,
+ const std::list<uint64_t>& cols,
+ uint32_t batchSize) {
+ orc::ReaderOptions opts;
+ if (cols.size() > 0) {
+ opts.include(cols);
+ }
+ std::unique_ptr<orc::MemoryPool> pool(new TestMemoryPool());
+ opts.setMemoryPool(*(pool.get()));
+
+ std::unique_ptr<orc::Reader> reader =
+ orc::createReader(orc::readLocalFile(std::string(filename)), opts);
+
+ std::unique_ptr<orc::ColumnVectorBatch> batch =
+ reader->createRowBatch(batchSize);
+ uint64_t readerMemory = reader->getMemoryUse();
+ uint64_t batchMemory = batch->getMemoryUsage();
+ while (reader->next(*batch)) {}
+ uint64_t actualMemory =
+ static_cast<TestMemoryPool*>(pool.get())->getMaxMemory();
+ std::cout << "Reader memory estimate: " << readerMemory
+ << "\nBatch memory estimate: " ;
+ if (batch->hasVariableLength()) {
+ std::cout << "Cannot estimate because reading ARRAY or MAP columns";
+ } else {
+ std::cout << batchMemory
+ << "\nTotal memory estimate: " << readerMemory + batchMemory;
+ }
+ std::cout << "\nActual max memory used: " << actualMemory << "\n";
+}
+
int main(int argc, char* argv[]) {
if (argc < 2) {
std::cout << "Usage: file-memory [--columns=column1,column2,...] "
@@ -84,7 +115,8 @@ int main(int argc, char* argv[]) {
value = std::strtok(ORC_NULLPTR, "," );
}
} else if ( (param=strstr(argv[i], BATCH_PREFIX.c_str())) ) {
- batchSize = static_cast<uint32_t>(std::atoi(param+BATCH_PREFIX.length()));
+ batchSize =
+ static_cast<uint32_t>(std::atoi(param+BATCH_PREFIX.length()));
} else {
filename = argv[i];
}
@@ -95,37 +127,11 @@ int main(int argc, char* argv[]) {
return 1;
}
- orc::ReaderOptions opts;
- if (cols.size() > 0) {
- opts.include(cols);
- }
- std::unique_ptr<orc::MemoryPool> pool(new TestMemoryPool());
- opts.setMemoryPool(*(pool.get()));
-
- std::unique_ptr<orc::Reader> reader;
- try{
- reader = orc::createReader(orc::readLocalFile(std::string(filename)), opts);
- } catch (orc::ParseError e) {
- std::cout << "Error reading file " << filename << "! " << e.what() << "\n";
- return -1;
- }
-
- std::unique_ptr<orc::ColumnVectorBatch> batch =
- reader->createRowBatch(batchSize);
- uint64_t readerMemory = reader->getMemoryUse();
- uint64_t batchMemory = batch->getMemoryUsage();
- while (reader->next(*batch)) {}
- uint64_t actualMemory =
- static_cast<TestMemoryPool*>(pool.get())->getMaxMemory();
- std::cout << "Reader memory estimate: " << readerMemory
- << "\nBatch memory estimate: " ;
- if (batch->hasVariableLength()) {
- std::cout << "Cannot estimate because reading ARRAY or MAP columns";
- } else {
- std::cout << batchMemory
- << "\nTotal memory estimate: " << readerMemory + batchMemory;
+ try {
+ processFile(filename, cols, batchSize);
+ return 0;
+ } catch (std::exception& ex) {
+ std::cerr << "Caught exception: " << ex.what() << "\n";
+ return 1;
}
- std::cout << "\nActual max memory used: " << actualMemory << "\n";
-
- return 0;
}