You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/01/08 21:21:12 UTC
orc git commit: ORC-36. Create a test program that writes ORC files
from future versions.
Repository: orc
Updated Branches:
refs/heads/master 0d5a92e1a -> 194a0da8c
ORC-36. Create a test program that writes ORC files from future versions.
Fixes apache/orc#14
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/194a0da8
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/194a0da8
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/194a0da8
Branch: refs/heads/master
Commit: 194a0da8cbea0d713d1dd2f142dbf955b5c2029a
Parents: 0d5a92e
Author: Owen O'Malley <om...@apache.org>
Authored: Fri Jan 8 11:20:41 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Jan 8 12:20:40 2016 -0800
----------------------------------------------------------------------
c++/test/CMakeLists.txt | 9 ++++
c++/test/CreateTestFiles.cc | 87 +++++++++++++++++++++++++++++++++++++++
examples/version1999.orc | Bin 38 -> 46 bytes
3 files changed, 96 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/194a0da8/c++/test/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/c++/test/CMakeLists.txt b/c++/test/CMakeLists.txt
index cd417c8..4a09e9e 100644
--- a/c++/test/CMakeLists.txt
+++ b/c++/test/CMakeLists.txt
@@ -41,5 +41,14 @@ target_link_libraries (test-orc
${SNAPPY_LIBRARIES}
)
+add_executable (create-test-files
+ CreateTestFiles.cc
+)
+
+target_link_libraries (create-test-files
+ orc
+ ${PROTOBUF_LIBRARIES}
+)
+
add_test (test-orc test-orc)
http://git-wip-us.apache.org/repos/asf/orc/blob/194a0da8/c++/test/CreateTestFiles.cc
----------------------------------------------------------------------
diff --git a/c++/test/CreateTestFiles.cc b/c++/test/CreateTestFiles.cc
new file mode 100644
index 0000000..c64de98
--- /dev/null
+++ b/c++/test/CreateTestFiles.cc
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Adaptor.hh"
+#include "wrap/orc-proto-wrapper.hh"
+
+#include <fstream>
+#include <iostream>
+#include <string>
+
+/**
+ * Write an empty custom ORC file with a lot of control over format.
+ * We use this to create files to test the reader rather than anything
+ * that users would want to do.
+ */
+void writeCustomOrcFile(const std::string& filename,
+ const orc::proto::Metadata& metadata,
+ const orc::proto::Footer& footer,
+ const std::vector<uint>& version,
+ uint writerVersion) {
+ std::fstream output(filename.c_str(),
+ std::ios::out | std::ios::trunc | std::ios::binary);
+ output << "ORC";
+ if (!metadata.SerializeToOstream(&output)) {
+ std::cerr << "Failed to write metadata for " << filename << "\n";
+ exit(1);
+ }
+ if (!footer.SerializeToOstream(&output)) {
+ std::cerr << "Failed to write footer for " << filename << "\n";
+ exit(1);
+ }
+ orc::proto::PostScript ps;
+ ps.set_footerlength(static_cast<uint64_t>(footer.ByteSize()));
+ ps.set_compression(orc::proto::NONE);
+ ps.set_compressionblocksize(64*1024);
+ for(uint i=0; i < version.size(); ++i) {
+ ps.add_version(version[i]);
+ }
+ ps.set_metadatalength(static_cast<uint64_t>(metadata.ByteSize()));
+ ps.set_writerversion(writerVersion);
+ ps.set_magic("ORC");
+ if (!ps.SerializeToOstream(&output)) {
+ std::cerr << "Failed to write postscript for " << filename << "\n";
+ exit(1);
+ }
+ output.put(static_cast<char>(ps.ByteSize()));
+}
+
+/**
+ * Create a file from a future version 19.99.
+ */
+void writeVersion1999() {
+ orc::proto::Metadata meta;
+ orc::proto::Footer footer;
+ footer.set_headerlength(3);
+ footer.set_contentlength(3);
+ orc::proto::Type* type = footer.add_types();
+ type->set_kind(orc::proto::Type_Kind_STRUCT);
+ footer.set_numberofrows(0);
+ footer.set_rowindexstride(10000);
+ orc::proto::ColumnStatistics* stats = footer.add_statistics();
+ stats->set_numberofvalues(0);
+ stats->set_hasnull(false);
+ std::vector<uint> version{19, 99};
+ writeCustomOrcFile("version1999.orc", meta, footer, version, 1);
+}
+
+int main(int, char *[]) {
+ writeVersion1999();
+ google::protobuf::ShutdownProtobufLibrary();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/orc/blob/194a0da8/examples/version1999.orc
----------------------------------------------------------------------
diff --git a/examples/version1999.orc b/examples/version1999.orc
index c36ef97..1748c70 100644
Binary files a/examples/version1999.orc and b/examples/version1999.orc differ