You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2015/12/12 19:16:06 UTC
marmotta git commit: optional compressed input support for command
line client
Repository: marmotta
Updated Branches:
refs/heads/develop e0e5145de -> 134abbb8c
optional compressed input support for command line client
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/134abbb8
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/134abbb8
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/134abbb8
Branch: refs/heads/develop
Commit: 134abbb8ccc16124f06cd55fc533982bec7cd926
Parents: e0e5145
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Sat Dec 12 19:17:33 2015 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Sat Dec 12 19:17:33 2015 +0100
----------------------------------------------------------------------
libraries/ostrich/backend/CMakeLists.txt | 6 ++++
libraries/ostrich/backend/client/CMakeLists.txt | 2 +-
libraries/ostrich/backend/client/client.cc | 31 ++++++++++++++++++++
3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/marmotta/blob/134abbb8/libraries/ostrich/backend/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/CMakeLists.txt b/libraries/ostrich/backend/CMakeLists.txt
index d8232b7..608cb2a 100644
--- a/libraries/ostrich/backend/CMakeLists.txt
+++ b/libraries/ostrich/backend/CMakeLists.txt
@@ -14,10 +14,16 @@ find_package (Protobuf REQUIRED)
find_package (GRPC REQUIRED)
find_package (LevelDB REQUIRED)
find_package (GLog REQUIRED)
+find_package (Boost 1.54.0 COMPONENTS iostreams)
find_package (Tcmalloc)
add_definitions(-DNDEBUG)
+if(Boost_IOSTREAMS_FOUND)
+ message(STATUS "Enabling gzip/bzip2 support (Boost iostreams found)")
+ add_definitions(-DHAVE_IOSTREAMS)
+endif(Boost_IOSTREAMS_FOUND)
+
add_subdirectory(util)
add_subdirectory(model)
add_subdirectory(sparql)
http://git-wip-us.apache.org/repos/asf/marmotta/blob/134abbb8/libraries/ostrich/backend/client/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/client/CMakeLists.txt b/libraries/ostrich/backend/client/CMakeLists.txt
index 88ad11a..220799f 100644
--- a/libraries/ostrich/backend/client/CMakeLists.txt
+++ b/libraries/ostrich/backend/client/CMakeLists.txt
@@ -3,6 +3,6 @@ include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR
add_executable(marmotta_client client.cc)
target_link_libraries(marmotta_client
marmotta_model marmotta_service marmotta_parser marmotta_serializer
- ${GFLAGS_LIBRARY}
+ ${GFLAGS_LIBRARY} ${Boost_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES} ${GRPC_LIBRARIES})
http://git-wip-us.apache.org/repos/asf/marmotta/blob/134abbb8/libraries/ostrich/backend/client/client.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/client/client.cc b/libraries/ostrich/backend/client/client.cc
index 16c9022..56f317c 100644
--- a/libraries/ostrich/backend/client/client.cc
+++ b/libraries/ostrich/backend/client/client.cc
@@ -17,6 +17,14 @@
*/
#include <fstream>
+#ifdef HAVE_IOSTREAMS
+// support b/gzipped files
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/copy.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#endif
+
#include <grpc/grpc.h>
#include <grpc++/channel.h>
#include <grpc++/client_context.h>
@@ -51,6 +59,10 @@ using namespace marmotta;
namespace svc = marmotta::service::proto;
namespace spq = marmotta::sparql::proto;
+#ifdef HAVE_IOSTREAMS
+using namespace boost::iostreams;
+#endif
+
// A STL iterator wrapper around a client reader.
template <class T, class Proto>
class ClientReaderIterator : public util::CloseableIterator<T> {
@@ -212,6 +224,11 @@ DEFINE_string(host, "localhost", "Address/name of server to access.");
DEFINE_string(port, "10000", "Port of server to access.");
DEFINE_string(output, "", "File to write result to.");
+#ifdef HAVE_IOSTREAMS
+DEFINE_bool(gzip, false, "Input files are gzip compressed.");
+DEFINE_bool(bzip2, false, "Input files are bzip2 compressed.");
+#endif
+
int main(int argc, char** argv) {
GOOGLE_PROTOBUF_VERIFY_VERSION;
@@ -220,7 +237,21 @@ int main(int argc, char** argv) {
MarmottaClient client(FLAGS_host + ":" + FLAGS_port);
if ("import" == std::string(argv[1])) {
+#ifdef HAVE_IOSTREAMS
+ std::ifstream file(argv[2]);
+ filtering_streambuf<input> cin;
+ if (FLAGS_bzip2) {
+ cin.push(bzip2_decompressor());
+ }
+ if (FLAGS_gzip) {
+ cin.push(gzip_decompressor());
+ }
+ cin.push(file);
+
+ std::istream in(&cin);
+#else
std::ifstream in(argv[2]);
+#endif
std::cout << "Importing " << argv[2] << " ... " << std::endl;
client.importDataset(in, parser::FormatFromString(FLAGS_format));
std::cout << "Finished!" << std::endl;