You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2015/12/19 16:35:09 UTC
[7/8] marmotta git commit: refactor iterator implementations,
they were broken
refactor iterator implementations, they were broken
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/2e32da5d
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/2e32da5d
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/2e32da5d
Branch: refs/heads/develop
Commit: 2e32da5d9cc298272edce2a64e22088335107f21
Parents: 27889eb
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Sat Dec 19 16:15:46 2015 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Sat Dec 19 16:15:46 2015 +0100
----------------------------------------------------------------------
libraries/ostrich/backend/CMakeLists.txt | 4 +-
libraries/ostrich/backend/client/client.cc | 27 +-
.../ostrich/backend/model/rdf_operators.cc | 8 +-
.../ostrich/backend/persistence/CMakeLists.txt | 22 +-
.../backend/persistence/leveldb_persistence.cc | 63 +-
.../backend/persistence/leveldb_service.cc | 22 +-
.../backend/persistence/leveldb_sparql.cc | 25 +-
.../backend/persistence/marmotta_updatedb.cc | 225 +
.../backend/serializer/serializer_base.h | 4 +-
.../ostrich/backend/sparql/rasqal_adapter.cc | 13 +-
libraries/ostrich/backend/test/CMakeLists.txt | 17 +-
.../ostrich/backend/test/PersistenceTest.cc | 76 +
libraries/ostrich/backend/test/SparqlTest.cc | 29 +-
libraries/ostrich/backend/test/StatementTest.cc | 2 +-
.../ostrich/backend/test/gmock-gtest-all.cc | 12243 ++++++++++
libraries/ostrich/backend/test/gmock/gmock.h | 14978 ++++++++++++
libraries/ostrich/backend/test/gtest-all.cc | 9592 --------
libraries/ostrich/backend/test/gtest.h | 20061 ----------------
libraries/ostrich/backend/test/gtest/gtest.h | 21197 +++++++++++++++++
libraries/ostrich/backend/test/main.cc | 2 +-
libraries/ostrich/backend/util/iterator.h | 173 +-
21 files changed, 48949 insertions(+), 29834 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/CMakeLists.txt b/libraries/ostrich/backend/CMakeLists.txt
index 6a96f63..e05c28b 100644
--- a/libraries/ostrich/backend/CMakeLists.txt
+++ b/libraries/ostrich/backend/CMakeLists.txt
@@ -14,10 +14,10 @@ find_package (Protobuf REQUIRED)
find_package (GRPC REQUIRED)
find_package (LevelDB REQUIRED)
find_package (GLog REQUIRED)
-find_package (Boost 1.54.0 COMPONENTS iostreams)
+find_package (Boost 1.54.0 COMPONENTS iostreams filesystem system)
find_package (Tcmalloc)
-add_definitions(-DNDEBUG)
+#add_definitions(-DNDEBUG)
if (Boost_IOSTREAMS_FOUND)
message(STATUS "Enabling gzip/bzip2 support (Boost iostreams found)")
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/client/client.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/client/client.cc b/libraries/ostrich/backend/client/client.cc
index da875a4..396389e 100644
--- a/libraries/ostrich/backend/client/client.cc
+++ b/libraries/ostrich/backend/client/client.cc
@@ -69,28 +69,24 @@ class ClientReaderIterator : public util::CloseableIterator<T> {
public:
ClientReaderIterator() : finished(true) { }
- ClientReaderIterator(ClientReader<Proto>* r) : reader(r), finished(false) {
- // Immediately move to first element.
- operator++();
+ ClientReaderIterator(ClientReader<Proto>* r) : reader(r) {
+ finished = !reader->Read(&buffer);
}
- ClientReaderIterator& operator++() override {
+ const T& next() override {
+ current_ = T(buffer);
+
if (!finished) {
finished = !reader->Read(&buffer);
- current = T(buffer);
if (finished) {
reader->Finish();
}
}
- return *this;
- }
-
- T& operator*() override {
- return current;
+ return current_;
}
- T* operator->() override {
- return ¤t;
+ const T& current() const override {
+ return current_;
}
bool hasNext() override {
@@ -100,7 +96,7 @@ class ClientReaderIterator : public util::CloseableIterator<T> {
private:
ClientReader<Proto>* reader;
Proto buffer;
- T current;
+ T current_;
bool finished;
};
@@ -208,8 +204,9 @@ class MarmottaClient {
stub_->GetNamespaces(&context, pattern));
NamespaceReader it(reader.get());
- for (; it.hasNext(); ++it) {
- out << (*it).getPrefix() << " = " << (*it).getUri() << std::endl;
+ while (it.hasNext()) {
+ const auto& ns = it.next();
+ out << ns.getPrefix() << " = " << ns.getUri() << std::endl;
}
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/model/rdf_operators.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/model/rdf_operators.cc b/libraries/ostrich/backend/model/rdf_operators.cc
index b9e49ad..b2bc2a3 100644
--- a/libraries/ostrich/backend/model/rdf_operators.cc
+++ b/libraries/ostrich/backend/model/rdf_operators.cc
@@ -28,14 +28,18 @@ bool operator==(const Value &lhs, const Value &rhs) {
} else if (lhs.resource().has_bnode() && rhs.resource().has_bnode()) {
return lhs.resource().bnode() == rhs.resource().bnode();
}
+ return (lhs.resource().has_uri() == rhs.resource().has_uri()) &&
+ (lhs.resource().has_bnode() == rhs.resource().has_bnode());
} else if(lhs.has_literal() && rhs.has_literal()) {
if (lhs.literal().has_stringliteral() && rhs.literal().has_stringliteral()) {
return lhs.literal().stringliteral() == rhs.literal().stringliteral();
} else if (lhs.literal().has_dataliteral() && rhs.literal().has_dataliteral()) {
return lhs.literal().dataliteral() == rhs.literal().dataliteral();
}
+ return (lhs.literal().has_stringliteral() == rhs.literal().has_stringliteral()) &&
+ (lhs.literal().has_dataliteral() == rhs.literal().has_dataliteral());
}
- return false;
+ return (lhs.has_resource() == rhs.has_resource()) && (lhs.has_literal() == rhs.has_literal());
}
bool operator==(const Resource &lhs, const Resource &rhs) {
@@ -44,7 +48,7 @@ bool operator==(const Resource &lhs, const Resource &rhs) {
} else if (lhs.has_bnode() && rhs.has_bnode()) {
return lhs.bnode() == rhs.bnode();
}
- return false;
+ return (lhs.has_uri() == rhs.has_uri()) && (lhs.has_bnode() == rhs.has_bnode());
}
bool operator==(const Statement &lhs, const Statement &rhs) {
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/persistence/CMakeLists.txt b/libraries/ostrich/backend/persistence/CMakeLists.txt
index d4a274a..8392c61 100644
--- a/libraries/ostrich/backend/persistence/CMakeLists.txt
+++ b/libraries/ostrich/backend/persistence/CMakeLists.txt
@@ -1,11 +1,21 @@
include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/../model ${RAPTOR_INCLUDE_DIR}/raptor2)
+# Shared Marmotta Ostrich persistence implementation
+add_library(marmotta_leveldb
+ leveldb_persistence.cc leveldb_persistence.h leveldb_sparql.cc leveldb_sparql.h)
+target_link_libraries(marmotta_leveldb
+ marmotta_model marmotta_util marmotta_sparql marmotta_service
+ ${LevelDB_LIBRARY} ${GLOG_LIBRARY} ${PROTOBUF_LIBRARIES})
+
+# Server binary
add_executable(marmotta_persistence
- leveldb_persistence.cc leveldb_persistence.h leveldb_service.cc leveldb_service.h
- leveldb_server.cc leveldb_sparql.cc leveldb_sparql.h)
-target_link_libraries(marmotta_persistence
- marmotta_model marmotta_service marmotta_util marmotta_sparql
- ${LevelDB_LIBRARY} ${GFLAGS_LIBRARY} ${GLOG_LIBRARY}
- ${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES} ${GRPC_LIBRARIES} ${Tcmalloc_LIBRARIES})
+ leveldb_service.cc leveldb_service.h leveldb_server.cc )
+target_link_libraries(marmotta_persistence marmotta_service marmotta_leveldb
+ ${GFLAGS_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${GRPC_LIBRARIES} ${Tcmalloc_LIBRARIES})
install(TARGETS marmotta_persistence DESTINATION bin)
+# Command line admin tool
+add_executable(marmotta_updatedb marmotta_updatedb.cc)
+target_link_libraries(marmotta_updatedb marmotta_leveldb marmotta_parser marmotta_serializer
+ ${GFLAGS_LIBRARY} ${Boost_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${Tcmalloc_LIBRARIES})
+install(TARGETS marmotta_updatedb DESTINATION bin)
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/leveldb_persistence.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/persistence/leveldb_persistence.cc b/libraries/ostrich/backend/persistence/leveldb_persistence.cc
index 5f18eff..dd9eef9 100644
--- a/libraries/ostrich/backend/persistence/leveldb_persistence.cc
+++ b/libraries/ostrich/backend/persistence/leveldb_persistence.cc
@@ -187,7 +187,7 @@ class LevelDBIterator : public util::CloseableIterator<T> {
public:
LevelDBIterator(leveldb::Iterator *it)
- : it(it), parsed(false) {
+ : it(it) {
it->SeekToFirst();
}
@@ -195,35 +195,24 @@ class LevelDBIterator : public util::CloseableIterator<T> {
delete it;
};
- util::CloseableIterator<T> &operator++() override {
+ const T& next() override {
+ // Parse current position, then iterate to next position for next call.
+ proto.ParseFromString(it->value().ToString());
it->Next();
- parsed = false;
- return *this;
- };
-
- T &operator*() override {
- if (!parsed)
- proto.ParseFromString(it->value().ToString());
return proto;
};
- T *operator->() override {
- if (!parsed)
- proto.ParseFromString(it->value().ToString());
- return &proto;
+ const T& current() const override {
+ return proto;
};
virtual bool hasNext() override {
return it->Valid();
}
-
-
protected:
leveldb::Iterator* it;
-
T proto;
- bool parsed;
};
@@ -313,6 +302,11 @@ LevelDBPersistence::LevelDBPersistence(const std::string &path, int64_t cacheSiz
t.join();
}
+ CHECK_NOTNULL(db_spoc.get());
+ CHECK_NOTNULL(db_cspo.get());
+ CHECK_NOTNULL(db_opsc.get());
+ CHECK_NOTNULL(db_pcos.get());
+
LOG(INFO) << "LevelDB Database initialised.";
}
@@ -323,8 +317,8 @@ int64_t LevelDBPersistence::AddNamespaces(NamespaceIterator& it) {
leveldb::WriteBatch batch_prefix, batch_url;
- for (; it.hasNext(); ++it) {
- AddNamespace(*it, batch_prefix, batch_url);
+ while (it.hasNext()) {
+ AddNamespace(it.next(), batch_prefix, batch_url);
count++;
}
CHECK_STATUS(db_ns_prefix->Write(leveldb::WriteOptions(), &batch_prefix));
@@ -374,8 +368,8 @@ void LevelDBPersistence::GetNamespaces(
int64_t count = 0;
bool cbsuccess = true;
- for(auto it = GetNamespaces(pattern); cbsuccess && it->hasNext(); ++(*it)) {
- cbsuccess = callback(**it);
+ for(auto it = GetNamespaces(pattern); cbsuccess && it->hasNext();) {
+ cbsuccess = callback(it->next());
count++;
}
@@ -389,8 +383,8 @@ int64_t LevelDBPersistence::AddStatements(StatementIterator& it) {
int64_t count = 0;
leveldb::WriteBatch batch_spoc, batch_cspo, batch_opsc, batch_pcos;
- for (; it.hasNext(); ++it) {
- AddStatement(*it, batch_spoc, batch_cspo, batch_opsc, batch_pcos);
+ while (it.hasNext()) {
+ AddStatement(it.next(), batch_spoc, batch_cspo, batch_opsc, batch_pcos);
count++;
}
@@ -473,8 +467,8 @@ void LevelDBPersistence::GetStatements(
int64_t count = 0;
bool cbsuccess = true;
- for(auto it = GetStatements(pattern); cbsuccess && it->hasNext(); ++(*it)) {
- cbsuccess = callback(**it);
+ for(auto it = GetStatements(pattern); cbsuccess && it->hasNext(); ) {
+ cbsuccess = callback(it->next());
count++;
}
@@ -528,18 +522,19 @@ UpdateStatistics LevelDBPersistence::Update(LevelDBPersistence::UpdateIterator &
UpdateStatistics stats;
WriteBatch b_spoc, b_cspo, b_opsc, b_pcos, b_prefix, b_url;
- for (; it.hasNext(); ++it) {
- if (it->has_stmt_added()) {
- AddStatement(it->stmt_added(), b_spoc, b_cspo, b_opsc, b_pcos);
+ while (it.hasNext()) {
+ auto next = it.next();
+ if (next.has_stmt_added()) {
+ AddStatement(next.stmt_added(), b_spoc, b_cspo, b_opsc, b_pcos);
stats.added_stmts++;
- } else if (it->has_stmt_removed()) {
+ } else if (next.has_stmt_removed()) {
stats.removed_stmts +=
- RemoveStatements(it->stmt_removed(), b_spoc, b_cspo, b_opsc, b_pcos);
- } else if(it->has_ns_added()) {
- AddNamespace(it->ns_added(), b_prefix, b_url);
+ RemoveStatements(next.stmt_removed(), b_spoc, b_cspo, b_opsc, b_pcos);
+ } else if(next.has_ns_added()) {
+ AddNamespace(next.ns_added(), b_prefix, b_url);
stats.added_ns++;
- } else if(it->has_ns_removed()) {
- RemoveNamespace(it->ns_removed(), b_prefix, b_url);
+ } else if(next.has_ns_removed()) {
+ RemoveNamespace(next.ns_removed(), b_prefix, b_url);
}
}
std::vector<std::thread> writers;
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/leveldb_service.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/persistence/leveldb_service.cc b/libraries/ostrich/backend/persistence/leveldb_service.cc
index e31af2d..9ab9fd8 100644
--- a/libraries/ostrich/backend/persistence/leveldb_service.cc
+++ b/libraries/ostrich/backend/persistence/leveldb_service.cc
@@ -48,24 +48,21 @@ template <class Proto>
class ReaderIterator : public util::CloseableIterator<Proto> {
public:
- ReaderIterator(grpc::ServerReader<Proto>* r) : reader(r), finished(false) {
+ ReaderIterator(grpc::ServerReader<Proto>* r) : reader(r) {
// Immediately move to first element.
- operator++();
+ finished = !reader->Read(&next_);
}
- util::CloseableIterator<Proto>& operator++() override {
+ const Proto& next() override {
+ current_.Swap(&next_);
if (!finished) {
- finished = !reader->Read(&buffer);
+ finished = !reader->Read(&next_);
}
- return *this;
+ return current_;
}
- Proto& operator*() override {
- return buffer;
- }
-
- Proto* operator->() override {
- return &buffer;
+ const Proto& current() const override {
+ return current_;
}
bool hasNext() override {
@@ -74,7 +71,8 @@ class ReaderIterator : public util::CloseableIterator<Proto> {
private:
grpc::ServerReader<Proto>* reader;
- Proto buffer;
+ Proto current_;
+ Proto next_;
bool finished;
};
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/leveldb_sparql.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/persistence/leveldb_sparql.cc b/libraries/ostrich/backend/persistence/leveldb_sparql.cc
index 5d44db6..274c247 100644
--- a/libraries/ostrich/backend/persistence/leveldb_sparql.cc
+++ b/libraries/ostrich/backend/persistence/leveldb_sparql.cc
@@ -29,26 +29,13 @@ class WrapProtoStatementIterator : public StatementIterator {
WrapProtoStatementIterator(std::unique_ptr<persistence::LevelDBPersistence::StatementIterator> it)
: it(std::move(it)) { }
- util::CloseableIterator<rdf::Statement> &operator++() override {
- ++(*it);
- parsed = false;
- return *this;
+ const rdf::Statement& next() override {
+ current_ = std::move(it->next());
+ return current_;
};
- rdf::Statement &operator*() override {
- if (!parsed) {
- current = std::move(**it);
- parsed = true;
- }
- return current;
- };
-
- rdf::Statement *operator->() override {
- if (!parsed) {
- current = std::move(**it);
- parsed = true;
- }
- return ¤t;
+ const rdf::Statement& current() const override {
+ return current_;
};
bool hasNext() override {
@@ -57,7 +44,7 @@ class WrapProtoStatementIterator : public StatementIterator {
private:
std::unique_ptr<persistence::LevelDBPersistence::StatementIterator> it;
- rdf::Statement current;
+ rdf::Statement current_;
bool parsed;
};
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/marmotta_updatedb.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/persistence/marmotta_updatedb.cc b/libraries/ostrich/backend/persistence/marmotta_updatedb.cc
new file mode 100644
index 0000000..b26b019
--- /dev/null
+++ b/libraries/ostrich/backend/persistence/marmotta_updatedb.cc
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <fstream>
+
+#ifdef HAVE_IOSTREAMS
+// support b/gzipped files
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/copy.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#endif
+
+#include <google/protobuf/text_format.h>
+#include <google/protobuf/empty.pb.h>
+#include <google/protobuf/wrappers.pb.h>
+
+#include <gflags/gflags.h>
+#include <thread>
+#include <glog/logging.h>
+#include <sys/stat.h>
+
+#include "model/rdf_model.h"
+#include "parser/rdf_parser.h"
+#include "serializer/serializer.h"
+#include "persistence/leveldb_persistence.h"
+
+using namespace marmotta;
+using google::protobuf::TextFormat;
+
+#ifdef HAVE_IOSTREAMS
+using namespace boost::iostreams;
+#endif
+
+class MarmottaClient {
+ public:
+ MarmottaClient(marmotta::persistence::LevelDBPersistence* db)
+ : db(db){ }
+
+ void importDataset(std::istream& in, parser::Format format) {
+ auto start = std::chrono::steady_clock::now();
+ int64_t count = 0;
+
+ parser::Parser p("http://www.example.com", format);
+ util::ProducerConsumerIterator<rdf::proto::Statement> stmtit;
+ util::ProducerConsumerIterator<rdf::proto::Namespace> nsit;
+ p.setStatementHandler([&stmtit](const rdf::Statement& stmt) {
+ stmtit.add(stmt.getMessage());
+ });
+ p.setNamespaceHandler([&nsit](const rdf::Namespace& ns) {
+ nsit.add(ns.getMessage());
+ });
+
+ std::thread([&p, &in, &stmtit, &nsit]() {
+ p.parse(in);
+ stmtit.finish();
+ nsit.finish();
+ });
+
+ db->AddStatements(stmtit);
+ db->AddNamespaces(nsit);
+ }
+
+
+ void patternQuery(const rdf::Statement &pattern, std::ostream &out, serializer::Format format) {
+ }
+
+ void patternDelete(const rdf::Statement &pattern) {
+ db->RemoveStatements(pattern.getMessage());
+ }
+
+ void tupleQuery(const std::string& query, std::ostream &out) {
+ /*
+ ClientContext context;
+ spq::SparqlRequest request;
+ request.set_query(query);
+
+ std::unique_ptr<ClientReader<spq::SparqlResponse>> reader(
+ sparql_->TupleQuery(&context, request));
+
+ auto out_ = new google::protobuf::io::OstreamOutputStream(&out);
+ spq::SparqlResponse result;
+ while (reader->Read(&result)) {
+ TextFormat::Print(result, dynamic_cast<google::protobuf::io::ZeroCopyOutputStream*>(out_));
+ }
+ delete out_;
+ */
+ }
+
+ void listNamespaces(std::ostream &out) {
+ /*
+ ClientContext context;
+
+ google::protobuf::Empty pattern;
+
+ std::unique_ptr<ClientReader<rdf::proto::Namespace> > reader(
+ stub_->GetNamespaces(&context, pattern));
+
+ NamespaceReader it(reader.get());
+ for (; it.hasNext(); ++it) {
+ out << (*it).getPrefix() << " = " << (*it).getUri() << std::endl;
+ }
+ */
+ }
+
+ int64_t size() {
+ /*
+ ClientContext context;
+ google::protobuf::Int64Value result;
+
+ Status status = stub_->Size(&context, r, &result);
+ if (status.ok()) {
+ return result.value();
+ } else {
+ return -1;
+ }
+ */
+ }
+ private:
+ marmotta::persistence::LevelDBPersistence* db;
+};
+
+
+DEFINE_string(format, "rdfxml", "RDF format to use for parsing/serializing.");
+DEFINE_string(output, "", "File to write result to.");
+DEFINE_string(db, "/tmp/testdb", "Path to database. Will be created if non-existant.");
+DEFINE_int64(cache_size, 100 * 1048576, "Cache size used by the database (in bytes).");
+
+#ifdef HAVE_IOSTREAMS
+DEFINE_bool(gzip, false, "Input files are gzip compressed.");
+DEFINE_bool(bzip2, false, "Input files are bzip2 compressed.");
+#endif
+
+int main(int argc, char** argv) {
+ GOOGLE_PROTOBUF_VERIFY_VERSION;
+
+ // Initialize Google's logging library.
+ google::InitGoogleLogging(argv[0]);
+ google::ParseCommandLineFlags(&argc, &argv, true);
+
+ mkdir(FLAGS_db.c_str(), 0700);
+ marmotta::persistence::LevelDBPersistence persistence(FLAGS_db, FLAGS_cache_size);
+
+ MarmottaClient client(&persistence);
+
+ if ("import" == std::string(argv[1])) {
+#ifdef HAVE_IOSTREAMS
+ std::ifstream file(argv[2]);
+ filtering_streambuf<input> cin;
+ if (FLAGS_bzip2) {
+ cin.push(bzip2_decompressor());
+ }
+ if (FLAGS_gzip) {
+ cin.push(gzip_decompressor());
+ }
+ cin.push(file);
+
+ std::istream in(&cin);
+#else
+ std::ifstream in(argv[2]);
+#endif
+ std::cout << "Importing " << argv[2] << " ... " << std::endl;
+ client.importDataset(in, parser::FormatFromString(FLAGS_format));
+ std::cout << "Finished!" << std::endl;
+ }
+
+ if ("select" == std::string(argv[1])) {
+ rdf::proto::Statement query;
+ TextFormat::ParseFromString(argv[2], &query);
+ if (FLAGS_output != "") {
+ std::ofstream out(FLAGS_output);
+ client.patternQuery(rdf::Statement(query), out, serializer::FormatFromString(FLAGS_format));
+ } else {
+ client.patternQuery(rdf::Statement(query), std::cout, serializer::FormatFromString(FLAGS_format));
+ }
+ }
+
+ if ("sparql" == std::string(argv[1])) {
+ std::string query = argv[2];
+ if (FLAGS_output != "") {
+ std::ofstream out(FLAGS_output);
+ client.tupleQuery(query, out);
+ } else {
+ client.tupleQuery(query, std::cout);
+ }
+ }
+
+ if ("delete" == std::string(argv[1])) {
+ rdf::proto::Statement query;
+ TextFormat::ParseFromString(argv[2], &query);
+ client.patternDelete(rdf::Statement(query));
+ }
+
+ if ("size" == std::string(argv[1])) {
+ std::cout << "Size: " << client.size() << std::endl;
+ }
+
+
+ if ("namespaces" == std::string(argv[1])) {
+ if (FLAGS_output != "") {
+ std::ofstream out(FLAGS_output);
+ client.listNamespaces(out);
+ } else {
+ client.listNamespaces(std::cout);
+ }
+ }
+
+ google::protobuf::ShutdownProtobufLibrary();
+
+ return 0;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/serializer/serializer_base.h
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/serializer/serializer_base.h b/libraries/ostrich/backend/serializer/serializer_base.h
index 24a64f9..168da98 100644
--- a/libraries/ostrich/backend/serializer/serializer_base.h
+++ b/libraries/ostrich/backend/serializer/serializer_base.h
@@ -65,8 +65,8 @@ class SerializerBase {
void serialize(StatementIterator &it, std::ostream &out) {
prepare(out);
- for (; it.hasNext(); ++it) {
- serialize(*it);
+ while (it.hasNext()) {
+ serialize(it.next());
}
close();
};
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/sparql/rasqal_adapter.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/sparql/rasqal_adapter.cc b/libraries/ostrich/backend/sparql/rasqal_adapter.cc
index 7fc3cad..c9c89a6 100644
--- a/libraries/ostrich/backend/sparql/rasqal_adapter.cc
+++ b/libraries/ostrich/backend/sparql/rasqal_adapter.cc
@@ -65,26 +65,27 @@ rasqal_triple_parts bind_match(
struct rasqal_triples_match_s *rtm, void *user_data,
rasqal_variable *bindings[4], rasqal_triple_parts parts) {
StatementIterator *it = (StatementIterator *) rtm->user_data;
+ const rdf::Statement& s = it->next();
int r = 0;
#ifndef NDEBUG
- DLOG(INFO) << "Binding variables " << formatVariables(bindings) << " for statement " << (*it)->as_turtle();
+ DLOG(INFO) << "Binding variables " << formatVariables(bindings) << " for statement " << s.as_turtle();
#endif
if ((parts & RASQAL_TRIPLE_SUBJECT) != 0) {
- rasqal_variable_set_value(bindings[0], rasqal::AsLiteral(rtm->world, (*it)->getSubject()));
+ rasqal_variable_set_value(bindings[0], rasqal::AsLiteral(rtm->world, s.getSubject()));
r |= RASQAL_TRIPLE_SUBJECT;
}
if ((parts & RASQAL_TRIPLE_PREDICATE) != 0) {
- rasqal_variable_set_value(bindings[1], rasqal::AsLiteral(rtm->world, (*it)->getPredicate()));
+ rasqal_variable_set_value(bindings[1], rasqal::AsLiteral(rtm->world, s.getPredicate()));
r |= RASQAL_TRIPLE_PREDICATE;
}
if ((parts & RASQAL_TRIPLE_OBJECT) != 0) {
- rasqal_variable_set_value(bindings[2], rasqal::AsLiteral(rtm->world, (*it)->getObject()));
+ rasqal_variable_set_value(bindings[2], rasqal::AsLiteral(rtm->world, s.getObject()));
r |= RASQAL_TRIPLE_OBJECT;
}
if ((parts & RASQAL_TRIPLE_ORIGIN) != 0) {
- rasqal_variable_set_value(bindings[3], rasqal::AsLiteral(rtm->world, (*it)->getContext()));
+ rasqal_variable_set_value(bindings[3], rasqal::AsLiteral(rtm->world, s.getContext()));
r |= RASQAL_TRIPLE_ORIGIN;
}
@@ -94,8 +95,6 @@ rasqal_triple_parts bind_match(
// Increment the iterator contained in the triple match user data.
void next_match(struct rasqal_triples_match_s *rtm, void *user_data) {
DLOG(INFO) << "Next result";
- StatementIterator *it = (StatementIterator *) rtm->user_data;
- ++(*it);
}
// Return true in case the iterator has no next element.
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/test/CMakeLists.txt b/libraries/ostrich/backend/test/CMakeLists.txt
index 841b982..660f40f 100644
--- a/libraries/ostrich/backend/test/CMakeLists.txt
+++ b/libraries/ostrich/backend/test/CMakeLists.txt
@@ -1,6 +1,11 @@
-include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${RAPTOR_INCLUDE_DIR}/raptor2)
+enable_testing()
+include_directories(${GTEST_INCLUDE_DIRS})
+include_directories(..)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/..)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/../model)
+include_directories(${RAPTOR_INCLUDE_DIR}/raptor2)
-add_library(gtest STATIC gtest.h gtest-all.cc)
+add_library(gtest STATIC gtest/gtest.h gmock/gmock.h gmock-gtest-all.cc)
add_executable(model_tests StatementTest.cc main.cc)
target_link_libraries(model_tests gtest marmotta_model ${GLOG_LIBRARY})
@@ -8,5 +13,9 @@ target_link_libraries(model_tests gtest marmotta_model ${GLOG_LIBRARY})
add_executable(sparql_tests SparqlTest.cc main.cc)
target_link_libraries(sparql_tests gtest marmotta_model marmotta_sparql ${GLOG_LIBRARY})
-add_test(NAME ModelTest
- COMMAND model_tests)
+add_executable(persistence_tests main.cc PersistenceTest.cc)
+target_link_libraries(persistence_tests gtest marmotta_leveldb ${GLOG_LIBRARY} ${Boost_LIBRARIES})
+
+add_test(NAME ModelTest COMMAND model_tests)
+add_test(NAME SparqlTest COMMAND sparql_tests)
+add_test(NAME PersistenceTest COMMAND persistence_tests)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/PersistenceTest.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/test/PersistenceTest.cc b/libraries/ostrich/backend/test/PersistenceTest.cc
new file mode 100644
index 0000000..9878cd6
--- /dev/null
+++ b/libraries/ostrich/backend/test/PersistenceTest.cc
@@ -0,0 +1,76 @@
+//
+// Created by wastl on 19.12.15.
+//
+#include <cstdlib>
+#include <vector>
+
+#include <glog/logging.h>
+
+#include "gtest/gtest.h"
+#include "gmock/gmock.h"
+#include "boost/filesystem.hpp"
+
+#include "util/iterator.h"
+#include "model/rdf_operators.h"
+#include "persistence/leveldb_persistence.h"
+
+using namespace boost::filesystem;
+
+namespace marmotta {
+namespace rdf {
+namespace proto {
+std::ostream& operator<<(std::ostream& out, const Statement& stmt) {
+ out << rdf::Statement(stmt).as_turtle();
+ return out;
+}
+}
+}
+
+namespace persistence {
+namespace {
+
+
+class PersistenceTest : public ::testing::Test {
+ protected:
+ PersistenceTest() {
+ testdir = temp_directory_path()/unique_path();
+ create_directory(testdir);
+
+ LOG(INFO) << "Test DB Path: " << testdir.string();
+
+ db = new LevelDBPersistence(testdir.string(), 10 * 1048576);
+ }
+
+ ~PersistenceTest() {
+ LOG(INFO) << "Destroying Test DB";
+ delete db;
+ remove_all(testdir);
+ }
+
+ LevelDBPersistence* db;
+ path testdir;
+};
+
+TEST_F(PersistenceTest, TestAddStatements) {
+ std::vector<rdf::proto::Statement> stmts = {
+ rdf::Statement(rdf::URI("http://example.com/s1"), rdf::URI("http://example.com/p1"),
+ rdf::URI("http://example.com/o1")).getMessage(),
+ rdf::Statement(rdf::URI("http://example.com/s2"), rdf::URI("http://example.com/p2"),
+ rdf::URI("http://example.com/o2")).getMessage()
+ };
+
+ util::CollectionIterator<rdf::proto::Statement> it(stmts);
+ db->AddStatements(it);
+
+ EXPECT_EQ(2, db->Size());
+ for (const auto& stmt : stmts) {
+ auto it = db->GetStatements(stmt);
+ ASSERT_TRUE(it->hasNext());
+ EXPECT_EQ(stmt, it->next());
+ EXPECT_FALSE(it->hasNext());
+ }
+}
+
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/SparqlTest.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/test/SparqlTest.cc b/libraries/ostrich/backend/test/SparqlTest.cc
index 7e56eba..356e43f 100644
--- a/libraries/ostrich/backend/test/SparqlTest.cc
+++ b/libraries/ostrich/backend/test/SparqlTest.cc
@@ -2,7 +2,7 @@
// Created by wastl on 09.12.15.
//
#include <glog/logging.h>
-#include "gtest.h"
+#include "gtest/gtest.h"
#include "sparql/rasqal_adapter.h"
#include "model/rdf_operators.h"
@@ -10,33 +10,8 @@ namespace marmotta {
namespace sparql {
namespace {
-class MockStatementIterator : public StatementIterator {
- public:
- MockStatementIterator(std::vector<rdf::Statement> statements)
- : statements(statements), index(0) {
- }
-
- StatementIterator& operator++() override {
- index++;
- return *this;
- };
-
- rdf::Statement& operator*() override {
- return statements[index];
- };
-
- rdf::Statement* operator->() override {
- return &statements[index];
- };
- bool hasNext() override {
- return index < statements.size();
- };
-
- private:
- std::vector<rdf::Statement> statements;
- int index;
-};
+using MockStatementIterator = util::CollectionIterator<rdf::Statement>;
class MockTripleSource : public TripleSource {
http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/StatementTest.cc
----------------------------------------------------------------------
diff --git a/libraries/ostrich/backend/test/StatementTest.cc b/libraries/ostrich/backend/test/StatementTest.cc
index 56fb11e..458f037 100644
--- a/libraries/ostrich/backend/test/StatementTest.cc
+++ b/libraries/ostrich/backend/test/StatementTest.cc
@@ -2,7 +2,7 @@
// Created by wastl on 18.04.15.
//
-#include "gtest.h"
+#include "gtest/gtest.h"
#include "model/rdf_model.h"
#include "model/rdf_operators.h"