You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2014/10/27 06:34:17 UTC
[1/7] git commit: Added Java replicated log implementation of State.
Repository: mesos
Updated Branches:
refs/heads/master 1f66bb216 -> d418c17b8
Added Java replicated log implementation of State.
Review: https://reviews.apache.org/r/24341
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/61ce00fe
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/61ce00fe
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/61ce00fe
Branch: refs/heads/master
Commit: 61ce00fe8adda1e6883a5642bae93fdf54aa43f7
Parents: 1f66bb2
Author: Benjamin Hindman <be...@gmail.com>
Authored: Tue Aug 5 13:19:15 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Oct 26 22:27:43 2014 -0700
----------------------------------------------------------------------
src/Makefile.am | 8 ++
.../jni/org_apache_mesos_state_LogState.cpp | 114 +++++++++++++++++++
.../src/org/apache/mesos/state/LogState.java | 57 ++++++++++
3 files changed, 179 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/61ce00fe/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 2617f77..576bdab 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -798,6 +798,7 @@ MESOS_JAR_SOURCE = \
$(srcdir)/java/src/org/apache/mesos/state/AbstractState.java \
$(srcdir)/java/src/org/apache/mesos/state/InMemoryState.java \
$(srcdir)/java/src/org/apache/mesos/state/LevelDBState.java \
+ $(srcdir)/java/src/org/apache/mesos/state/LogState.java \
$(srcdir)/java/src/org/apache/mesos/state/State.java \
$(srcdir)/java/src/org/apache/mesos/state/Variable.java \
$(srcdir)/java/src/org/apache/mesos/state/ZooKeeperState.java
@@ -839,6 +840,7 @@ libjava_la_SOURCES = \
java/jni/org_apache_mesos_MesosExecutorDriver.cpp \
java/jni/org_apache_mesos_state_AbstractState.cpp \
java/jni/org_apache_mesos_state_LevelDBState.cpp \
+ java/jni/org_apache_mesos_state_LogState.cpp \
java/jni/org_apache_mesos_state_Variable.cpp \
java/jni/org_apache_mesos_state_ZooKeeperState.cpp \
jvm/jvm.cpp \
@@ -875,6 +877,7 @@ nodist_libjava_la_SOURCES = \
java/jni/org_apache_mesos_Log.h \
java/jni/org_apache_mesos_state_AbstractState.h \
java/jni/org_apache_mesos_state_LevelDBState.h \
+ java/jni/org_apache_mesos_state_LogState.h \
java/jni/org_apache_mesos_state_Variable.h \
java/jni/org_apache_mesos_state_ZooKeeperState.h
@@ -905,6 +908,11 @@ java/jni/org_apache_mesos_state_LevelDBState.h: $(MESOS_JAR)
-classpath $(MESOS_JAR):@PROTOBUF_JAR@ \
org.apache.mesos.state.LevelDBState
+java/jni/org_apache_mesos_state_LogState.h: $(MESOS_JAR)
+ $(JAVA_HOME)/bin/javah -d java/jni \
+ -classpath $(MESOS_JAR):@PROTOBUF_JAR@ \
+ org.apache.mesos.state.LogState
+
java/jni/org_apache_mesos_state_Variable.h: $(MESOS_JAR)
$(JAVA_HOME)/bin/javah -d java/jni \
-classpath $(MESOS_JAR):@PROTOBUF_JAR@ \
http://git-wip-us.apache.org/repos/asf/mesos/blob/61ce00fe/src/java/jni/org_apache_mesos_state_LogState.cpp
----------------------------------------------------------------------
diff --git a/src/java/jni/org_apache_mesos_state_LogState.cpp b/src/java/jni/org_apache_mesos_state_LogState.cpp
new file mode 100644
index 0000000..e966b2d
--- /dev/null
+++ b/src/java/jni/org_apache_mesos_state_LogState.cpp
@@ -0,0 +1,114 @@
+#include <jni.h>
+
+#include <string>
+
+#include <stout/duration.hpp>
+
+#include "log/log.hpp"
+
+#include "state/state.hpp"
+#include "state/log.hpp"
+
+#include "construct.hpp"
+#include "convert.hpp"
+
+using namespace mesos::internal::log;
+using namespace mesos::internal::state;
+
+using std::string;
+
+extern "C" {
+
+/*
+ * Class: org_apache_mesos_state_LogState
+ * Method: initialize
+ * Signature: (Ljava/lang/String;JLjava/util/concurrent/TimeUnit;Ljava/lang/String;JLjava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_apache_mesos_state_LogState_initialize
+ (JNIEnv* env,
+ jobject thiz,
+ jstring jservers,
+ jlong jtimeout,
+ jobject junit,
+ jstring jznode,
+ jlong jquorum,
+ jstring jpath)
+{
+ string servers = construct<string>(env, jservers);
+
+ jclass clazz = env->GetObjectClass(junit);
+
+ // long seconds = unit.toSeconds(time);
+ jmethodID toSeconds = env->GetMethodID(clazz, "toSeconds", "(J)J");
+
+ jlong jseconds = env->CallLongMethod(junit, toSeconds, jtimeout);
+
+ Seconds timeout(jseconds);
+
+ string znode = construct<string>(env, jznode);
+
+ long quorum = jquorum;
+
+ string path = construct<string>(env, jpath);
+
+ // Create the Log instance using ZooKeeper.
+ Log* log = new Log(quorum, path, servers, timeout, znode);
+
+ // Create the C++ Storage and State instances and initialize the
+ // __storage and __state variables.
+ Storage* storage = new LogStorage(log);
+ State* state = new State(storage);
+
+ clazz = env->GetObjectClass(thiz);
+
+ jfieldID __log = env->GetFieldID(clazz, "__log", "J");
+ env->SetLongField(thiz, __log, (jlong) log);
+
+ clazz = env->GetSuperclass(clazz);
+
+ jfieldID __storage = env->GetFieldID(clazz, "__storage", "J");
+ env->SetLongField(thiz, __storage, (jlong) storage);
+
+ jfieldID __state = env->GetFieldID(clazz, "__state", "J");
+ env->SetLongField(thiz, __state, (jlong) state);
+}
+
+
+/*
+ * Class: org_apache_mesos_state_LogState
+ * Method: finalize
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_apache_mesos_state_LogState_finalize
+ (JNIEnv* env, jobject thiz)
+{
+ // TODO(benh): Consider calling 'finalize' on the super class
+ // instead of deleting 'state' and 'storage' here manually. We'd
+ // still need to delete 'log' ourselves, of course.
+
+ jclass clazz = env->GetObjectClass(thiz);
+
+ clazz = env->GetSuperclass(clazz);
+
+ jfieldID __state = env->GetFieldID(clazz, "__state", "J");
+
+ State* state = (State*) env->GetLongField(thiz, __state);
+
+ delete state;
+
+ jfieldID __storage = env->GetFieldID(clazz, "__storage", "J");
+
+ Storage* storage = (Storage*) env->GetLongField(thiz, __storage);
+
+ delete storage;
+
+ clazz = env->GetObjectClass(thiz);
+
+ jfieldID __log = env->GetFieldID(clazz, "__log", "J");
+
+ Log* log = (Log*) env->GetLongField(thiz, __log);
+
+ delete log;
+}
+
+} // extern "C" {
http://git-wip-us.apache.org/repos/asf/mesos/blob/61ce00fe/src/java/src/org/apache/mesos/state/LogState.java
----------------------------------------------------------------------
diff --git a/src/java/src/org/apache/mesos/state/LogState.java b/src/java/src/org/apache/mesos/state/LogState.java
new file mode 100644
index 0000000..2516da9
--- /dev/null
+++ b/src/java/src/org/apache/mesos/state/LogState.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mesos.state;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Implementation of State that uses a replicated log to store
+ * variables/values.
+ */
+public class LogState extends AbstractState {
+ /**
+ * Constructs a new instance of LogState.
+ *
+ * @param servers List of ZooKeeper servers, e.g., 'ip1:port1,ip2:port2'.
+ * @param timeout ZooKeeper session timeout.
+ * @param unit Unit for session timeout.
+ * @param znode Path to znode where log replicas should be found.
+ * @param quorum Number of replicas necessary to persist a write.
+ * @param path Path the local replica uses to read/write data.
+ */
+ public LogState(String servers,
+ long timeout,
+ TimeUnit unit,
+ String znode,
+ long quorum,
+ String path) {
+ initialize(servers, timeout, unit, znode, quorum, path);
+ }
+
+ protected native void initialize(String servers,
+ long timeout,
+ TimeUnit unit,
+ String znode,
+ long quorum,
+ String path);
+
+ protected native void finalize();
+
+ private long __log;
+}
[5/7] git commit: Added DIFF to the replicated log state storage
implementation.
Posted by be...@apache.org.
Added DIFF to the replicated log state storage implementation.
Review: https://reviews.apache.org/r/24536
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/33e625f0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/33e625f0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/33e625f0
Branch: refs/heads/master
Commit: 33e625f0fa0a12bb87dfa82a8e4d2200fcbd7062
Parents: 3b3d60f
Author: Benjamin Hindman <be...@gmail.com>
Authored: Sun Aug 10 17:47:06 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Oct 26 22:33:34 2014 -0700
----------------------------------------------------------------------
src/Makefile.am | 8 +-
.../jni/org_apache_mesos_state_LogState.cpp | 9 +-
.../src/org/apache/mesos/state/LogState.java | 26 ++-
src/messages/state.proto | 9 ++
src/state/log.cpp | 162 +++++++++++++++++--
src/state/log.hpp | 2 +-
src/tests/state_tests.cpp | 106 ++++++++++--
7 files changed, 284 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/33e625f0/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 576bdab..00d5d3c 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -89,6 +89,8 @@ MESOS_CPPFLAGS += -I$(top_srcdir)/include
MESOS_CPPFLAGS += -I$(top_srcdir)/$(LIBPROCESS)/include
MESOS_CPPFLAGS += -I$(top_srcdir)/$(STOUT)/include
MESOS_CPPFLAGS += -I../include
+MESOS_CPPFLAGS += -I/usr/include/subversion-1
+MESOS_CPPFLAGS += -I/usr/include/apr-1
# Protobuf headers that depend on mesos.pb.h need this.
MESOS_CPPFLAGS += -I../include/mesos
@@ -581,7 +583,11 @@ libmesos_la_LDFLAGS = -release $(PACKAGE_VERSION)
libmesos_la_LIBTOOLFLAGS = --tag=CXX
# Add the convenience library.
-libmesos_la_LIBADD = libmesos_no_3rdparty.la
+libmesos_la_LIBADD = \
+ libmesos_no_3rdparty.la \
+ -lsvn_subr-1 \
+ -lsvn_delta-1 \
+ -lapr-1
libmesos_la_LIBADD += ../$(LIBPROCESS)/libprocess.la
http://git-wip-us.apache.org/repos/asf/mesos/blob/33e625f0/src/java/jni/org_apache_mesos_state_LogState.cpp
----------------------------------------------------------------------
diff --git a/src/java/jni/org_apache_mesos_state_LogState.cpp b/src/java/jni/org_apache_mesos_state_LogState.cpp
index e966b2d..6382b9c 100644
--- a/src/java/jni/org_apache_mesos_state_LogState.cpp
+++ b/src/java/jni/org_apache_mesos_state_LogState.cpp
@@ -22,7 +22,7 @@ extern "C" {
/*
* Class: org_apache_mesos_state_LogState
* Method: initialize
- * Signature: (Ljava/lang/String;JLjava/util/concurrent/TimeUnit;Ljava/lang/String;JLjava/lang/String;)V
+ * Signature: (Ljava/lang/String;JLjava/util/concurrent/TimeUnit;Ljava/lang/String;JLjava/lang/String;I)V
*/
JNIEXPORT void JNICALL Java_org_apache_mesos_state_LogState_initialize
(JNIEnv* env,
@@ -32,7 +32,8 @@ JNIEXPORT void JNICALL Java_org_apache_mesos_state_LogState_initialize
jobject junit,
jstring jznode,
jlong jquorum,
- jstring jpath)
+ jstring jpath,
+ jint jdiffsBetweenSnapshots)
{
string servers = construct<string>(env, jservers);
@@ -56,7 +57,9 @@ JNIEXPORT void JNICALL Java_org_apache_mesos_state_LogState_initialize
// Create the C++ Storage and State instances and initialize the
// __storage and __state variables.
- Storage* storage = new LogStorage(log);
+ int diffsBetweenSnapshots = jdiffsBetweenSnapshots;
+
+ Storage* storage = new LogStorage(log, diffsBetweenSnapshots);
State* state = new State(storage);
clazz = env->GetObjectClass(thiz);
http://git-wip-us.apache.org/repos/asf/mesos/blob/33e625f0/src/java/src/org/apache/mesos/state/LogState.java
----------------------------------------------------------------------
diff --git a/src/java/src/org/apache/mesos/state/LogState.java b/src/java/src/org/apache/mesos/state/LogState.java
index 2516da9..ed418b6 100644
--- a/src/java/src/org/apache/mesos/state/LogState.java
+++ b/src/java/src/org/apache/mesos/state/LogState.java
@@ -41,7 +41,28 @@ public class LogState extends AbstractState {
String znode,
long quorum,
String path) {
- initialize(servers, timeout, unit, znode, quorum, path);
+ initialize(servers, timeout, unit, znode, quorum, path, 0);
+ }
+
+ /**
+ * Constructs a new instance of LogState.
+ *
+ * @param servers List of ZooKeeper servers, e.g., 'ip1:port1,ip2:port2'.
+ * @param timeout ZooKeeper session timeout.
+ * @param unit Unit for session timeout.
+ * @param znode Path to znode where log replicas should be found.
+ * @param quorum Number of replicas necessary to persist a write.
+ * @param path Path the local replica uses to read/write data.
+ * @param diffsBetweenSnapshots Number of diffs to write between snapshots.
+ */
+ public LogState(String servers,
+ long timeout,
+ TimeUnit unit,
+ String znode,
+ long quorum,
+ String path,
+ int diffsBetweenSnapshots) {
+ initialize(servers, timeout, unit, znode, quorum, path, diffsBetweenSnapshots);
}
protected native void initialize(String servers,
@@ -49,7 +70,8 @@ public class LogState extends AbstractState {
TimeUnit unit,
String znode,
long quorum,
- String path);
+ String path,
+ int diffsBetweenSnapshots);
protected native void finalize();
http://git-wip-us.apache.org/repos/asf/mesos/blob/33e625f0/src/messages/state.proto
----------------------------------------------------------------------
diff --git a/src/messages/state.proto b/src/messages/state.proto
index 59276e5..15071c2 100644
--- a/src/messages/state.proto
+++ b/src/messages/state.proto
@@ -30,6 +30,7 @@ message Entry {
message Operation {
enum Type {
SNAPSHOT = 1;
+ DIFF = 3;
EXPUNGE = 2;
}
@@ -38,6 +39,13 @@ message Operation {
required Entry entry = 1;
}
+ // Describes a "diff" operation where the 'value' of the entry is
+ // just the diff itself, but the 'uuid' represents the UUID of the
+ // entry after applying this diff.
+ message Diff {
+ required Entry entry = 1;
+ }
+
// Describes an "expunge" operation.
message Expunge {
required string name = 1;
@@ -45,5 +53,6 @@ message Operation {
required Type type = 1;
optional Snapshot snapshot = 2;
+ optional Diff diff = 4;
optional Expunge expunge = 3;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/33e625f0/src/state/log.cpp
----------------------------------------------------------------------
diff --git a/src/state/log.cpp b/src/state/log.cpp
index fd8b28a..95326a6 100644
--- a/src/state/log.cpp
+++ b/src/state/log.cpp
@@ -12,11 +12,17 @@
#include <process/mutex.hpp>
#include <process/process.hpp>
+#include <process/metrics/metrics.hpp>
+#include <process/metrics/timer.hpp>
+
+#include <stout/bytes.hpp>
+#include <stout/duration.hpp>
#include <stout/foreach.hpp>
#include <stout/lambda.hpp>
#include <stout/hashmap.hpp>
#include <stout/nothing.hpp>
#include <stout/option.hpp>
+#include <stout/svn.hpp>
#include <stout/uuid.hpp>
#include "log/log.hpp"
@@ -54,7 +60,7 @@ namespace state {
class LogStorageProcess : public Process<LogStorageProcess>
{
public:
- LogStorageProcess(Log* log);
+ LogStorageProcess(Log* log, size_t diffsBetweenSnapshots);
virtual ~LogStorageProcess();
@@ -91,6 +97,7 @@ private:
Future<bool> __set(const state::Entry& entry, const UUID& uuid);
Future<bool> ___set(
const state::Entry& entry,
+ size_t diff,
const Option<Log::Position>& position);
Future<bool> _expunge(const state::Entry& entry);
@@ -104,6 +111,8 @@ private:
Log::Reader reader;
Log::Writer writer;
+ const size_t diffsBetweenSnapshots;
+
// Used to serialize Log::Writer::append/truncate operations.
Mutex mutex;
@@ -122,27 +131,78 @@ private:
// actual appending of the data.
struct Snapshot
{
- Snapshot(const Log::Position& position, const state::Entry& entry)
- : position(position), entry(entry) {}
+ Snapshot(const Log::Position& position,
+ const state::Entry& entry,
+ size_t diffs = 0)
+ : position(position),
+ entry(entry),
+ diffs(diffs) {}
+
+ Try<Snapshot> patch(
+ const Log::Position& position,
+ const Operation::Diff& diff) const
+ {
+ if (diff.entry().name() != entry.name()) {
+ return Error("Attempted to patch the wrong snapshot");
+ }
+
+ Try<string> patch = svn::patch(
+ entry.value(),
+ svn::Diff(diff.entry().value()));
+
+ if (patch.isError()) {
+ return Error(patch.error());
+ }
+ Entry entry(diff.entry());
+ entry.set_value(patch.get());
+
+ return Snapshot(position, entry, diffs + 1);
+ }
+
+ // Position in the log where this snapshot is located. NOTE: if
+ // 'diffs' is greater than 0 this still represents the location of
+ // the snapshot, not the last DIFF record in the log.
const Log::Position position;
// TODO(benh): Rather than storing the entire state::Entry we
// should just store the position, name, and UUID and cache the
// data so we don't use too much memory.
const state::Entry entry;
+
+ // This value represents the number of Operation::DIFFs in the
+ // underlying log that make up this "snapshot". If this snapshot
+ // is actually represented in the log this value is 0.
+ const size_t diffs;
};
// All known snapshots indexed by name. Note that 'hashmap::get'
// must be used instead of 'operator []' since Snapshot doesn't have
// a default/empty constructor.
hashmap<string, Snapshot> snapshots;
+
+ struct Metrics
+ {
+ Metrics()
+ : diff("log_storage/diff")
+ {
+ process::metrics::add(diff);
+ }
+
+ ~Metrics()
+ {
+ process::metrics::remove(diff);
+ }
+
+ process::metrics::Timer<Milliseconds> diff;
+ } metrics;
};
-LogStorageProcess::LogStorageProcess(Log* log)
+LogStorageProcess::LogStorageProcess(Log* log, size_t diffsBetweenSnapshots)
: reader(log),
- writer(log) {}
+ writer(log),
+ diffsBetweenSnapshots(diffsBetweenSnapshots) {}
LogStorageProcess::~LogStorageProcess() {}
@@ -236,12 +296,31 @@ Future<Nothing> LogStorageProcess::apply(const list<Log::Entry>& entries)
case Operation::SNAPSHOT: {
CHECK(operation.has_snapshot());
- // Add or update the snapshot.
+ // Add or update (override) the snapshot.
Snapshot snapshot(entry.position, operation.snapshot().entry());
snapshots.put(snapshot.entry.name(), snapshot);
break;
}
+ case Operation::DIFF: {
+ CHECK(operation.has_diff());
+
+ Option<Snapshot> snapshot = snapshots.get(operation.diff().entry().name());
+
+ CHECK_SOME(snapshot);
+
+ Try<Snapshot> patched =
+ snapshot.get().patch(entry.position, operation.diff());
+
+ if (patched.isError()) {
+ return Failure("Failed to apply the diff: " + patched.error());
+ }
+
+ // Replace the snapshot with the patched snapshot.
+ snapshots.put(patched.get().entry.name(), patched.get());
+ break;
+ }
+
case Operation::EXPUNGE: {
CHECK(operation.has_expunge());
snapshots.erase(operation.expunge().name());
@@ -288,6 +367,13 @@ Future<Nothing> LogStorageProcess::_truncate()
minimum = min(minimum, snapshot.position);
}
+ // TODO(benh): It's possible that the minimum position we've found
+ // will leave a lot of "unnecessary" entries in the log (e.g., a
+ // snapshot that has been overwritten at a later position). In this
+ // circumstance we should "compact/defrag" the log by writing/moving
+ // snapshots to the end of the log first applying any diffs as
+ // necessary.
+
CHECK_SOME(truncated);
if (minimum.isSome() && minimum.get() > truncated.get()) {
@@ -363,32 +449,76 @@ Future<bool> LogStorageProcess::__set(
const state::Entry& entry,
const UUID& uuid)
{
- // Check the version first (if we've already got a snapshot).
Option<Snapshot> snapshot = snapshots.get(entry.name());
- if (snapshot.isSome()) {
- if (UUID::fromBytes(snapshot.get().entry.uuid()) != uuid) {
- return false;
+ // Check the version first (if we've already got a snapshot).
+ if (snapshot.isSome() &&
+ UUID::fromBytes(snapshot.get().entry.uuid()) != uuid) {
+ return false;
+ }
+
+ // Check if we should try to compute a diff.
+ if (snapshot.isSome() && snapshot.get().diffs < diffsBetweenSnapshots) {
+ // Keep metrics for the time to calculate diffs.
+ metrics.diff.start();
+
+ // Construct the diff of the last snapshot.
+ Try<svn::Diff> diff = svn::diff(
+ snapshot.get().entry.value(),
+ entry.value());
+
+ Duration elapsed = metrics.diff.stop();
+
+ if (diff.isError()) {
+ // TODO(benh): Fallback and try and write a whole snapshot?
+ return Failure("Failed to construct diff: " + diff.error());
+ }
+
+ VLOG(1) << "Created an SVN diff in " << elapsed
+ << " of size " << Bytes(diff.get().data.size()) << " which is "
+ << (diff.get().data.size() / (double) entry.value().size()) * 100.0
+ << "% the original size (" << Bytes(entry.value().size()) << ")";
+
+ // Only write the diff if it provides a reduction in size.
+ if (diff.get().data.size() < entry.value().size()) {
+ // Append a diff operation.
+ Operation operation;
+ operation.set_type(Operation::DIFF);
+ operation.mutable_diff()->mutable_entry()->CopyFrom(entry);
+ operation.mutable_diff()->mutable_entry()->set_value(diff.get().data);
+
+ string value;
+ if (!operation.SerializeToString(&value)) {
+ return Failure("Failed to serialize DIFF Operation");
+ }
+
+ return writer.append(value)
+ .then(defer(self(),
+ &Self::___set,
+ entry,
+ snapshot.get().diffs + 1,
+ lambda::_1));
}
}
- // Now serialize and append a snapshot operation.
+ // Write the full snapshot.
Operation operation;
operation.set_type(Operation::SNAPSHOT);
operation.mutable_snapshot()->mutable_entry()->CopyFrom(entry);
string value;
if (!operation.SerializeToString(&value)) {
- return Failure("Failed to serialize Operation");
+ return Failure("Failed to serialize SNAPSHOT Operation");
}
return writer.append(value)
- .then(defer(self(), &Self::___set, entry, lambda::_1));
+ .then(defer(self(), &Self::___set, entry, 0, lambda::_1));
}
Future<bool> LogStorageProcess::___set(
const state::Entry& entry,
+ size_t diffs,
const Option<Log::Position>& position)
{
if (position.isNone()) {
@@ -401,7 +531,7 @@ Future<bool> LogStorageProcess::___set(
CHECK(!snapshots.contains(entry.name()) ||
snapshots.get(entry.name()).get().position < position.get());
- Snapshot snapshot(position.get(), entry);
+ Snapshot snapshot(position.get(), entry, diffs);
snapshots.put(snapshot.entry.name(), snapshot);
truncate();
@@ -488,9 +618,9 @@ Future<std::set<string> > LogStorageProcess::_names()
}
-LogStorage::LogStorage(Log* log)
+LogStorage::LogStorage(Log* log, size_t diffsBetweenSnapshots)
{
- process = new LogStorageProcess(log);
+ process = new LogStorageProcess(log, diffsBetweenSnapshots);
spawn(process);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/33e625f0/src/state/log.hpp
----------------------------------------------------------------------
diff --git a/src/state/log.hpp b/src/state/log.hpp
index 6bd054f..a0ca4f8 100644
--- a/src/state/log.hpp
+++ b/src/state/log.hpp
@@ -26,7 +26,7 @@ class LogStorageProcess;
class LogStorage : public Storage
{
public:
- LogStorage(log::Log* log);
+ LogStorage(log::Log* log, size_t diffsBetweenSnapshots = 0);
virtual ~LogStorage();
http://git-wip-us.apache.org/repos/asf/mesos/blob/33e625f0/src/tests/state_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/state_tests.cpp b/src/tests/state_tests.cpp
index 0948b9f..f37d606 100644
--- a/src/tests/state_tests.cpp
+++ b/src/tests/state_tests.cpp
@@ -18,8 +18,10 @@
#include <gmock/gmock.h>
+#include <list>
#include <set>
#include <string>
+#include <vector>
#include <mesos/mesos.hpp>
@@ -33,14 +35,19 @@
#include <stout/os.hpp>
#include <stout/try.hpp>
+#include <stout/protobuf.hpp>
+
#include "common/type_utils.hpp"
#include "log/log.hpp"
#include "log/replica.hpp"
+
#include "log/tool/initialize.hpp"
#include "master/registry.hpp"
+#include "messages/state.hpp"
+
#include "state/in_memory.hpp"
#include "state/leveldb.hpp"
#include "state/log.hpp"
@@ -60,6 +67,7 @@ using namespace mesos::internal::log;
using namespace process;
using state::LevelDBStorage;
+using state::Operation;
using state::Storage;
#ifdef MESOS_HAS_JAVA
using state::ZooKeeperStorage;
@@ -68,8 +76,10 @@ using state::ZooKeeperStorage;
using state::protobuf::State;
using state::protobuf::Variable;
+using std::list;
using std::set;
using std::string;
+using std::vector;
using mesos::internal::tests::TemporaryDirectoryTest;
@@ -84,7 +94,7 @@ void FetchAndStoreAndFetch(State* state)
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
@@ -101,7 +111,7 @@ void FetchAndStoreAndFetch(State* state)
variable = future1.get();
Slaves slaves2 = variable.get();
- ASSERT_TRUE(slaves2.slaves().size() == 1);
+ ASSERT_EQ(1, slaves2.slaves().size());
EXPECT_EQ("localhost", slaves2.slaves(0).info().hostname());
}
@@ -114,7 +124,7 @@ void FetchAndStoreAndStoreAndFetch(State* state)
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
@@ -137,7 +147,7 @@ void FetchAndStoreAndStoreAndFetch(State* state)
variable = future1.get();
Slaves slaves2 = variable.get();
- ASSERT_TRUE(slaves2.slaves().size() == 1);
+ ASSERT_EQ(1, slaves2.slaves().size());
EXPECT_EQ("localhost", slaves2.slaves(0).info().hostname());
}
@@ -150,7 +160,7 @@ void FetchAndStoreAndStoreFailAndFetch(State* state)
Variable<Slaves> variable1 = future1.get();
Slaves slaves1 = variable1.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave1 = slaves1.add_slaves();
slave1->mutable_info()->set_hostname("localhost1");
@@ -162,7 +172,7 @@ void FetchAndStoreAndStoreFailAndFetch(State* state)
ASSERT_SOME(future2.get());
Slaves slaves2 = variable1.get();
- EXPECT_TRUE(slaves2.slaves().size() == 0);
+ ASSERT_EQ(0, slaves2.slaves().size());
Slave* slave2 = slaves2.add_slaves();
slave2->mutable_info()->set_hostname("localhost2");
@@ -179,7 +189,7 @@ void FetchAndStoreAndStoreFailAndFetch(State* state)
variable1 = future1.get();
slaves1 = variable1.get();
- ASSERT_TRUE(slaves1.slaves().size() == 1);
+ ASSERT_EQ(1, slaves1.slaves().size());
EXPECT_EQ("localhost1", slaves1.slaves(0).info().hostname());
}
@@ -192,7 +202,7 @@ void FetchAndStoreAndExpungeAndFetch(State* state)
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
@@ -227,7 +237,7 @@ void FetchAndStoreAndExpungeAndExpunge(State* state)
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
@@ -258,7 +268,7 @@ void FetchAndStoreAndExpungeAndStoreAndFetch(State* state)
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
@@ -285,7 +295,7 @@ void FetchAndStoreAndExpungeAndStoreAndFetch(State* state)
variable = future1.get();
Slaves slaves2 = variable.get();
- ASSERT_TRUE(slaves2.slaves().size() == 1);
+ ASSERT_EQ(1, slaves2.slaves().size());
EXPECT_EQ("localhost", slaves2.slaves(0).info().hostname());
}
@@ -298,7 +308,7 @@ void Names(State* state)
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
@@ -311,7 +321,7 @@ void Names(State* state)
Future<set<string> > names = state->names();
AWAIT_READY(names);
- ASSERT_TRUE(names.get().size() == 1);
+ ASSERT_EQ(1u, names.get().size());
EXPECT_NE(names.get().find("slaves"), names.get().end());
}
@@ -490,7 +500,7 @@ protected:
pids.insert(replica2->pid());
log = new Log(2, path1, pids);
- storage = new state::LogStorage(log);
+ storage = new state::LogStorage(log, 1024);
state = new State(storage);
}
@@ -573,7 +583,7 @@ TEST_F(LogStateTest, Timeout)
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
- EXPECT_TRUE(slaves1.slaves().size() == 0);
+ ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
@@ -601,6 +611,72 @@ TEST_F(LogStateTest, Timeout)
}
+TEST_F(LogStateTest, Diff)
+{
+ Future<Variable<Slaves>> future1 = state->fetch<Slaves>("slaves");
+ AWAIT_READY(future1);
+
+ Variable<Slaves> variable = future1.get();
+
+ Slaves slaves = variable.get();
+ ASSERT_EQ(0, slaves.slaves().size());
+
+ for (size_t i = 0; i < 1024; i++) {
+ Slave* slave = slaves.add_slaves();
+ slave->mutable_info()->set_hostname("localhost" + stringify(i));
+ }
+
+ variable = variable.mutate(slaves);
+
+ Future<Option<Variable<Slaves>>> future2 = state->store(variable);
+ AWAIT_READY(future2);
+ ASSERT_SOME(future2.get());
+
+ variable = future2.get().get();
+
+ Slave* slave = slaves.add_slaves();
+ slave->mutable_info()->set_hostname("localhost1024");
+
+ variable = variable.mutate(slaves);
+
+ future2 = state->store(variable);
+ AWAIT_READY(future2);
+ ASSERT_SOME(future2.get());
+
+ Log::Reader reader(log);
+
+ Future<Log::Position> beginning = reader.beginning();
+ Future<Log::Position> ending = reader.ending();
+
+ AWAIT_READY(beginning);
+ AWAIT_READY(ending);
+
+ Future<list<Log::Entry>> entries = reader.read(beginning.get(), ending.get());
+
+ AWAIT_READY(entries);
+
+ // Convert each Log::Entry to a Operation.
+ vector<Operation> operations;
+
+ foreach (const Log::Entry& entry, entries.get()) {
+ // Parse the Operation from the Log::Entry.
+ Operation operation;
+
+ google::protobuf::io::ArrayInputStream stream(
+ entry.data.data(),
+ entry.data.size());
+
+ ASSERT_TRUE(operation.ParseFromZeroCopyStream(&stream));
+
+ operations.push_back(operation);
+ }
+
+ ASSERT_EQ(2u, operations.size());
+ EXPECT_EQ(Operation::SNAPSHOT, operations[0].type());
+ EXPECT_EQ(Operation::DIFF, operations[1].type());
+}
+
+
#ifdef MESOS_HAS_JAVA
class ZooKeeperStateTest : public tests::ZooKeeperTest
{
[2/7] git commit: Added functionality to create SVN based diffs of
arbitrary strings.
Posted by be...@apache.org.
Added functionality to create SVN based diffs of arbitrary strings.
Review: https://reviews.apache.org/r/24535
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3b3d60f0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3b3d60f0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3b3d60f0
Branch: refs/heads/master
Commit: 3b3d60f0dc605ff49a3eb2f0e546e4bbb2333b23
Parents: 61ce00f
Author: Benjamin Hindman <be...@gmail.com>
Authored: Sun Aug 10 11:15:56 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Oct 26 22:33:32 2014 -0700
----------------------------------------------------------------------
3rdparty/libprocess/3rdparty/Makefile.am | 8 +-
3rdparty/libprocess/3rdparty/stout/Makefile.am | 1 +
.../3rdparty/stout/include/Makefile.am | 1 +
.../3rdparty/stout/include/stout/svn.hpp | 192 +++++++++++++++++++
.../3rdparty/stout/tests/svn_tests.cpp | 74 +++++++
5 files changed, 275 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b3d60f0/3rdparty/libprocess/3rdparty/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/Makefile.am b/3rdparty/libprocess/3rdparty/Makefile.am
index 1e24886..4c6b2f1 100644
--- a/3rdparty/libprocess/3rdparty/Makefile.am
+++ b/3rdparty/libprocess/3rdparty/Makefile.am
@@ -146,6 +146,7 @@ stout_tests_SOURCES = \
$(STOUT)/tests/base64_tests.cpp \
$(STOUT)/tests/bytes_tests.cpp \
$(STOUT)/tests/cache_tests.cpp \
+ $(STOUT)/tests/svn_tests.cpp \
$(STOUT)/tests/duration_tests.cpp \
$(STOUT)/tests/dynamiclibrary_tests.cpp \
$(STOUT)/tests/error_tests.cpp \
@@ -185,6 +186,8 @@ endif
stout_tests_CPPFLAGS = \
-I$(srcdir)/$(STOUT)/include \
-I$(PROTOBUF)/src \
+ -I/usr/include/subversion-1 \
+ -I/usr/include/apr-1 \
$(AM_CPPFLAGS)
if WITH_BUNDLED_GMOCK
@@ -211,7 +214,10 @@ stout_tests_LDADD = \
libgmock.la \
$(LIBGLOG) \
$(LIBPROTOBUF) \
- -ldl
+ -ldl \
+ -lsvn_subr-1 \
+ -lsvn_delta-1 \
+ -lapr-1
# We use a check-local target for now to avoid the parallel test
# runner that ships with newer versions of autotools.
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b3d60f0/3rdparty/libprocess/3rdparty/stout/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/Makefile.am b/3rdparty/libprocess/3rdparty/stout/Makefile.am
index 4136062..e0a7838 100644
--- a/3rdparty/libprocess/3rdparty/stout/Makefile.am
+++ b/3rdparty/libprocess/3rdparty/stout/Makefile.am
@@ -43,6 +43,7 @@ EXTRA_DIST = \
tests/some_tests.cpp \
tests/strings_tests.cpp \
tests/subcommand_tests.cpp \
+ tests/svn_tests.cpp \
tests/thread_tests.cpp \
tests/uuid_tests.cpp \
tests/version_tests.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b3d60f0/3rdparty/libprocess/3rdparty/stout/include/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/include/Makefile.am b/3rdparty/libprocess/3rdparty/stout/include/Makefile.am
index d529013..3048e84 100644
--- a/3rdparty/libprocess/3rdparty/stout/include/Makefile.am
+++ b/3rdparty/libprocess/3rdparty/stout/include/Makefile.am
@@ -67,6 +67,7 @@ nobase_include_HEADERS = \
stout/stringify.hpp \
stout/strings.hpp \
stout/subcommand.hpp \
+ stout/svn.hpp \
stout/tests/utils.hpp \
stout/thread.hpp \
stout/try.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b3d60f0/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp b/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp
new file mode 100644
index 0000000..db0e7e2
--- /dev/null
+++ b/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp
@@ -0,0 +1,192 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __STOUT_SVN_HPP__
+#define __STOUT_SVN_HPP__
+
+#include <apr_pools.h>
+
+#include <stdlib.h>
+
+#include <svn_delta.h>
+#include <svn_error.h>
+#include <svn_pools.h>
+
+#include <string>
+
+#include <stout/try.hpp>
+
+namespace svn {
+
+struct Diff
+{
+ Diff(const std::string& data) : data(data) {}
+
+ std::string data;
+};
+
+
+// Note, this function is exposed publicly in the event that someone
+// wants to coordinate the intialization of APR done automatically by
+// calls to svn::diff, svn::patch, etc. That is, if you're using the
+// svn::* functions in your code and you're also using APR you can
+// initialize the APR before you start doing things with threads that
+// might call svn::* functions.
+inline void initialize()
+{
+ // We use a static variable to initialize the Apache Portable
+ // Runtime (APR) library in a thread-safe way (at least with respect
+ // to calls within svn::* since there is no way to guarantee that
+ // another library isn't concurrently initializing the APR). Thread
+ // safety is provided by the fact that the static variable 'apr'
+ // should get constructed (i.e., the APR constructor invoked) and
+ // destructed in a thread safe way (as of GCC 4.3 and required for
+ // C++11).
+ struct APR
+ {
+ APR()
+ {
+ apr_initialize();
+ }
+
+ ~APR()
+ {
+ apr_terminate();
+ }
+ };
+
+ static APR apr;
+}
+
+
+inline Try<Diff> diff(const std::string& from, const std::string& to)
+{
+ // Initialize the Apache Portable Runtime subsystem, as necessary
+ // for using the svn library.
+ initialize();
+
+ // Note that svn_pool_create wraps apr_pool_create_ex, which is
+ // thread safe, see: http://goo.gl/NX0hps.
+ apr_pool_t* pool = svn_pool_create(NULL);
+
+ // First we need to produce a text delta stream by diffing 'source'
+ // against 'target'.
+ svn_string_t source;
+ source.data = from.data();
+ source.len = from.length();
+
+ svn_string_t target;
+ target.data = to.data();
+ target.len = to.length();
+
+ svn_txdelta_stream_t* delta;
+ svn_txdelta(
+ &delta,
+ svn_stream_from_string(&source, pool),
+ svn_stream_from_string(&target, pool),
+ pool);
+
+ // Now we want to convert this text delta stream into an svndiff
+ // format based diff. Setup the handler that will consume the text
+ // delta and produce the svndiff.
+ svn_txdelta_window_handler_t handler;
+ void* baton = NULL;
+ svn_stringbuf_t* diff = svn_stringbuf_create_ensure(1024, pool);
+
+ svn_txdelta_to_svndiff2(
+ &handler,
+ &baton,
+ svn_stream_from_stringbuf(diff, pool),
+ 0,
+ pool);
+
+ // Now feed the text delta to the handler.
+ svn_error_t* error = svn_txdelta_send_txstream(delta, handler, baton, pool);
+
+ if (error != NULL) {
+ char buffer[1024];
+ std::string message(svn_err_best_message(error, buffer, 1024));
+ svn_pool_destroy(pool);
+ return Error(message);
+ }
+
+ Diff d(std::string(diff->data, diff->len));
+
+ svn_pool_destroy(pool);
+
+ return d;
+}
+
+
+inline Try<std::string> patch(const std::string& s, const Diff& diff)
+{
+ // Initialize the Apache Portable Runtime subsystem, as necessary
+ // for using the svn library.
+ initialize();
+
+ // Note that svn_pool_create wraps apr_pool_create_ex, which is
+ // thread safe, see: http://goo.gl/NX0hps.
+ apr_pool_t* pool = svn_pool_create(NULL);
+
+ // We want to apply the svndiff format diff to the source trying to
+ // produce a result. First setup a handler for applying a text delta
+ // to the source stream.
+ svn_string_t source;
+ source.data = s.data();
+ source.len = s.length();
+
+ svn_txdelta_window_handler_t handler;
+ void* baton = NULL;
+
+ svn_stringbuf_t* patched = svn_stringbuf_create_ensure(s.length(), pool);
+
+ svn_txdelta_apply(
+ svn_stream_from_string(&source, pool),
+ svn_stream_from_stringbuf(patched, pool),
+ NULL,
+ NULL,
+ pool,
+ &handler,
+ &baton);
+
+ // Setup a stream that converts an svndiff format diff to a text
+ // delta, so that we can use our handler to patch the source string.
+ svn_stream_t* stream = svn_txdelta_parse_svndiff(
+ handler,
+ baton,
+ TRUE,
+ pool);
+
+ // Now feed the diff into the stream to compute the patched result.
+ const char* data = diff.data.data();
+ apr_size_t length = diff.data.length();
+
+ svn_error_t* error = svn_stream_write(stream, data, &length);
+
+ if (error != NULL) {
+ char buffer[1024];
+ std::string message(svn_err_best_message(error, buffer, 1024));
+ svn_pool_destroy(pool);
+ return Error(message);
+ }
+
+ std::string result(patched->data, patched->len);
+
+ svn_pool_destroy(pool);
+
+ return result;
+}
+
+} // namespace svn {
+
+#endif // __STOUT_SVN_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b3d60f0/3rdparty/libprocess/3rdparty/stout/tests/svn_tests.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/tests/svn_tests.cpp b/3rdparty/libprocess/3rdparty/stout/tests/svn_tests.cpp
new file mode 100644
index 0000000..d5e0f6e
--- /dev/null
+++ b/3rdparty/libprocess/3rdparty/stout/tests/svn_tests.cpp
@@ -0,0 +1,74 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <cstdlib> // For rand().
+#include <string>
+
+#include <stout/bytes.hpp>
+#include <stout/gtest.hpp>
+#include <stout/svn.hpp>
+
+using std::string;
+
+
+TEST(SVN, DiffPatch)
+{
+ string source;
+
+ while (Bytes(source.size()) < Megabytes(1)) {
+ source += (char) rand() % 256;
+ }
+
+ // Make the target string have 512 different bytes in the middle.
+ string target = source;
+
+ for (size_t index = 0; index < 512; index++) {
+ target[1024 + index] = (char) rand() % 256;
+ }
+
+ ASSERT_NE(source, target);
+
+ Try<svn::Diff> diff = svn::diff(source, target);
+
+ ASSERT_SOME(diff);
+
+ Try<string> result = svn::patch(source, diff.get());
+
+ ASSERT_SOME_EQ(target, result);
+ ASSERT_SOME_NE(source, result);
+}
+
+
+TEST(SVN, EmptyDiffPatch)
+{
+ string source;
+
+ while (Bytes(source.size()) < Megabytes(1)) {
+ source += (char) rand() % 256;
+ }
+
+ // Make the target string equal the source string.
+ string target = source;
+
+ Try<svn::Diff> diff = svn::diff(source, target);
+
+ ASSERT_SOME(diff);
+
+ Try<string> result = svn::patch(source, diff.get());
+
+ ASSERT_SOME_EQ(target, result);
+ ASSERT_SOME_EQ(source, result);
+}
[3/7] git commit: Updated svn::diff/patch to use newer versions of
functions.
Posted by be...@apache.org.
Updated svn::diff/patch to use newer versions of functions.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6a778df0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6a778df0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6a778df0
Branch: refs/heads/master
Commit: 6a778df0f6cd31004fd209ddf821f524652e4c19
Parents: fc6f59e
Author: Benjamin Hindman <be...@gmail.com>
Authored: Sat Oct 25 16:26:03 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Oct 26 22:33:34 2014 -0700
----------------------------------------------------------------------
3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6a778df0/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp b/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp
index db0e7e2..117ab0a 100644
--- a/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp
+++ b/3rdparty/libprocess/3rdparty/stout/include/stout/svn.hpp
@@ -90,10 +90,11 @@ inline Try<Diff> diff(const std::string& from, const std::string& to)
target.len = to.length();
svn_txdelta_stream_t* delta;
- svn_txdelta(
+ svn_txdelta2(
&delta,
svn_stream_from_string(&source, pool),
svn_stream_from_string(&target, pool),
+ false,
pool);
// Now we want to convert this text delta stream into an svndiff
@@ -103,11 +104,12 @@ inline Try<Diff> diff(const std::string& from, const std::string& to)
void* baton = NULL;
svn_stringbuf_t* diff = svn_stringbuf_create_ensure(1024, pool);
- svn_txdelta_to_svndiff2(
+ svn_txdelta_to_svndiff3(
&handler,
&baton,
svn_stream_from_stringbuf(diff, pool),
0,
+ SVN_DELTA_COMPRESSION_LEVEL_DEFAULT,
pool);
// Now feed the text delta to the handler.
[6/7] git commit: Added --with-apr and --with-svn to libprocess
configure.
Posted by be...@apache.org.
Added --with-apr and --with-svn to libprocess configure.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7a1020ed
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7a1020ed
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7a1020ed
Branch: refs/heads/master
Commit: 7a1020ed2d24782dc6059674036fb52240291caa
Parents: 6a778df
Author: Benjamin Hindman <be...@gmail.com>
Authored: Sat Oct 25 16:27:06 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Oct 26 22:33:34 2014 -0700
----------------------------------------------------------------------
3rdparty/libprocess/3rdparty/Makefile.am | 2 -
3rdparty/libprocess/configure.ac | 57 +++++++++++++++++++++++++++
2 files changed, 57 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7a1020ed/3rdparty/libprocess/3rdparty/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/Makefile.am b/3rdparty/libprocess/3rdparty/Makefile.am
index 4c6b2f1..e933c1c 100644
--- a/3rdparty/libprocess/3rdparty/Makefile.am
+++ b/3rdparty/libprocess/3rdparty/Makefile.am
@@ -186,8 +186,6 @@ endif
stout_tests_CPPFLAGS = \
-I$(srcdir)/$(STOUT)/include \
-I$(PROTOBUF)/src \
- -I/usr/include/subversion-1 \
- -I/usr/include/apr-1 \
$(AM_CPPFLAGS)
if WITH_BUNDLED_GMOCK
http://git-wip-us.apache.org/repos/asf/mesos/blob/7a1020ed/3rdparty/libprocess/configure.ac
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/configure.ac b/3rdparty/libprocess/configure.ac
index ee482fc..ec4d5a5 100644
--- a/3rdparty/libprocess/configure.ac
+++ b/3rdparty/libprocess/configure.ac
@@ -132,6 +132,17 @@ AC_ARG_WITH([zlib],
will be far less responsive; not recommended]),
[], [with_zlib=yes])
+AC_ARG_WITH([apr],
+ AS_HELP_STRING([--with-apr=@<:@=DIR@:>@],
+ [specify where to locate the apr-1 library]),
+ [], [])
+
+AC_ARG_WITH([svn],
+ AS_HELP_STRING([--with-svn=@<:@=DIR@:>@],
+ [specify where to locate the svn-1 library]),
+ [], [])
+
+
# There is no prefix installation of the JAR.
AC_ARG_VAR([PROTOBUF_JAR], [full path to protobuf jar on prefixed builds])
@@ -181,6 +192,52 @@ if test -n "`echo $with_glog`"; then
LDFLAGS="$LDFLAGS -L${with_glog}/lib"
fi
+
+# Check if libapr-1 prefix path was provided, and if so, add it to
+# the CPPFLAGS and LDFLAGS with respective /include/apr-1 and /lib path
+# suffixes. We include /include/apr-1 because we include <apr*>
+# headers directly.
+if test -n "`echo $with_apr`" ; then
+ CPPFLAGS="-I${with_apr}/include/apr-1 -I${with_apr}/include/apr-1.0 $CPPFLAGS"
+ LDFLAGS="-L${with_apr}/lib $LDFLAGS"
+else
+ CPPFLAGS="-I/usr/include/apr-1 -I/usr/include/apr-1.0 $CPPFLAGS"
+fi
+
+AC_CHECK_LIB([apr-1], [apr_initialize], [],
+ [AC_MSG_ERROR([cannot find libapr-1
+-------------------------------------------------------------------
+libapr-1 is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+
+# Check if libsvn-1 prefix path was provided, and if so, add it to
+# the CPPFLAGS and LDFLAGS with respective /include and /lib path
+# suffixes. We include /include/subversion-1 because we include
+# <svn_*> directly.
+if test -n "`echo $with_svn`" ; then
+ CPPFLAGS="-I${with_svn}/include/subversion-1 $CPPFLAGS"
+ LDFLAGS="-L${with_svn}/lib $LDFLAGS"
+else
+ CPPFLAGS="-I/usr/include/subversion-1 $CPPFLAGS"
+fi
+
+AC_CHECK_LIB([svn_subr-1], [svn_stringbuf_create_ensure], [],
+ [AC_MSG_ERROR([cannot find libsvn_subr-1
+-------------------------------------------------------------------
+libsubversion-1 is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+AC_CHECK_LIB([svn_delta-1], [svn_txdelta], [],
+ [AC_MSG_ERROR([cannot find libsvn_delta-1
+-------------------------------------------------------------------
+libsubversion-1 is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+
GMOCKSRC="gmock-all.cc"
GTESTSRC="gtest-all.cc"
[7/7] git commit: Ensured post-reviews.py added newline between
subject and body.
Posted by be...@apache.org.
Ensured post-reviews.py added newline between subject and body.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/fc6f59e1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/fc6f59e1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/fc6f59e1
Branch: refs/heads/master
Commit: fc6f59e1e4fd52eaef43d1a9c173c086a68fb5f4
Parents: 33e625f
Author: Benjamin Hindman <be...@gmail.com>
Authored: Sat Oct 25 16:15:25 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Oct 26 22:33:34 2014 -0700
----------------------------------------------------------------------
support/post-reviews.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/fc6f59e1/support/post-reviews.py
----------------------------------------------------------------------
diff --git a/support/post-reviews.py b/support/post-reviews.py
index a8ad94f..33c95fa 100755
--- a/support/post-reviews.py
+++ b/support/post-reviews.py
@@ -142,7 +142,7 @@ for i in range(len(shas)):
message = execute(['git',
'--no-pager',
'log',
- '--pretty=format:%s%n%b',
+ '--pretty=format:%s%n%n%b',
previous + '..' + sha])
review_request_id = None
[4/7] git commit: Added --with-apr and --with-svn to Mesos configure.
Posted by be...@apache.org.
Added --with-apr and --with-svn to Mesos configure.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d418c17b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d418c17b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d418c17b
Branch: refs/heads/master
Commit: d418c17b89856a9e9d43ece0a7c656f2489a922a
Parents: 7a1020e
Author: Benjamin Hindman <be...@gmail.com>
Authored: Sat Oct 25 16:29:04 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Oct 26 22:33:34 2014 -0700
----------------------------------------------------------------------
configure.ac | 55 ++++++++++++++++++++++++++++++++++++++++++++
docs/getting-started.md | 19 ++++++++++++++-
src/Makefile.am | 2 --
3 files changed, 73 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/d418c17b/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index c8c1f94..342e762 100644
--- a/configure.ac
+++ b/configure.ac
@@ -193,6 +193,16 @@ AC_ARG_WITH([zlib],
[specify where to locate the zlib library]),
[], [])
+AC_ARG_WITH([apr],
+ AS_HELP_STRING([--with-apr=@<:@=DIR@:>@],
+ [specify where to locate the apr-1 library]),
+ [], [])
+
+AC_ARG_WITH([svn],
+ AS_HELP_STRING([--with-svn=@<:@=DIR@:>@],
+ [specify where to locate the svn-1 library]),
+ [], [])
+
AC_ARG_ENABLE([bundled-distribute],
AS_HELP_STRING([--disable-bundled-distribute],
[excludes building and using the bundled distribute
@@ -630,6 +640,51 @@ libcurl is required for mesos to build.
])])
+# Check if libapr-1 prefix path was provided, and if so, add it to
+# the CPPFLAGS and LDFLAGS with respective /include/apr-1 and /lib path
+# suffixes. We include /include/apr-1 because we include <apr*>
+# headers directly.
+if test -n "`echo $with_apr`" ; then
+ CPPFLAGS="-I${with_apr}/include/apr-1 -I${with_apr}/include/apr-1.0 $CPPFLAGS"
+ LDFLAGS="-L${with_apr}/lib $LDFLAGS"
+else
+ CPPFLAGS="-I/usr/include/apr-1 -I/usr/include/apr-1.0 $CPPFLAGS"
+fi
+
+AC_CHECK_LIB([apr-1], [apr_initialize], [],
+ [AC_MSG_ERROR([cannot find libapr-1
+-------------------------------------------------------------------
+libapr-1 is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+
+# Check if libsvn-1 prefix path was provided, and if so, add it to
+# the CPPFLAGS and LDFLAGS with respective /include and /lib path
+# suffixes. We include /include/subversion-1 because we include
+# <svn_*> directly.
+if test -n "`echo $with_svn`" ; then
+ CPPFLAGS="-I${with_svn}/include/subversion-1 $CPPFLAGS"
+ LDFLAGS="-L${with_svn}/lib $LDFLAGS"
+else
+ CPPFLAGS="-I/usr/include/subversion-1 $CPPFLAGS"
+fi
+
+AC_CHECK_LIB([svn_subr-1], [svn_stringbuf_create_ensure], [],
+ [AC_MSG_ERROR([cannot find libsvn_subr-1
+-------------------------------------------------------------------
+libsubversion-1 is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+AC_CHECK_LIB([svn_delta-1], [svn_txdelta], [],
+ [AC_MSG_ERROR([cannot find libsvn_delta-1
+-------------------------------------------------------------------
+libsubversion-1 is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+
# Check if Sasl2 prefix path was provided, and if so, add it to
# the CPPFLAGS and LDFLAGS with respective /include and /lib path
# suffixes.
http://git-wip-us.apache.org/repos/asf/mesos/blob/d418c17b/docs/getting-started.md
----------------------------------------------------------------------
diff --git a/docs/getting-started.md b/docs/getting-started.md
index 1602914..4014b28 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -46,6 +46,12 @@ There are different ways you can get Mesos:
# Install Maven (***Only required for Mesos 0.18.1 or newer***).
$ sudo apt-get install maven
+ # Install devel libapr1 (***Only required for Mesos 0.21.0 or newer***)
+ $ sudo apt-get install libapr1-dev
+
+ # Install devel libsvn (***Only required for Mesos 0.21.0 or newer***)
+ $ sudo apt-get install libsvn-dev
+
- If you are building from git repository, you will need to additionally install the following packages.
# Install autotoconf and automake.
@@ -58,9 +64,20 @@ There are different ways you can get Mesos:
- Following are the instructions for stock CentOS 6.5. If you are using a different OS, please install the packages accordingly.
+ Mesos 0.21.0+ requires subversion 1.8+ devel package which is not available by default by yum.
+ Add one of the repo that has subversion-devel 1.8 available, i.e:
+
+ Add new repo /etc/yum.repos.d/wandisco-svn.repo, with:
+
+ [WandiscoSVN]
+ name=Wandisco SVN Repo
+ baseurl=http://opensource.wandisco.com/centos/6/svn-1.8/RPMS/$basearch/
+ enabled=1
+ gpgcheck=0
+
$ sudo yum groupinstall -y "Development Tools"
- $ sudo yum install -y python-devel java-1.7.0-openjdk-devel zlib-devel libcurl-devel openssl-devel cyrus-sasl-devel cyrus-sasl-md5
+ $ sudo yum install -y python-devel java-1.7.0-openjdk-devel zlib-devel libcurl-devel openssl-devel cyrus-sasl-devel cyrus-sasl-md5 apr-devel subversion-devel
# Install maven.
$ wget http://mirror.nexcess.net/apache/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz
http://git-wip-us.apache.org/repos/asf/mesos/blob/d418c17b/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 00d5d3c..6820d8a 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -89,8 +89,6 @@ MESOS_CPPFLAGS += -I$(top_srcdir)/include
MESOS_CPPFLAGS += -I$(top_srcdir)/$(LIBPROCESS)/include
MESOS_CPPFLAGS += -I$(top_srcdir)/$(STOUT)/include
MESOS_CPPFLAGS += -I../include
-MESOS_CPPFLAGS += -I/usr/include/subversion-1
-MESOS_CPPFLAGS += -I/usr/include/apr-1
# Protobuf headers that depend on mesos.pb.h need this.
MESOS_CPPFLAGS += -I../include/mesos