You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/01/29 22:28:46 UTC
[1/8] incubator-kudu git commit: add async method to locate table
Repository: incubator-kudu
Updated Branches:
refs/heads/master eb592b298 -> 82fdeb64e
add async method to locate table
Now AsyncKuduClient only have one method called syncLocateTable to
locate table in a sync way, and it will block the user's thread,
which is not friendly in asynchronous programming.
This patch add another method to locate table asynchronously.
Change-Id: I0868ed77366ff7d45f92f99dfd033c244d53b48f
Reviewed-on: http://gerrit.cloudera.org:8080/1916
Reviewed-by: Jean-Daniel Cryans
Tested-by: Jean-Daniel Cryans
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/48a7b2f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/48a7b2f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/48a7b2f0
Branch: refs/heads/master
Commit: 48a7b2f0be477d897573d46f135daeb5f90abf8b
Parents: eb592b2
Author: zhangzhen <zh...@xiaomi.com>
Authored: Wed Jan 27 17:18:39 2016 +0800
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Fri Jan 29 18:39:48 2016 +0000
----------------------------------------------------------------------
.../java/org/kududb/client/AsyncKuduClient.java | 110 ++++++++++++-------
.../main/java/org/kududb/client/KuduTable.java | 29 +++++
.../java/org/kududb/client/TestKuduTable.java | 10 +-
3 files changed, 106 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/48a7b2f0/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
index 53e9066..1a396b1 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
@@ -36,6 +36,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.Message;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
+
import org.jboss.netty.buffer.ChannelBuffer;
import org.kududb.Common;
import org.kududb.Schema;
@@ -43,6 +44,7 @@ import org.kududb.annotations.InterfaceAudience;
import org.kududb.annotations.InterfaceStability;
import org.kududb.consensus.Metadata;
import org.kududb.master.Master;
+import org.kududb.master.Master.GetTableLocationsResponsePB;
import org.kududb.util.AsyncUtil;
import org.kududb.util.NetUtil;
import org.kududb.util.Pair;
@@ -62,6 +64,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.GuardedBy;
+
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
@@ -1019,50 +1022,73 @@ public class AsyncKuduClient implements AutoCloseable {
byte[] startPartitionKey,
byte[] endPartitionKey,
long deadline) throws Exception {
- List<LocatedTablet> ret = Lists.newArrayList();
- byte[] lastEndPartitionKey = null;
+ return locateTable(tableId, startPartitionKey, endPartitionKey, deadline).join();
+ }
- DeadlineTracker deadlineTracker = new DeadlineTracker();
- deadlineTracker.setDeadline(deadline);
- while (true) {
- if (deadlineTracker.timedOut()) {
- throw new NonRecoverableException("Took too long getting the list of tablets, " +
- "deadline=" + deadline);
- }
- GetTableLocationsRequest rpc =
- new GetTableLocationsRequest(masterTable, startPartitionKey, endPartitionKey, tableId);
- rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- final Deferred<Master.GetTableLocationsResponsePB> d = sendRpcToTablet(rpc);
- Master.GetTableLocationsResponsePB response =
- d.join(deadlineTracker.getMillisBeforeDeadline());
- // Table doesn't exist or is being created.
- if (response.getTabletLocationsCount() == 0) {
- break;
- }
- for (Master.TabletLocationsPB tabletPb : response.getTabletLocationsList()) {
- LocatedTablet locs = new LocatedTablet(tabletPb);
- ret.add(locs);
-
- Partition partition = locs.getPartition();
- if (lastEndPartitionKey != null &&
- !partition.isEndPartition() &&
- Bytes.memcmp(partition.getPartitionKeyEnd(), lastEndPartitionKey) < 0) {
- throw new IllegalStateException(
- "Server returned tablets out of order: " +
- "end partition key '" + Bytes.pretty(partition.getPartitionKeyEnd()) + "' followed " +
- "end partition key '" + Bytes.pretty(lastEndPartitionKey) + "'");
- }
- lastEndPartitionKey = partition.getPartitionKeyEnd();
- }
- // If true, we're done, else we have to go back to the master with the last end key
- if (lastEndPartitionKey.length == 0 ||
- (endPartitionKey != null && Bytes.memcmp(lastEndPartitionKey, endPartitionKey) > 0)) {
- break;
- } else {
- startPartitionKey = lastEndPartitionKey;
- }
+ private Deferred<List<LocatedTablet>> loopLocateTable(final String tableId,
+ final byte[] startPartitionKey, final byte[] endPartitionKey, final List<LocatedTablet> ret,
+ final DeadlineTracker deadlineTracker) {
+ if (deadlineTracker.timedOut()) {
+ return Deferred.fromError(new NonRecoverableException(
+ "Took too long getting the list of tablets, " + deadlineTracker));
}
- return ret;
+ GetTableLocationsRequest rpc = new GetTableLocationsRequest(masterTable, startPartitionKey,
+ endPartitionKey, tableId);
+ rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ final Deferred<Master.GetTableLocationsResponsePB> d = sendRpcToTablet(rpc);
+ return d.addCallbackDeferring(
+ new Callback<Deferred<List<LocatedTablet>>, Master.GetTableLocationsResponsePB>() {
+ @Override
+ public Deferred<List<LocatedTablet>> call(GetTableLocationsResponsePB response) {
+ // Table doesn't exist or is being created.
+ if (response.getTabletLocationsCount() == 0) {
+ Deferred.fromResult(ret);
+ }
+ byte[] lastEndPartition = startPartitionKey;
+ for (Master.TabletLocationsPB tabletPb : response.getTabletLocationsList()) {
+ LocatedTablet locs = new LocatedTablet(tabletPb);
+ ret.add(locs);
+ Partition partition = locs.getPartition();
+ if (lastEndPartition != null && !partition.isEndPartition()
+ && Bytes.memcmp(partition.getPartitionKeyEnd(), lastEndPartition) < 0) {
+ return Deferred.fromError(new IllegalStateException(
+ "Server returned tablets out of order: " + "end partition key '"
+ + Bytes.pretty(partition.getPartitionKeyEnd()) + "' followed "
+ + "end partition key '" + Bytes.pretty(lastEndPartition) + "'"));
+ }
+ lastEndPartition = partition.getPartitionKeyEnd();
+ }
+ // If true, we're done, else we have to go back to the master with the last end key
+ if (lastEndPartition.length == 0
+ || (endPartitionKey != null && Bytes.memcmp(lastEndPartition, endPartitionKey) > 0)) {
+ return Deferred.fromResult(ret);
+ } else {
+ return loopLocateTable(tableId, lastEndPartition, endPartitionKey, ret,
+ deadlineTracker);
+ }
+ }
+ });
+ }
+
+ /**
+ * Get all or some tablets for a given table. This may query the master multiple times if there
+ * are a lot of tablets.
+ * @param tableId the table to locate tablets from
+ * @param startPartitionKey where to start in the table, pass null to start at the beginning
+ * @param endPartitionKey where to stop in the table, pass null to get all the tablets until the
+ * end of the table
+ * @param deadline max time spent in milliseconds for the deferred result of this method to
+ * get called back, if deadline is reached, the deferred result will get erred back
+ * @return a deferred object that yields a list of the tablets in the table, which can be queried
+ * for metadata about each tablet
+ * @throws Exception MasterErrorException if the table doesn't exist
+ */
+ Deferred<List<LocatedTablet>> locateTable(final String tableId,
+ final byte[] startPartitionKey, final byte[] endPartitionKey, long deadline) {
+ final List<LocatedTablet> ret = Lists.newArrayList();
+ final DeadlineTracker deadlineTracker = new DeadlineTracker();
+ deadlineTracker.setDeadline(deadline);
+ return loopLocateTable(tableId, startPartitionKey, endPartitionKey, ret, deadlineTracker);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/48a7b2f0/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java b/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java
index 58b9a67..b0dd4e7 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java
@@ -20,6 +20,8 @@ import org.kududb.Schema;
import org.kududb.annotations.InterfaceAudience;
import org.kududb.annotations.InterfaceStability;
+import com.stumbleupon.async.Deferred;
+
import java.util.List;
/**
@@ -140,6 +142,18 @@ public class KuduTable {
}
/**
+ * Asynchronously get all the tablets for this table.
+ * @param deadline max time spent in milliseconds for the deferred result of this method to
+ * get called back, if deadline is reached, the deferred result will get erred back
+ * @return a {@link Deferred} object that yields a list containing the metadata and
+ * locations for each of the tablets in the table
+ */
+ public Deferred<List<LocatedTablet>> asyncGetTabletsLocations(
+ long deadline) throws Exception {
+ return asyncGetTabletsLocations(null, null, deadline);
+ }
+
+ /**
* Get all or some tablets for this table. This may query the master multiple times if there
* are a lot of tablets.
* This method blocks until it gets all the tablets.
@@ -156,4 +170,19 @@ public class KuduTable {
return client.syncLocateTable(tableId, startKey, endKey, deadline);
}
+ /**
+ * Asynchronously get all or some tablets for this table.
+ * @param startKey where to start in the table, pass null to start at the beginning
+ * @param endKey where to stop in the table, pass null to get all the tablets until the end of
+ * the table
+ * @param deadline max time spent in milliseconds for the deferred result of this method to
+ * get called back, if deadline is reached, the deferred result will get erred back
+ * @return a {@link Deferred} object that yields a list containing the metadata and locations
+ * for each of the tablets in the table
+ */
+ public Deferred<List<LocatedTablet>> asyncGetTabletsLocations(
+ byte[] startKey, byte[] endKey, long deadline) throws Exception {
+ return client.locateTable(tableId, startKey, endKey, deadline);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/48a7b2f0/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java b/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
index abe77f1..b19753b 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
@@ -172,28 +172,35 @@ public class TestKuduTable extends BaseKuduTest {
List<LocatedTablet>tablets = table.getTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP);
assertEquals(10, tablets.size());
+ assertEquals(10, table.asyncGetTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP).join().size());
tablets = table.getTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP);
assertEquals(10, tablets.size());
+ assertEquals(10, table.asyncGetTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP).join().size());
tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP);
assertEquals(5, tablets.size());
+ assertEquals(5, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP).join().size());
tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP);
assertEquals(10, tablets.size());
+ assertEquals(10, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP).join().size());
tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(31), DEFAULT_SLEEP);
assertEquals(26, tablets.size());
+ assertEquals(26, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(31), DEFAULT_SLEEP).join().size());
tablets = table.getTabletsLocations(getKeyInBytes(5), null, DEFAULT_SLEEP);
assertEquals(26, tablets.size());
+ assertEquals(26, table.asyncGetTabletsLocations(getKeyInBytes(5), null, DEFAULT_SLEEP).join().size());
tablets = table.getTabletsLocations(null, getKeyInBytes(10000), DEFAULT_SLEEP);
assertEquals(31, tablets.size());
+ assertEquals(31, table.asyncGetTabletsLocations(null, getKeyInBytes(10000), DEFAULT_SLEEP).join().size());
tablets = table.getTabletsLocations(getKeyInBytes(20), getKeyInBytes(10000), DEFAULT_SLEEP);
assertEquals(11, tablets.size());
-
+ assertEquals(11, table.asyncGetTabletsLocations(getKeyInBytes(20), getKeyInBytes(10000), DEFAULT_SLEEP).join().size());
// Test listing tables.
assertEquals(0, client.getTablesList(table1).join(DEFAULT_SLEEP).getTablesList().size());
@@ -232,6 +239,7 @@ public class TestKuduTable extends BaseKuduTest {
List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
assertEquals(splitsCount + 1, tablets.size());
+ assertEquals(splitsCount + 1, table.asyncGetTabletsLocations(DEFAULT_SLEEP).join().size());
for (LocatedTablet tablet : tablets) {
assertEquals(1, tablet.getReplicas().size());
}
[2/8] incubator-kudu git commit: KUDU-1250. [java client] Track row
errors in KuduSession when AUTO_FLUSH_BACKGROUND
Posted by ad...@apache.org.
KUDU-1250. [java client] Track row errors in KuduSession when AUTO_FLUSH_BACKGROUND
This patch adds a critically missing piece in the Java sync client.
It was impossible to track row errors when using AUTO_FLUSH_BACKGROUND.
You could still do it with the async client by attaching callbacks.
The APIs are named the same as in the C++ client.
Change-Id: I0216619f3a49f2b70b8719dac99a787cf46202a1
Reviewed-on: http://gerrit.cloudera.org:8080/1924
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Jean-Daniel Cryans
Reviewed-by: Dan Burkert <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/d70305c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/d70305c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/d70305c4
Branch: refs/heads/master
Commit: d70305c4f03015ffff8292446b0c9c580fc4dbc0
Parents: 48a7b2f
Author: Jean-Daniel Cryans <jd...@cloudera.com>
Authored: Wed Jan 27 11:23:20 2016 -0800
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Fri Jan 29 18:45:35 2016 +0000
----------------------------------------------------------------------
.../org/kududb/client/AsyncKuduSession.java | 18 ++++++++-
.../java/org/kududb/client/KuduSession.java | 17 ++++++--
.../org/kududb/client/SessionConfiguration.java | 41 +++++++++++++-------
.../java/org/kududb/client/TestRowErrors.java | 35 +++++++++++++----
4 files changed, 87 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/d70305c4/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java
index 470d8af..e5fdb42 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java
@@ -87,6 +87,7 @@ public class AsyncKuduSession implements SessionConfiguration {
private final AsyncKuduClient client;
private final Random randomizer = new Random();
+ private final ErrorCollector errorCollector;
private int interval = 1000;
private int mutationBufferSpace = 1000; // TODO express this in terms of data size.
private float mutationBufferLowWatermarkPercentage = 0.5f;
@@ -143,6 +144,7 @@ public class AsyncKuduSession implements SessionConfiguration {
this.consistencyMode = CLIENT_PROPAGATED;
this.timeoutMs = client.getDefaultOperationTimeoutMs();
setMutationBufferLowWatermark(this.mutationBufferLowWatermarkPercentage);
+ errorCollector = new ErrorCollector(mutationBufferSpace);
}
@Override
@@ -230,6 +232,16 @@ public class AsyncKuduSession implements SessionConfiguration {
this.ignoreAllDuplicateRows = ignoreAllDuplicateRows;
}
+ @Override
+ public int countPendingErrors() {
+ return errorCollector.countErrors();
+ }
+
+ @Override
+ public RowErrorsAndOverflowStatus getPendingErrors() {
+ return errorCollector.getErrors();
+ }
+
/**
* Flushes the buffered operations and marks this sessions as closed.
* See the javadoc on {@link #flush()} on how to deal with exceptions coming out of this method.
@@ -559,7 +571,7 @@ public class AsyncKuduSession implements SessionConfiguration {
/**
* Creates callbacks to handle a multi-put and adds them to the request.
- * @param request The request for which we must handle the response.
+ * @param request the request for which we must handle the response
*/
private void addBatchCallbacks(final Batch request) {
final class BatchCallback implements
@@ -573,7 +585,11 @@ public class AsyncKuduSession implements SessionConfiguration {
// Send individualized responses to all the operations in this batch.
for (OperationResponse operationResponse : response.getIndividualResponses()) {
operationResponse.getOperation().callback(operationResponse);
+ if (flushMode == FlushMode.AUTO_FLUSH_BACKGROUND && operationResponse.hasRowError()) {
+ errorCollector.addError(operationResponse.getRowError());
+ }
}
+
return response;
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/d70305c4/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java b/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java
index c60043e..f13f9c6 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java
@@ -54,9 +54,10 @@ public class KuduSession implements SessionConfiguration {
* <li>AUTO_FLUSH_SYNC: the call returns when the operation is persisted,
* else it throws an exception.
* <li>AUTO_FLUSH_BACKGROUND: the call returns when the operation has been added to the buffer.
- * The operation's state is then unreachable, meaning that there's no way to know if the
- * operation is persisted. This call should normally perform only fast in-memory operations but
- * it may have to wait when the buffer is full and there's another buffer being flushed.
+ * This call should normally perform only fast in-memory operations but
+ * it may have to wait when the buffer is full and there's another buffer being flushed. Row
+ * errors can be checked by calling {@link #countPendingErrors()} and can be retrieved by calling
+ * {@link #getPendingErrors()}.
* <li>MANUAL_FLUSH: the call returns when the operation has been added to the buffer,
* else it throws an exception such as a NonRecoverableException if the buffer is full.
* </ul>
@@ -168,4 +169,14 @@ public class KuduSession implements SessionConfiguration {
public void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows) {
session.setIgnoreAllDuplicateRows(ignoreAllDuplicateRows);
}
+
+ @Override
+ public int countPendingErrors() {
+ return session.countPendingErrors();
+ }
+
+ @Override
+ public RowErrorsAndOverflowStatus getPendingErrors() {
+ return session.getPendingErrors();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/d70305c4/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java b/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java
index 24bfa2b..94e0a66 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java
@@ -29,7 +29,7 @@ public interface SessionConfiguration {
@InterfaceAudience.Public
@InterfaceStability.Evolving
- public enum FlushMode {
+ enum FlushMode {
// Every write will be sent to the server in-band with the Apply()
// call. No batching will occur. This is the default flush mode. In this
// mode, the Flush() call never has any effect, since each Apply() call
@@ -58,21 +58,21 @@ public interface SessionConfiguration {
* Get the current flush mode.
* @return flush mode, AUTO_FLUSH_SYNC by default
*/
- public FlushMode getFlushMode();
+ FlushMode getFlushMode();
/**
* Set the new flush mode for this session.
* @param flushMode new flush mode, can be the same as the previous one.
* @throws IllegalArgumentException if the buffer isn't empty.
*/
- public void setFlushMode(FlushMode flushMode);
+ void setFlushMode(FlushMode flushMode);
/**
* Set the number of operations that can be buffered.
* @param size number of ops.
* @throws IllegalArgumentException if the buffer isn't empty.
*/
- public void setMutationBufferSpace(int size);
+ void setMutationBufferSpace(int size);
/**
* Set the low watermark for this session. The default is set to half the mutation buffer space.
@@ -85,51 +85,51 @@ public interface SessionConfiguration {
* @throws IllegalArgumentException if the buffer isn't empty or if the watermark isn't between
* 0 and 1
*/
- public void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage);
+ void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage);
/**
* Set the flush interval, which will be used for the next scheduling decision.
* @param interval interval in milliseconds.
*/
- public void setFlushInterval(int interval);
+ void setFlushInterval(int interval);
/**
* Get the current timeout.
* @return operation timeout in milliseconds, 0 if none was configured.
*/
- public long getTimeoutMillis();
+ long getTimeoutMillis();
/**
* Sets the timeout for the next applied operations.
* The default timeout is 0, which disables the timeout functionality.
* @param timeout Timeout in milliseconds.
*/
- public void setTimeoutMillis(long timeout);
+ void setTimeoutMillis(long timeout);
/**
* Returns true if this session has already been closed.
*/
- public boolean isClosed();
+ boolean isClosed();
/**
* Check if there are operations that haven't been completely applied.
* @return true if operations are pending, else false.
*/
- public boolean hasPendingOperations();
+ boolean hasPendingOperations();
/**
* Set the new external consistency mode for this session.
* @param consistencyMode new external consistency mode, can the same as the previous one.
* @throws IllegalArgumentException if the buffer isn't empty.
*/
- public void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode);
+ void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode);
/**
* Tells if the session is currently ignoring row errors when the whole list returned by a tablet
* server is of the AlreadyPresent type.
* @return true if the session is enforcing this, else false
*/
- public boolean isIgnoreAllDuplicateRows();
+ boolean isIgnoreAllDuplicateRows();
/**
* Configures the option to ignore all the row errors if they are all of the AlreadyPresent type.
@@ -139,5 +139,20 @@ public interface SessionConfiguration {
* This is disabled by default.
* @param ignoreAllDuplicateRows true if this session should enforce this, else false
*/
- public void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows);
+ void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows);
+
+ /**
+ * Return the number of errors which are pending. Errors may accumulate when
+ * using the AUTO_FLUSH_BACKGROUND mode.
+ * @return a count of errors
+ */
+ int countPendingErrors();
+
+ /**
+ * Return any errors from previous calls. If there were more errors
+ * than could be held in the session's error storage, the overflow state is set to true.
+ * Resets the pending errors.
+ * @return an object that contains the errors and the overflow status
+ */
+ RowErrorsAndOverflowStatus getPendingErrors();
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/d70305c4/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java b/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java
index 1d68377..39e977d 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java
@@ -25,23 +25,20 @@ import static org.junit.Assert.*;
public class TestRowErrors extends BaseKuduTest {
- // Generate a unique table name
- private static final String TABLE_NAME =
- TestRowErrors.class.getName() + "-" + System.currentTimeMillis();
-
private static KuduTable table;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
BaseKuduTest.setUpBeforeClass();
- createTable(TABLE_NAME, basicSchema, new CreateTableOptions());
- table = openTable(TABLE_NAME);
}
@Test(timeout = 100000)
- public void test() throws Exception {
+ public void singleTabletTest() throws Exception {
+ String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis();
+ createTable(tableName, basicSchema, new CreateTableOptions());
+ table = openTable(tableName);
AsyncKuduSession session = client.newSession();
// Insert 3 rows to play with.
@@ -70,6 +67,30 @@ public class TestRowErrors extends BaseKuduTest {
assertTrue(errors.get(1).getOperation() == dupeForTwo);
}
+ /**
+ * Test collecting errors from multiple tablets.
+ * @throws Exception
+ */
+ @Test(timeout = 100000)
+ public void multiTabletTest() throws Exception {
+ String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis();
+ createFourTabletsTableWithNineRows(tableName);
+ table = openTable(tableName);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
+
+ int dupRows = 3;
+ session.apply(createInsert(12));
+ session.apply(createInsert(22));
+ session.apply(createInsert(32));
+
+ session.flush();
+
+ RowErrorsAndOverflowStatus reos = session.getPendingErrors();
+ assertEquals(dupRows, reos.getRowErrors().length);
+ assertEquals(0, session.countPendingErrors());
+ }
+
private Insert createInsert(int key) {
return createBasicSchemaInsert(table, key);
}
[5/8] incubator-kudu git commit: cmake: remove unnecessary
link_directories()
Posted by ad...@apache.org.
cmake: remove unnecessary link_directories()
While trying to reduce my unit tests' rpath, I stumbled upon this. It's no
longer necessary (perhaps due to the llvm 3.7 upgrade), and removing it
trims the library search path and rpath for all binaries.
In the end shortening the rpaths didn't help (chrpath refuses to grow a
binary's rpath, only rewrite it), but I think the patch still makes sense.
Change-Id: I60af144c324cc41bf01b3d878a549530f578fec8
Reviewed-on: http://gerrit.cloudera.org:8080/1941
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/2277d8d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/2277d8d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/2277d8d7
Branch: refs/heads/master
Commit: 2277d8d786268973292e98d5847f2877a8af66f6
Parents: f3f3996
Author: Adar Dembo <ad...@cloudera.com>
Authored: Wed Jan 27 19:49:54 2016 -0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Fri Jan 29 20:05:57 2016 +0000
----------------------------------------------------------------------
CMakeLists.txt | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/2277d8d7/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ae0e328..5bbcd08 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -804,7 +804,6 @@ if(${LLVM_PACKAGE_VERSION} VERSION_LESS 3.4)
endif()
message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
-link_directories(${LLVM_LIBRARY_DIRS})
## librt
if (NOT APPLE)
[6/8] incubator-kudu git commit: tpch: avoid build/latest symlink
Posted by ad...@apache.org.
tpch: avoid build/latest symlink
Since the script only builds the tpch1 target, there's no guarantee that the
build/latest symlink exists. Use the BUILD_TYPE instead.
Change-Id: Ib327cd512f21a909ada253eae7a7b06189fd5c70
Reviewed-on: http://gerrit.cloudera.org:8080/1961
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/dd9f9d2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/dd9f9d2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/dd9f9d2f
Branch: refs/heads/master
Commit: dd9f9d2ff5ae26ceedd6eba26a52232ab869498e
Parents: 2277d8d
Author: Adar Dembo <ad...@cloudera.com>
Authored: Fri Jan 29 12:26:18 2016 -0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Fri Jan 29 20:28:58 2016 +0000
----------------------------------------------------------------------
src/kudu/scripts/tpch.sh | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/dd9f9d2f/src/kudu/scripts/tpch.sh
----------------------------------------------------------------------
diff --git a/src/kudu/scripts/tpch.sh b/src/kudu/scripts/tpch.sh
index bbad1ee..447f02b 100755
--- a/src/kudu/scripts/tpch.sh
+++ b/src/kudu/scripts/tpch.sh
@@ -128,11 +128,11 @@ OUTDIR=$ROOT/build/$BUILD_TYPE/tpch
rm -Rf $KUDU_DATA_DIR # Clean up data dir.
mkdir -p $OUTDIR # Create log file output dir.
-./build/latest/bin/tpch1 -logtostderr=1 \
- -tpch_path_to_data=$LINEITEM_TBL_PATH \
- -mini_cluster_base_dir=$KUDU_DATA_DIR \
- -tpch_num_query_iterations=$TPCH_NUM_QUERY_ITERS \
- >$OUTDIR/benchmark.log 2>&1
+./build/$BUILD_TYPE/bin/tpch1 -logtostderr=1 \
+ -tpch_path_to_data=$LINEITEM_TBL_PATH \
+ -mini_cluster_base_dir=$KUDU_DATA_DIR \
+ -tpch_num_query_iterations=$TPCH_NUM_QUERY_ITERS \
+ >$OUTDIR/benchmark.log 2>&1
cat $OUTDIR/benchmark.log
INSERT_TIME=$(grep "Time spent loading" $OUTDIR/benchmark.log | \
[7/8] incubator-kudu git commit: cmake: require out-of-tree builds
Posted by ad...@apache.org.
cmake: require out-of-tree builds
With out-of-tree builds once again working, this patch "flips the switch"
and requires all builds to be out-of-tree. There's an override for complex
workflows; to use it, run cmake with -DKUDU_ALLOW_IN_SOURCE_BUILD=1.
Change-Id: I57a9eab7d35419a3668b9dce384a049cddd1afef
Reviewed-on: http://gerrit.cloudera.org:8080/1762
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/f461e0a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/f461e0a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/f461e0a5
Branch: refs/heads/master
Commit: f461e0a5e6a5b2a936c253065f078c8fc83e070a
Parents: dd9f9d2
Author: Adar Dembo <ad...@cloudera.com>
Authored: Mon Jan 11 16:00:36 2016 -0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Fri Jan 29 20:49:20 2016 +0000
----------------------------------------------------------------------
CMakeLists.txt | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f461e0a5/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5bbcd08..c67b54a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,6 +22,22 @@
# 1. https://cmake.org/Bug/view.php?id=14963
cmake_minimum_required(VERSION 3.2.0)
+# Prevent builds from the top-level source directory. This ensures that build
+# output is well isolated from the source tree.
+#
+# May be overridden by setting KUDU_ALLOW_IN_SOURCE_BUILDS; this is only
+# recommended for experts!
+if("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" AND
+ NOT "${KUDU_ALLOW_IN_SOURCE_BUILD}")
+ message(FATAL_ERROR
+ "Kudu may not be built from the top-level source directory. Create a new "
+ "directory and run cmake from there, passing the path to the top-level "
+ "source directory as the last argument. "
+ "To override this, rerun CMake with -DKUDU_ALLOW_IN_SOURCE_BUILD=1. "
+ "Also, delete 'CMakeCache.txt' and 'CMakeFiles' from the top-level source "
+ "directory, otherwise future builds will not work.")
+endif()
+
# Provide a 'latest' symlink to this build directory if the "blessed"
# multi-build layout is detected:
#
[8/8] incubator-kudu git commit: Add info to setup.py and bump
version to 0.1.1
Posted by ad...@apache.org.
Add info to setup.py and bump version to 0.1.1
This adds more info to setup.py (URL, new apache mailing list) and bumps
the version to 0.1.1 so that we can push a new version to pypi that
reflects the recent changes.
Change-Id: I29394483e9eadfdea839e4b0b32b9db9f9c3fff3
Reviewed-on: http://gerrit.cloudera.org:8080/1946
Reviewed-by: David Ribeiro Alves <da...@cloudera.com>
Tested-by: David Ribeiro Alves <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/82fdeb64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/82fdeb64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/82fdeb64
Branch: refs/heads/master
Commit: 82fdeb64e19dc7f5c5183866b4242a17e58e686a
Parents: f461e0a
Author: David Alves <da...@cloudera.com>
Authored: Thu Jan 28 10:29:52 2016 -0800
Committer: David Ribeiro Alves <da...@cloudera.com>
Committed: Fri Jan 29 21:15:31 2016 +0000
----------------------------------------------------------------------
python/setup.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/82fdeb64/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 1b250f3..993dace 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -33,7 +33,7 @@ if Cython.__version__ < '0.19.1':
MAJOR = 0
MINOR = 1
-MICRO = 0
+MICRO = 1
VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
ISRELEASED = True
@@ -141,6 +141,8 @@ CLASSIFIERS = [
'Programming Language :: Cython'
]
+URL = 'http://getkudu.io'
+
setup(
name="kudu-python",
packages=['kudu', 'kudu.tests'],
@@ -158,7 +160,8 @@ setup(
long_description=LONG_DESCRIPTION,
license='Apache License, Version 2.0',
classifiers=CLASSIFIERS,
- author="Apache Kudu (incubating) team",
+ maintainer="Apache Kudu (incubating) team",
maintainer_email="dev@kudu.incubator.apache.org",
+ url=URL,
test_suite="kudu.tests"
)
[3/8] incubator-kudu git commit: Correct 'RAFT' to 'Raft'
Posted by ad...@apache.org.
Correct 'RAFT' to 'Raft'
Change-Id: Id084193ddb70f50d7d86abd2520cb7437c913e5d
Reviewed-on: http://gerrit.cloudera.org:8080/1958
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/588abf6f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/588abf6f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/588abf6f
Branch: refs/heads/master
Commit: 588abf6fa9d572ae907407e3fe0e7c62fb5d1e4f
Parents: d70305c
Author: Misty Stanley-Jones <ms...@cloudera.com>
Authored: Fri Jan 29 10:18:05 2016 -0800
Committer: Misty Stanley-Jones <mi...@apache.org>
Committed: Fri Jan 29 18:50:52 2016 +0000
----------------------------------------------------------------------
docs/introduction.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/588abf6f/docs/introduction.adoc
----------------------------------------------------------------------
diff --git a/docs/introduction.adoc b/docs/introduction.adoc
index aa10406..35f7b41 100644
--- a/docs/introduction.adoc
+++ b/docs/introduction.adoc
@@ -151,7 +151,7 @@ substantial reduction in network traffic for heavy write scenarios.
== Architectural Overview
The following diagram shows a Kudu cluster with three masters and multiple tablet
-servers, each serving multiple tablets. It illustrates how RAFT consensus is used
+servers, each serving multiple tablets. It illustrates how Raft consensus is used
to allow for both leaders and followers for both the masters and tablet servers. In
addition, a tablet server can be a leader for some tablets, and a follower for others.
Leaders are shown in gold, while followers are shown in blue.
[4/8] incubator-kudu git commit: cmake: blessed build layout fixes
and improvements
Posted by ad...@apache.org.
cmake: blessed build layout fixes and improvements
This patch improves the experience of out-of-tree builds by relying on the
idea of a "blessed" build layout. To summarize, despite now having the
flexibility to place build directories anywhere, we will still encourage
developers to lay them out in a particular way. All scripts and tests will
work well this way, and we'll make only a best effort for them to work
otherwise.
The blessed build layout is where all build directories are subdirectories
of a main build/ subdirectory, itself found in the root directory of the
tree. For example:
- kudu/build/ [unused]
- kudu/build/debug [build directory for debug build]
- kudu/build/asan [build directory for address sanitizer build]
- kudu/build/tsan [build directory for thread sanitizer build]
Changes of note:
1. If using the blessed build layout, a build/latest symlink is created (or
replaced) on every build, pointing to that build's subdirectory.
2. To avoid the confusion of build output in build/latest/latest, binaries
go in bin/ and libraries in lib/. CMake is smart enough to make the
corresponding rpath changes to the binaries automatically.
3. Eclipse instructions note that the blessed layout doesn't work properly.
4. Various end-to-end scripts (make_site.sh, tpch.sh, etc.) use the
appropriate subdirectory in build/, and are more lenient w.r.t. reusing
existing build output.
5. Java unit tests require no hand-holding for the blessed layout.
6. Python unit tests require just KUDU_HOME for the blessed layout.
Change-Id: I61b4db5a87728d4f3ce9bc13f36d448dbd503c5c
Reviewed-on: http://gerrit.cloudera.org:8080/1914
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/f3f3996a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/f3f3996a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/f3f3996a
Branch: refs/heads/master
Commit: f3f3996a61494db9cff29dd2e84397478fb8edc3
Parents: 588abf6
Author: Adar Dembo <ad...@cloudera.com>
Authored: Tue Jan 26 16:53:01 2016 -0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Fri Jan 29 20:05:46 2016 +0000
----------------------------------------------------------------------
CMakeLists.txt | 40 +++++++++++++++-----
README.adoc | 38 +++++++++++--------
build-support/dist_test.py | 10 ++---
build-support/jenkins/build-and-test.sh | 11 +++---
build-support/jenkins/post-build-clean.sh | 4 +-
build-support/run_dist_test.py | 10 ++++-
docs/support/scripts/make_docs.sh | 2 +-
docs/support/scripts/make_site.sh | 11 ++++--
java/README.md | 2 +-
.../test/java/org/kududb/client/TestUtils.java | 2 +-
java/kudu-csd/generate_mdl.py | 4 +-
python/kudu/tests/common.py | 7 +++-
python/setup.py | 26 ++++++++++---
src/kudu/benchmarks/bin/parse_rpc_bench.sh | 2 +-
src/kudu/client/CMakeLists.txt | 2 +-
src/kudu/client/client_samples-test.sh | 4 +-
src/kudu/client/client_symbol-test.sh | 2 +-
src/kudu/codegen/CMakeLists.txt | 2 +-
.../integration-tests/external_mini_cluster.h | 2 +-
src/kudu/scripts/benchmarks.sh | 40 ++++++++++----------
src/kudu/scripts/tpch.sh | 24 ++++++------
src/kudu/util/CMakeLists.txt | 7 +++-
22 files changed, 157 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0ac59cf..ae0e328 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,6 +22,23 @@
# 1. https://cmake.org/Bug/view.php?id=14963
cmake_minimum_required(VERSION 3.2.0)
+# Provide a 'latest' symlink to this build directory if the "blessed"
+# multi-build layout is detected:
+#
+# build/
+# build/<first build directory>
+# build/<second build directory>
+# ...
+if ("${CMAKE_CURRENT_BINARY_DIR}" MATCHES "^${CMAKE_CURRENT_SOURCE_DIR}/build/[^/]+$")
+ if (NOT APPLE)
+ set(MORE_ARGS "-T")
+ endif()
+ add_custom_target(latest_symlink ALL
+ ln ${MORE_ARGS} -sf ${CMAKE_CURRENT_BINARY_DIR}
+ ${CMAKE_CURRENT_SOURCE_DIR}/build/latest
+ COMMENT "Recreating ../build/latest symlink")
+endif()
+
# TODO: can we somehow pass this into the java build?
file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/version.txt" KUDU_VERSION_NUMBER)
@@ -378,20 +395,17 @@ if ("${KUDU_LINK}" STREQUAL "d")
add_definitions(-fPIC)
endif()
-# set compile output directory
-set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/latest")
-file(MAKE_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
# where to put generated archives (.a files)
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib")
+file(MAKE_DIRECTORY "${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}")
# where to put generated libraries (.so files)
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib")
+file(MAKE_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
# where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(EXECUTABLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/bin")
+file(MAKE_DIRECTORY "${EXECUTABLE_OUTPUT_PATH}")
include_directories(${CMAKE_CURRENT_BINARY_DIR}/src)
include_directories(src)
@@ -915,6 +929,14 @@ if (UNIX)
endif (UNIX)
############################################################
+# "make site" target
+############################################################
+if (UNIX)
+ add_custom_target(site
+ ${CMAKE_CURRENT_SOURCE_DIR}/docs/support/scripts/make_site.sh)
+endif (UNIX)
+
+############################################################
# Subdirectories
############################################################
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/README.adoc
----------------------------------------------------------------------
diff --git a/README.adoc b/README.adoc
index 0478a26..1a163a4 100644
--- a/README.adoc
+++ b/README.adoc
@@ -27,7 +27,9 @@ A single Kudu source tree may be used for multiple builds, each with its
own build directory. Build directories may be placed anywhere in the
filesystem with the exception of the root directory of the source tree. The
Kudu build is invoked with a working directory of the build directory
-itself, so you must ensure it exists (i.e. create it with _mkdir -p_).
+itself, so you must ensure it exists (i.e. create it with _mkdir -p_). It's
+recommended to place all build directories within the _build_ subdirectory;
+_build/latest_ will be symlinked to most recently created one.
The rest of this document assumes the build directory
_<root directory of kudu source tree>/build/debug_.
@@ -66,7 +68,7 @@ $ make -j8 # or whatever level of parallelism your machine can handle
----
The build artifacts, including the test binaries, will be stored in
-_build/debug/latest/_.
+_build/debug/bin/_.
To omit the Kudu unit tests during the build, add -DNO_TESTS=1 to the
invocation of cmake. For example:
@@ -92,14 +94,14 @@ This command will report any tests that failed, and the test logs will be
written to _build/debug/test-logs_.
Individual tests can be run by directly invoking the test binaries in
-_build/debug/latest_. Since Kudu uses the Google C++ Test Framework (gtest),
+_build/debug/bin_. Since Kudu uses the Google C++ Test Framework (gtest),
specific test cases can be run with gtest flags:
[source,bash]
----
# List all the tests within a test binary, then run a single test
-$ build/debug/latest/tablet-test --gtest_list_tests
-$ build/debug/latest/tablet-test --gtest_filter=TestTablet/9.TestFlush
+$ build/debug/bin/tablet-test --gtest_list_tests
+$ build/debug/bin/tablet-test --gtest_filter=TestTablet/9.TestFlush
----
gtest also allows more complex filtering patterns. See the upstream
@@ -204,7 +206,7 @@ $ ctest -j8
[NOTE]
====
Note that we rely on a list of runtime suppressions in _build-support/tsan-suppressions.txt_.
-If you simply run a unit test like _build/tsan/latest/foo-test_, you won't get these suppressions.
+If you simply run a unit test like _build/tsan/bin/foo-test_, you won't get these suppressions.
Instead, use a command like:
[source,bash]
@@ -216,16 +218,12 @@ or
[source,bash]
----
-$ build-support/run-test.sh build/tsan/latest/foo-test [--test-arguments-here]
+$ build-support/run-test.sh build/tsan/bin/foo-test [--test-arguments-here]
----
...and then view the logs in _build/tsan/test-logs/_
-In order for all of the suppressions to work, you need libraries with debug
-symbols installed, particularly for libstdc\+\+. On Ubuntu 13.10, the package
-libstdc++6-4.8-dbg is needed for TSAN builds to pass. It's not a bad idea to
-install debug symbol packages for libboost, libc, and cyrus-sasl as well.
====
TSAN may truncate a few lines of the stack trace when reporting where the error
@@ -377,17 +375,22 @@ Eclipse can be used as an IDE for Kudu. To generate Eclipse project files, run:
[source,bash]
----
+$ mkdir -p <sibling directory to source tree>
+$ cd <sibling directory to source tree>
$ rm -rf CMakeCache.txt CMakeFiles/
-$ cmake -G "Eclipse CDT4 - Unix Makefiles" -DCMAKE_CXX_COMPILER_ARG1=-std=c++11 ../..
+$ cmake -G "Eclipse CDT4 - Unix Makefiles" -DCMAKE_CXX_COMPILER_ARG1=-std=c++11 <source tree>
----
+When the Eclipse generator is run in a subdirectory of the source tree, the
+resulting project is incomplete. That's why it's recommended to use a directory
+that's a sibling to the source tree. See [1] for more details.
+
It's critical that _CMakeCache.txt_ be removed prior to running the generator,
otherwise the extra Eclipse generator logic (the CMakeFindEclipseCDT4.make module)
won't run and standard system includes will be missing from the generated project.
-Thanks to http://public.kitware.com/Bug/view.php?id=15102, the Eclipse generator
-ignores the -std=c++11 definition and we must add it manually on the command line
-via CMAKE_CXX_COMPILER_ARG1.
+Thanks to [2], the Eclipse generator ignores the `-std=c++11` definition and we must
+add it manually on the command line via `CMAKE_CXX_COMPILER_ARG1`.
By default, the Eclipse CDT indexer will index everything under the _kudu/_
source tree. It tends to choke on certain complicated source files within
@@ -401,7 +404,7 @@ the dialog box, select "C/C++ Project Paths", select the Source tab, highlight
"Exclusion filter: (None)", and click "Edit...". In the new dialog box, click
"Add Multiple...". Select every subdirectory inside _thirdparty_ except _installed_
and _installed-deps_. Click OK all the way out and rebuild the project index by
-right clicking the project in the Project Explorer and selecting Index --> Rebuild.
+right clicking the project in the Project Explorer and selecting Index -> Rebuild.
With this exclusion, the only false positives (shown as "red squigglies") that
CDT presents appear to be in atomicops functions (`NoBarrier_CompareAndSwap` for
@@ -413,3 +416,6 @@ but one side effect is that nearly every source file appears in the indexer twic
once via a target and once via the raw source file. To fix this, simply delete the
[Targets] linked resource via the Project Explorer. Doing this should have no effect
on writing code, though it may affect your ability to build from within Eclipse.
+
+1. https://cmake.org/pipermail/cmake-developers/2011-November/014153.html
+2. http://public.kitware.com/Bug/view.php?id=15102
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/build-support/dist_test.py
----------------------------------------------------------------------
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index ceb37b7..2f53293 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -39,7 +39,7 @@ import subprocess
import time
TEST_TIMEOUT_SECS = int(os.environ.get('TEST_TIMEOUT_SECS', '900'))
-ARTIFACT_ARCHIVE_GLOBS = ["build/test-logs/*"]
+ARTIFACT_ARCHIVE_GLOBS = ["build/*/test-logs/*"]
ISOLATE_SERVER = os.environ.get('ISOLATE_SERVER',
"http://isolate.cloudera.org:4242/")
DIST_TEST_HOME = os.environ.get('DIST_TEST_HOME',
@@ -67,9 +67,9 @@ DEPS_FOR_ALL = \
# Tests that use the external minicluster require these.
# TODO: declare these dependencies per-test.
- "build/latest/kudu-tserver",
- "build/latest/kudu-master",
- "build/latest/kudu-ts-cli",
+ "build/latest/bin/kudu-tserver",
+ "build/latest/bin/kudu-master",
+ "build/latest/bin/kudu-ts-cli",
# parser-test requires these data files.
# TODO: again, we should do this with some per-test metadata file.
@@ -78,7 +78,7 @@ DEPS_FOR_ALL = \
#".../example-tweets.txt",
# Tests that require tooling require these.
- "build/latest/kudu-admin",
+ "build/latest/bin/kudu-admin",
]
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index abc1f48..1b30682 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -85,6 +85,7 @@ ulimit -c unlimited
BUILD_TYPE=${BUILD_TYPE:-DEBUG}
BUILD_TYPE=$(echo "$BUILD_TYPE" | tr a-z A-Z) # capitalize
+BUILD_TYPE_LOWER=$(echo "$BUILD_TYPE" | tr A-Z a-z)
# Set up defaults for environment variables.
DEFAULT_ALLOW_SLOW_TESTS=1
@@ -112,7 +113,7 @@ if [ ! -w "$TEST_TMPDIR" ]; then
fi
SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
-BUILD_ROOT=$SOURCE_ROOT/build
+BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER
# Remove testing artifacts from the previous run before we do anything
# else. Otherwise, if we fail during the "build" step, Jenkins will
@@ -296,8 +297,8 @@ if [ "$DO_COVERAGE" == "1" ]; then
echo
echo Generating coverage report...
echo ------------------------------------------------------------
- if ! ./thirdparty/gcovr-3.0/scripts/gcovr -r . -e '.*\.pb\..*' --xml \
- > build/coverage.xml ; then
+ if ! ./thirdparty/gcovr-3.0/scripts/gcovr -r $SOURCE_ROOT --xml \
+ > $BUILD_ROOT/coverage.xml ; then
EXIT_STATUS=1
FAILURES="$FAILURES"$'Coverage report failed\n'
fi
@@ -389,7 +390,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then
} else {
print "unknown_shard";
}')
- for log_file in $arch_dir/build/test-logs/* ; do
+ for log_file in $arch_dir/build/$BUILD_TYPE_LOWER/test-logs/* ; do
mv $log_file $TEST_LOGDIR/${shard_idx}_$(basename $log_file)
done
rm -Rf $arch_dir
@@ -400,7 +401,7 @@ if [ "$HEAPCHECK" = normal ]; then
echo
echo Checking that heap checker ran correctly
echo ------------------------------------------------------------
- FAILED_TESTS=$(zgrep -L -- "WARNING: Perftools heap leak checker is active -- Performance may suffer" build/test-logs/*-test.txt*)
+ FAILED_TESTS=$(zgrep -L -- "WARNING: Perftools heap leak checker is active -- Performance may suffer" $BUILD_ROOT/test-logs/*-test.txt*)
if [ -n "$FAILED_TESTS" ]; then
echo "Some tests didn't heap check properly:"
for FTEST in $FAILED_TESTS; do
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/build-support/jenkins/post-build-clean.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/post-build-clean.sh b/build-support/jenkins/post-build-clean.sh
index aad62d6..def3e21 100755
--- a/build-support/jenkins/post-build-clean.sh
+++ b/build-support/jenkins/post-build-clean.sh
@@ -32,10 +32,10 @@ cd $ROOT
# up partially compiling, etc.
# Clean up intermediate object files in the src tree
-find src -name \*.o -exec rm -f {} \;
+find build/latest/src -name \*.o -exec rm -f {} \;
# Clean up the actual build artifacts
-rm -Rf build/latest/
+rm -Rf build/latest/bin build/latest/lib
# Clean up any java build artifacts
find java -name \*.jar -delete -o -name \*.class -delete
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/build-support/run_dist_test.py
----------------------------------------------------------------------
diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py
index 7e25072..71c0950 100755
--- a/build-support/run_dist_test.py
+++ b/build-support/run_dist_test.py
@@ -100,6 +100,7 @@ def main():
sys.exit(1)
test_exe = args[0]
test_name, _ = os.path.splitext(os.path.basename(test_exe))
+ test_dir = os.path.dirname(test_exe)
env = os.environ.copy()
for env_pair in options.env:
@@ -120,8 +121,13 @@ def main():
env['LD_LIBRARY_PATH'] = ":".join(
[os.path.join(ROOT, "build/dist-test-system-libs/"),
- os.path.abspath(os.path.dirname(test_exe))])
- env['GTEST_OUTPUT'] = 'xml:' + os.path.join(ROOT, 'build/test-logs/')
+ os.path.abspath(os.path.join(test_dir, "..", "lib"))])
+
+ # GTEST_OUTPUT must be canonicalized and have a trailing slash for gtest to
+ # properly interpret it as a directory.
+ env['GTEST_OUTPUT'] = 'xml:' + os.path.abspath(
+ os.path.join(test_dir, "..", "test-logs")) + '/'
+
env['ASAN_SYMBOLIZER_PATH'] = os.path.join(ROOT, "thirdparty/installed/bin/llvm-symbolizer")
rc = subprocess.call([os.path.join(ROOT, "build-support/run-test.sh")] + args,
env=env)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/docs/support/scripts/make_docs.sh
----------------------------------------------------------------------
diff --git a/docs/support/scripts/make_docs.sh b/docs/support/scripts/make_docs.sh
index 33f715d..8c94804 100755
--- a/docs/support/scripts/make_docs.sh
+++ b/docs/support/scripts/make_docs.sh
@@ -140,7 +140,7 @@ for binary in ${binaries[@]}; do
# Create the XML file.
# This command exits with a nonzero value.
- $BUILD_ROOT/latest/$binary --helpxml > ${GEN_DOC_DIR}/$(basename $binary).xml || true
+ $BUILD_ROOT/bin/$binary --helpxml > ${GEN_DOC_DIR}/$(basename $binary).xml || true
)
# Create the supported config reference
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/docs/support/scripts/make_site.sh
----------------------------------------------------------------------
diff --git a/docs/support/scripts/make_site.sh b/docs/support/scripts/make_site.sh
index e51b6f9..486ce11 100755
--- a/docs/support/scripts/make_site.sh
+++ b/docs/support/scripts/make_site.sh
@@ -24,8 +24,9 @@
########################################################################
set -e
+BUILD_TYPE=release
SOURCE_ROOT=$(cd $(dirname $0)/../../..; pwd)
-BUILD_ROOT="$SOURCE_ROOT/build"
+BUILD_ROOT="$SOURCE_ROOT/build/$BUILD_TYPE"
SITE_OUTPUT_DIR="$BUILD_ROOT/site"
set -x
@@ -36,10 +37,14 @@ $SOURCE_ROOT/build-support/enable_devtoolset.sh $SOURCE_ROOT/thirdparty/build-if
echo "Successfully built third-party dependencies."
# Build the binaries so we can auto-generate the command-line references
-rm -rf "$BUILD_ROOT"
mkdir -p "$BUILD_ROOT"
cd "$BUILD_ROOT"
-$SOURCE_ROOT/build-support/enable_devtoolset.sh $SOURCE_ROOT/thirdparty/installed/bin/cmake -DNO_TESTS=1 $SOURCE_ROOT
+rm -rf CMakeCache CMakeFiles/
+$SOURCE_ROOT/build-support/enable_devtoolset.sh \
+ $SOURCE_ROOT/thirdparty/installed/bin/cmake \
+ -DNO_TESTS=1 \
+ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+ $SOURCE_ROOT
make -j$(getconf _NPROCESSORS_ONLN)
# Check out the gh-pages repo into $SITE_OUTPUT_DIR
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/java/README.md
----------------------------------------------------------------------
diff --git a/java/README.md b/java/README.md
index 4732d42..326c29b 100644
--- a/java/README.md
+++ b/java/README.md
@@ -58,7 +58,7 @@ is also possible to provide your own.
By default, the unit tests will start a master and a tablet
server using the flags file located in the src/test/resources/
directory. The tests will locate the master and tablet server
-binaries by looking in 'build/latest/' from the root of
+binaries by looking in 'build/latest/bin' from the root of
the git repository. If you have recently built the C++ code
for Kudu, those should be present already.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java b/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
index f4866b8..c40cda4 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
@@ -96,7 +96,7 @@ public class TestUtils {
File myPath = new File(urlToPath(myUrl));
while (myPath != null) {
if (new File(myPath, ".git").isDirectory()) {
- return new File(myPath, "build/latest").getAbsolutePath();
+ return new File(myPath, "build/latest/bin").getAbsolutePath();
}
myPath = myPath.getParentFile();
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/java/kudu-csd/generate_mdl.py
----------------------------------------------------------------------
diff --git a/java/kudu-csd/generate_mdl.py b/java/kudu-csd/generate_mdl.py
index 28e8a49..2d34aa1 100755
--- a/java/kudu-csd/generate_mdl.py
+++ b/java/kudu-csd/generate_mdl.py
@@ -21,7 +21,7 @@
# metadata dumped by our daemon processes.
#
# Requires that the daemon processes have already been built and available
-# in the build/latest directory.
+# in the build/latest/bin directory.
#
# Outputs the MDL file on stdout by default or to a file specified in the first
# argument.
@@ -37,7 +37,7 @@ import sys
BINARIES=["kudu-master", "kudu-tserver"]
-RELATIVE_BUILD_DIR="../../build/latest"
+RELATIVE_BUILD_DIR="../../build/latest/bin"
def find_binary(bin_name):
dirname, _ = os.path.split(os.path.abspath(__file__))
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/python/kudu/tests/common.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/common.py b/python/kudu/tests/common.py
index 07d97e2..988b7d7 100644
--- a/python/kudu/tests/common.py
+++ b/python/kudu/tests/common.py
@@ -41,8 +41,11 @@ class KuduTestBase(object):
@classmethod
def start_cluster(cls):
- local_path = tempfile.mkdtemp(dir=os.getenv("TEST_TMPDIR", None))
- bin_path = "{0}/latest".format(os.getenv("KUDU_BUILD"))
+ local_path = tempfile.mkdtemp(dir=os.getenv("TEST_TMPDIR"))
+ kudu_build = os.getenv("KUDU_BUILD")
+ if not kudu_build:
+ kudu_build = os.path.join(os.getenv("KUDU_HOME"), "build", "latest")
+ bin_path = "{0}/bin".format(kudu_build)
os.makedirs("{0}/master/".format(local_path))
os.makedirs("{0}/master/data".format(local_path))
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 2c076b2..1b250f3 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -69,11 +69,27 @@ class clean(_clean):
# If we're in the context of the Kudu git repository, build against the
# latest in-tree build artifacts
-if 'KUDU_HOME' in os.environ and 'KUDU_BUILD' in os.environ:
- sys.stderr.write("Building from in-tree build artifacts\n")
- kudu_include_dirs = [os.path.join(os.environ['KUDU_HOME'], 'src')]
- kudu_include_dirs.append(os.path.join(os.environ['KUDU_BUILD'], 'src'))
- kudu_lib_dir = os.path.join(os.environ['KUDU_BUILD'], 'latest/exported')
+if 'KUDU_HOME' in os.environ:
+ kudu_home = os.environ['KUDU_HOME']
+ sys.stderr.write("Using KUDU_HOME directory: %s\n" % (kudu_home,))
+ if not os.path.isdir(kudu_home):
+ sys.stderr.write("%s is not a valid KUDU_HOME directory" % (kudu_home,))
+ sys.exit(1)
+
+ kudu_include_dirs = [os.path.join(kudu_home, 'src')]
+
+ if 'KUDU_BUILD' in os.environ:
+ kudu_build = os.environ['KUDU_BUILD']
+ sys.stderr.write("Using KUDU_BUILD directory: %s\n" % (kudu_build,))
+ else:
+ kudu_build = os.path.join(kudu_home, 'build', 'latest')
+ sys.stderr.write("Using inferred KUDU_BUILD directory: %s/\n" % (kudu_build,))
+ if not os.path.isdir(kudu_build):
+ sys.stderr.write("%s is not a valid KUDU_BUILD directory" % (kudu_build,))
+ sys.exit(1)
+
+ kudu_include_dirs.append(os.path.join(kudu_build, 'src'))
+ kudu_lib_dir = os.path.join(kudu_build, 'lib', 'exported')
else:
if os.path.exists("/usr/local/include/kudu"):
prefix = "/usr/local"
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/benchmarks/bin/parse_rpc_bench.sh
----------------------------------------------------------------------
diff --git a/src/kudu/benchmarks/bin/parse_rpc_bench.sh b/src/kudu/benchmarks/bin/parse_rpc_bench.sh
index 919f0de..63b1fda 100755
--- a/src/kudu/benchmarks/bin/parse_rpc_bench.sh
+++ b/src/kudu/benchmarks/bin/parse_rpc_bench.sh
@@ -28,7 +28,7 @@ if [[ -z $FILE || $FILE == "-h" || $FILE == "--help" ]]; then
echo
echo '$ cd $KUDU_HOME'
echo '$ BUILD_TYPE=RELEASE ./build-support/jenkins/build-and-test.sh'
- echo '$ KUDU_ALLOW_SLOW_TESTS=1 ./build/latest/rpc-bench --gtest_repeat=10 2>&1 | tee rpc-bench-output.log'
+ echo '$ KUDU_ALLOW_SLOW_TESTS=1 ./build/latest/bin/rpc-bench --gtest_repeat=10 2>&1 | tee rpc-bench-output.log'
echo '$ ./src/kudu/benchmarks/bin/parse_rpc_bench.sh rpc-bench-output.log'
echo
echo 'Example output:'
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/client/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/client/CMakeLists.txt b/src/kudu/client/CMakeLists.txt
index 74902a7..e5533f4 100644
--- a/src/kudu/client/CMakeLists.txt
+++ b/src/kudu/client/CMakeLists.txt
@@ -64,7 +64,7 @@ ADD_EXPORTABLE_LIBRARY(kudu_client
DEPS ${CLIENT_LIBS}
EXPORTED_SHARED
EXPORTED_OUTPUT_NAME kudu_client
- EXPORTED_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}/exported"
+ EXPORTED_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/exported"
EXPORTED_DEPS ${EXPORTED_CLIENT_LIBS})
################################################################################
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/client/client_samples-test.sh
----------------------------------------------------------------------
diff --git a/src/kudu/client/client_samples-test.sh b/src/kudu/client/client_samples-test.sh
index 6c8f5e7..4624196 100755
--- a/src/kudu/client/client_samples-test.sh
+++ b/src/kudu/client/client_samples-test.sh
@@ -76,8 +76,8 @@ done
# use the one from thirdparty.
CMAKE=$(which cmake || :)
if [ -z "$CMAKE" ]; then
- # TODO: temporary hack which assumes this script is in src/build/latest
- CMAKE=$OUTPUT_DIR/../../thirdparty/installed/bin/cmake
+ # TODO: temporary hack which assumes this script is in src/build/<type>/bin
+ CMAKE=$OUTPUT_DIR/../../../thirdparty/installed/bin/cmake
fi
# Build the client samples using the client library.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/client/client_symbol-test.sh
----------------------------------------------------------------------
diff --git a/src/kudu/client/client_symbol-test.sh b/src/kudu/client/client_symbol-test.sh
index 2975967..2b5d0bc 100755
--- a/src/kudu/client/client_symbol-test.sh
+++ b/src/kudu/client/client_symbol-test.sh
@@ -34,7 +34,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
DYLIB_SUFFIX="dylib"
fi
-LIB=$(dirname "$BASH_SOURCE")/exported/libkudu_client.$DYLIB_SUFFIX
+LIB=$(dirname "$BASH_SOURCE")/../lib/exported/libkudu_client.$DYLIB_SUFFIX
if [ -r $LIB ]; then
echo "Found kudu client library: $LIB"
else
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/codegen/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/codegen/CMakeLists.txt b/src/kudu/codegen/CMakeLists.txt
index a57c04c..f29a0d0 100644
--- a/src/kudu/codegen/CMakeLists.txt
+++ b/src/kudu/codegen/CMakeLists.txt
@@ -53,7 +53,7 @@ llvm_map_components_to_libnames(llvm_LIBRARIES "${LLVM_REQ_COMPONENTS}")
## Create .ll file for precompiled functions (and their dependencies)
set(CLANG_EXEC ${THIRDPARTY_PREFIX}/bin/clang++)
set(IR_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/precompiled.cc)
-set(IR_OUTPUT ${BUILD_OUTPUT_ROOT_DIRECTORY}/precompiled.ll)
+set(IR_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/precompiled.ll)
set(IR_OUTPUT_CC ${IR_OUTPUT}.cc)
# Retrieve all includes directories needed for precompilation
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/integration-tests/external_mini_cluster.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/external_mini_cluster.h b/src/kudu/integration-tests/external_mini_cluster.h
index e4bfadf..d19fc98 100644
--- a/src/kudu/integration-tests/external_mini_cluster.h
+++ b/src/kudu/integration-tests/external_mini_cluster.h
@@ -96,7 +96,7 @@ struct ExternalMiniClusterOptions {
// The path where the kudu daemons should be run from.
// Default: "", which uses the same path as the currently running executable.
- // This works for unit tests, since they all end up in build/latest/.
+ // This works for unit tests, since they all end up in build/latest/bin.
std::string daemon_bin_path;
// Extra flags for tablet servers and masters respectively.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/scripts/benchmarks.sh
----------------------------------------------------------------------
diff --git a/src/kudu/scripts/benchmarks.sh b/src/kudu/scripts/benchmarks.sh
index e5f6e27..f430d9e 100755
--- a/src/kudu/scripts/benchmarks.sh
+++ b/src/kudu/scripts/benchmarks.sh
@@ -57,8 +57,8 @@ SCAN_ALL_COMMITTED=ScanAllCommitted
FS_SCANINSERT_MRS=FullStackScanInsertMRSOnly
FS_SCANINSERT_DISK=FullStackScanInsertWithDisk
-LOG_DIR_NAME=build/bench-logs
-OUT_DIR_NAME=build/bench-out
+LOG_DIR_NAME=build/latest/bench-logs
+OUT_DIR_NAME=build/latest/bench-out
HTML_FILE="benchmarks.html"
# Most tests will run this many times.
@@ -151,16 +151,16 @@ build_kudu() {
THIRDPARTY_BIN=$BASE_DIR/thirdparty/installed/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof
- # Build Kudu
- rm -rf build
- mkdir -p build
- pushd build
-
BUILD_TYPE=release
# Workaround for gperftools issue #497
export LD_BIND_NOW=1
- $BASE_DIR/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ..
+ # Build Kudu
+ mkdir -p build/$BUILD_TYPE
+ pushd build/$BUILD_TYPE
+ rm -rf CMakeCache.txt CMakeFiles/
+
+ $BASE_DIR/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ../..
# clean up before we run
rm -Rf /tmp/kudutpch1-$UID
@@ -178,7 +178,7 @@ run_benchmarks() {
mkdir -p "$OUTDIR"
# run all of the variations of mt-tablet-test
- ./build/latest/mt-tablet-test \
+ ./build/latest/bin/mt-tablet-test \
--gtest_filter=\*DoTestAllAtOnce\* \
--num_counter_threads=0 \
--tablet_test_flush_threshold_mb=32 \
@@ -190,49 +190,49 @@ run_benchmarks() {
# run rpc-bench test 5 times. 10 seconds per run
for i in $(seq 1 $NUM_SAMPLES); do
- KUDU_ALLOW_SLOW_TESTS=true ./build/latest/rpc-bench &> $LOGDIR/$RPC_BENCH_TEST$i.log
+ KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/rpc-bench &> $LOGDIR/$RPC_BENCH_TEST$i.log
done
# run cbtree-test 5 times. 20 seconds per run
for i in $(seq 1 $NUM_SAMPLES); do
- KUDU_ALLOW_SLOW_TESTS=true ./build/latest/cbtree-test \
+ KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/cbtree-test \
--gtest_filter=TestCBTree.TestScanPerformance &> $LOGDIR/${CBTREE_TEST}$i.log
done
# run bloomfile-test 5 times. ~3.3 seconds per run
for i in $(seq 1 $NUM_SAMPLES); do
- ./build/latest/bloomfile-test --benchmark_queries=10000000 --bloom_size_bytes=32768 \
+ ./build/latest/bin/bloomfile-test --benchmark_queries=10000000 --bloom_size_bytes=32768 \
--n_keys=100000 --gtest_filter=*Benchmark &> $LOGDIR/$BLOOM_TEST$i.log
done
# run mt-bloomfile-test 5 times. 20-30 seconds per run.
# The block cache is set to 1MB to generate churn.
for i in $(seq 1 $NUM_SAMPLES); do
- ./build/latest/mt-bloomfile-test --benchmark_queries=2000000 --bloom_size_bytes=32768 \
+ ./build/latest/bin/mt-bloomfile-test --benchmark_queries=2000000 --bloom_size_bytes=32768 \
--n_keys=5000000 --block_cache_capacity_mb=1 &> $LOGDIR/$MT_BLOOM_TEST$i.log
done
# run wire_protocol-test 5 times. 6 seconds per run
for i in $(seq 1 $NUM_SAMPLES); do
- KUDU_ALLOW_SLOW_TESTS=true ./build/latest/wire_protocol-test --gtest_filter=*Benchmark \
+ KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/wire_protocol-test --gtest_filter=*Benchmark \
&> $LOGDIR/$WIRE_PROTOCOL_TEST$i.log
done
# run compaction-test 5 times, 6 seconds each
for i in $(seq 1 $NUM_SAMPLES); do
- KUDU_ALLOW_SLOW_TESTS=true ./build/latest/compaction-test \
+ KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/compaction-test \
--gtest_filter=TestCompaction.BenchmarkMerge* &> $LOGDIR/${COMPACT_MERGE_BENCH}$i.log
done
# run memrowset benchmark 5 times, ~10 seconds per run
for i in $(seq 1 $NUM_SAMPLES) ; do
- ./build/latest/memrowset-test --roundtrip_num_rows=10000000 \
+ ./build/latest/bin/memrowset-test --roundtrip_num_rows=10000000 \
--gtest_filter=\*InsertCount\* &> $LOGDIR/${MEMROWSET_BENCH}$i.log
done
# Run single-threaded TS insert latency benchmark, 5-6 seconds per run
for i in $(seq 1 $NUM_SAMPLES) ; do
- KUDU_ALLOW_SLOW_TESTS=true ./build/latest/tablet_server-test \
+ KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/tablet_server-test \
--gtest_filter=*MicroBench* \
--single_threaded_insert_latency_bench_warmup_rows=1000 \
--single_threaded_insert_latency_bench_insert_rows=10000 &> $LOGDIR/${TS_INSERT_LATENCY}$i.log
@@ -240,13 +240,13 @@ run_benchmarks() {
# Run multi-threaded TS insert benchmark
for i in $(seq 1 $NUM_SAMPLES) ; do
- KUDU_ALLOW_SLOW_TESTS=1 build/latest/tablet_server-stress-test \
+ KUDU_ALLOW_SLOW_TESTS=1 build/latest/bin/tablet_server-stress-test \
--num_inserts_per_thread=30000 &> $LOGDIR/${TS_8THREAD_BENCH}$i.log
done
# Run full stack scan/insert test using MRS only, ~26s each
for i in $(seq 1 $NUM_SAMPLES) ; do
- ./build/latest/full_stack-insert-scan-test \
+ ./build/latest/bin/full_stack-insert-scan-test \
--gtest_filter=FullStackInsertScanTest.MRSOnlyStressTest \
--concurrent_inserts=50 \
--inserts_per_client=200000 \
@@ -256,7 +256,7 @@ run_benchmarks() {
# Run full stack scan/insert test with disk, ~50s each
for i in $(seq 1 $NUM_SAMPLES) ; do
- ./build/latest/full_stack-insert-scan-test \
+ ./build/latest/bin/full_stack-insert-scan-test \
--gtest_filter=FullStackInsertScanTest.WithDiskStressTest \
--concurrent_inserts=50 \
--inserts_per_client=200000 \
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/scripts/tpch.sh
----------------------------------------------------------------------
diff --git a/src/kudu/scripts/tpch.sh b/src/kudu/scripts/tpch.sh
index 10f7173..bbad1ee 100755
--- a/src/kudu/scripts/tpch.sh
+++ b/src/kudu/scripts/tpch.sh
@@ -40,7 +40,6 @@
# Constants
##########################################################
ROOT=$(readlink -f $(dirname $0)/../../..)
-OUTDIR=$ROOT/build/tpch
##########################################################
# Overridable params
@@ -106,16 +105,16 @@ $ROOT/build-support/enable_devtoolset.sh $ROOT/thirdparty/build-if-necessary.sh
THIRDPARTY_BIN=$(pwd)/thirdparty/installed/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof
-# Build Kudu
-rm -rf build
-mkdir -p build
-pushd build
-
BUILD_TYPE=release
# Workaround for gperftools issue #497
export LD_BIND_NOW=1
-$ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ..
+# Build Kudu
+mkdir -p build/$BUILD_TYPE
+pushd build/$BUILD_TYPE
+rm -rf CMakeCache CMakeFiles/
+
+$ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ../..
NUM_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
make -j${NUM_PROCS} tpch1 2>&1 | tee build.log
@@ -125,14 +124,15 @@ popd
cat $LINEITEM_TBL_PATH > /dev/null
cat $LINEITEM_TBL_PATH > /dev/null
+OUTDIR=$ROOT/build/$BUILD_TYPE/tpch
rm -Rf $KUDU_DATA_DIR # Clean up data dir.
mkdir -p $OUTDIR # Create log file output dir.
-./build/latest/tpch1 -logtostderr=1 \
- -tpch_path_to_data=$LINEITEM_TBL_PATH \
- -mini_cluster_base_dir=$KUDU_DATA_DIR \
- -tpch_num_query_iterations=$TPCH_NUM_QUERY_ITERS \
- >$OUTDIR/benchmark.log 2>&1
+./build/latest/bin/tpch1 -logtostderr=1 \
+ -tpch_path_to_data=$LINEITEM_TBL_PATH \
+ -mini_cluster_base_dir=$KUDU_DATA_DIR \
+ -tpch_num_query_iterations=$TPCH_NUM_QUERY_ITERS \
+ >$OUTDIR/benchmark.log 2>&1
cat $OUTDIR/benchmark.log
INSERT_TIME=$(grep "Time spent loading" $OUTDIR/benchmark.log | \
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/f3f3996a/src/kudu/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt
index 4da9eca..b98c464 100644
--- a/src/kudu/util/CMakeLists.txt
+++ b/src/kudu/util/CMakeLists.txt
@@ -60,8 +60,11 @@ ADD_EXPORTABLE_LIBRARY(version_info_proto
############################################################
# Version stamp
############################################################
-include_directories(${BUILD_OUTPUT_ROOT_DIRECTORY})
-set(VERSION_STAMP_FILE ${BUILD_OUTPUT_ROOT_DIRECTORY}/kudu/generated/version_defines.h)
+
+# Unlike CMAKE_CURRENT_BINARY_DIR, CMAKE_BINARY_DIR is always the root of
+# the build directory.
+set(VERSION_STAMP_FILE ${CMAKE_BINARY_DIR}/src/kudu/generated/version_defines.h)
+
list(APPEND GEN_VERSION_INFO_COMMAND "${BUILD_SUPPORT_DIR}/gen_version_info.py")
list(APPEND GEN_VERSION_INFO_COMMAND "--version=${KUDU_VERSION_NUMBER}")
list(APPEND GEN_VERSION_INFO_COMMAND "--build-type=${CMAKE_BUILD_TYPE}")