You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2019/07/22 19:11:09 UTC
[kudu] branch master updated: KUDU-2823 [java client] Support
setting dimension for the newly created tablet
This is an automated email from the ASF dual-hosted git repository.
adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new c6fa433 KUDU-2823 [java client] Support setting dimension for the newly created tablet
c6fa433 is described below
commit c6fa43362f79b4adc1b489d04d5b0820b44b68ff
Author: oclarms <oc...@gmail.com>
AuthorDate: Wed Jul 17 17:38:13 2019 +0800
KUDU-2823 [java client] Support setting dimension for the newly created tablet
Change-Id: I5fc7797011df578000544147938bd15a6027f4c8
Reviewed-on: http://gerrit.cloudera.org:8080/13875
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
---
.../org/apache/kudu/client/AlterTableOptions.java | 40 ++++++++++++++++++++++
.../org/apache/kudu/client/AsyncKuduClient.java | 3 ++
.../org/apache/kudu/client/CreateTableOptions.java | 19 ++++++++++
.../java/org/apache/kudu/client/LocatedTablet.java | 4 +++
.../java/org/apache/kudu/client/TestKuduTable.java | 33 ++++++++++++++++++
src/kudu/master/catalog_manager.cc | 7 ++++
src/kudu/master/master.proto | 6 +++-
7 files changed, 111 insertions(+), 1 deletion(-)
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
index b4fbbab..8071257 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
@@ -283,6 +283,43 @@ public class AlterTableOptions {
PartialRow upperBound,
RangePartitionBound lowerBoundType,
RangePartitionBound upperBoundType) {
+ return addRangePartition(lowerBound, upperBound, null, lowerBoundType, upperBoundType);
+ }
+
+ /**
+ * Add a range partition to the table with dimension label.
+ *
+ * If either row is empty, then that end of the range will be unbounded. If a range column is
+ * missing a value, the logical minimum value for that column type will be used as the default.
+ *
+ * Multiple range partitions may be added as part of a single alter table transaction by calling
+ * this method multiple times. Added range partitions must not overlap with each
+ * other or any existing range partitions (unless the existing range partitions are dropped as
+ * part of the alter transaction first). The lower bound must be less than the upper bound.
+ *
+ * This client will immediately be able to write and scan the new tablets when the alter table
+ * operation returns success, however other existing clients may have to wait for a timeout period
+ * to elapse before the tablets become visible. This period is configured by the master's
+ * 'table_locations_ttl_ms' flag, and defaults to 5 minutes.
+ *
+ * By default, the master will try to place newly created tablet replicas on tablet
+ * servers with a small number of tablet replicas. If the dimension label is provided,
+ * newly created replicas will be evenly distributed in the cluster based on the dimension
+ * label. In other words, the master will try to place newly created tablet replicas on
+ * tablet servers with a small number of tablet replicas belonging to this dimension label.
+ *
+ * @param lowerBound lower bound, may be empty but not null
+ * @param upperBound upper bound, may be empty but not null
+ * @param dimensionLabel the dimension label for the tablet to be created
+ * @param lowerBoundType the type of the lower bound, either inclusive or exclusive
+ * @param upperBoundType the type of the upper bound, either inclusive or exclusive
+ * @return this instance
+ */
+ public AlterTableOptions addRangePartition(PartialRow lowerBound,
+ PartialRow upperBound,
+ String dimensionLabel,
+ RangePartitionBound lowerBoundType,
+ RangePartitionBound upperBoundType) {
Preconditions.checkNotNull(lowerBound);
Preconditions.checkNotNull(upperBound);
Preconditions.checkArgument(lowerBound.getSchema().equals(upperBound.getSchema()));
@@ -294,6 +331,9 @@ public class AlterTableOptions {
builder.setRangeBounds(
new Operation.OperationsEncoder()
.encodeLowerAndUpperBounds(lowerBound, upperBound, lowerBoundType, upperBoundType));
+ if (dimensionLabel != null) {
+ builder.setDimensionLabel(dimensionLabel);
+ }
step.setAddRangePartition(builder);
if (!pb.hasSchema()) {
pb.setSchema(ProtobufHelper.schemaToPb(lowerBound.getSchema(),
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index 1de7306..d908692 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -2303,6 +2303,9 @@ public class AsyncKuduClient implements AutoCloseable {
Master.TabletLocationsPB.ReplicaPB.Builder builder = Master.TabletLocationsPB.ReplicaPB.newBuilder();
builder.setRole(replica.getRole());
builder.setTsInfo(tsInfo);
+ if (replica.hasDimensionLabel()) {
+ builder.setDimensionLabel(replica.getDimensionLabel());
+ }
replicas.add(builder.build());
}
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
index 1790037..9372ad9 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
@@ -192,6 +192,25 @@ public class CreateTableOptions {
}
/**
+ * Sets the dimension label for all tablets created at table creation time.
+ *
+ * By default, the master will try to place newly created tablet replicas on tablet
+ * servers with a small number of tablet replicas. If the dimension label is provided,
+ * newly created replicas will be evenly distributed in the cluster based on the dimension
+ * label. In other words, the master will try to place newly created tablet replicas on
+ * tablet servers with a small number of tablet replicas belonging to this dimension label.
+ *
+ * @param dimensionLabel the dimension label for the tablet to be created.
+ * @return this instance
+ */
+ public CreateTableOptions setDimensionLabel(String dimensionLabel) {
+ Preconditions.checkArgument(dimensionLabel != null,
+ "dimension label must not be null");
+ pb.setDimensionLabel(dimensionLabel);
+ return this;
+ }
+
+ /**
* Sets the table's extra configuration properties.
*
* If the value of the kv pair is empty, the property will be ignored.
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
index ae31903..837156e 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
@@ -137,6 +137,10 @@ public class LocatedTablet {
return pb.getRole().toString();
}
+ public String getDimensionLabel() {
+ return pb.hasDimensionLabel() ? pb.getDimensionLabel() : null;
+ }
+
@Override
public String toString() {
return pb.toString();
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
index d1332e3..064b3cc 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
@@ -30,7 +30,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -746,4 +748,35 @@ public class TestKuduTable {
assertEquals("wrong value comment post alter", "",
table2.getSchema().getColumn("value").getComment());
}
+
+ @Test(timeout = 100000)
+ public void testDimensionLabel() throws Exception {
+ // Create a table with dimension label.
+ KuduTable table = client.createTable(tableName, basicSchema,
+ getBasicTableOptionsWithNonCoveredRange().setDimensionLabel("labelA"));
+
+ // Add a range partition to the table with dimension label.
+ AlterTableOptions ato = new AlterTableOptions();
+ PartialRow bLowerBound = BASIC_SCHEMA.newPartialRow();
+ bLowerBound.addInt("key", 300);
+ PartialRow bUpperBound = BASIC_SCHEMA.newPartialRow();
+ bUpperBound.addInt("key", 400);
+ ato.addRangePartition(bLowerBound, bUpperBound, "labelB",
+ RangePartitionBound.INCLUSIVE_BOUND,
+ RangePartitionBound.EXCLUSIVE_BOUND);
+ client.alterTable(tableName, ato);
+
+ Map<String, Integer> dimensionMap = new HashMap<>();
+ for (LocatedTablet tablet : table.getTabletsLocations(DEFAULT_SLEEP)) {
+ for (LocatedTablet.Replica replica : tablet.getReplicas()) {
+ Integer number = dimensionMap.get(replica.getDimensionLabel());
+ if (number == null) {
+ number = 0;
+ }
+ dimensionMap.put(replica.getDimensionLabel(), number + 1);
+ }
+ }
+ assertEquals(9, dimensionMap.get("labelA").intValue());
+ assertEquals(3, dimensionMap.get("labelB").intValue());
+ }
}
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 1ea7408..46ebf25 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -4763,6 +4763,10 @@ Status CatalogManager::BuildLocationsForTablet(
};
auto role = GetParticipantRole(peer, cstate);
+ optional<string> dimension = none;
+ if (l_tablet.data().pb.has_dimension_label()) {
+ dimension = l_tablet.data().pb.dimension_label();
+ }
if (ts_infos_dict) {
int idx = *ComputeIfAbsent(
&ts_infos_dict->uuid_to_idx, peer.permanent_uuid(),
@@ -4776,6 +4780,9 @@ Status CatalogManager::BuildLocationsForTablet(
auto* interned_replica_pb = locs_pb->add_interned_replicas();
interned_replica_pb->set_ts_info_idx(idx);
interned_replica_pb->set_role(role);
+ if (dimension) {
+ interned_replica_pb->set_dimension_label(*dimension);
+ }
} else {
TabletLocationsPB_ReplicaPB* replica_pb = locs_pb->add_replicas();
replica_pb->set_allocated_ts_info(make_tsinfo_pb().release());
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index c74f3f1..448e074 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -368,16 +368,20 @@ message TSHeartbeatResponsePB {
//////////////////////////////
message TabletLocationsPB {
- // DEPRECATED: new clients should prefer the 'Interned' type below.
+ // DEPRECATED.
+ // TODO: new clients should prefer the 'Interned' type below.
+ // Remove 'ReplicaPB' when we stop using it internally.
message ReplicaPB {
required TSInfoPB ts_info = 1;
required consensus.RaftPeerPB.Role role = 2;
+ optional string dimension_label = 3;
}
message InternedReplicaPB {
// Index into the 'ts_infos' list in the top-level RPC response.
required uint32 ts_info_idx = 1;
required consensus.RaftPeerPB.Role role = 2;
+ optional string dimension_label = 3;
}
required bytes tablet_id = 1;