You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/04/03 17:13:22 UTC
[2/3] kudu git commit: [examples] Improve loadgen example
[examples] Improve loadgen example
This is a grab bag of improvements to the loadgen example:
- Updated pom
- README rewritten, in adoc format.
- Small improvements to the Java code.
Change-Id: Ibc0e655f56adcedec2c3dd8a8b3c900e3b2e7803
Reviewed-on: http://gerrit.cloudera.org:8080/9853
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Will Berkeley <wd...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/cc8c5e3e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/cc8c5e3e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/cc8c5e3e
Branch: refs/heads/master
Commit: cc8c5e3e8783c81f10233d531a361723fdceb3f9
Parents: 0804136
Author: Will Berkeley <wd...@apache.org>
Authored: Wed Mar 28 21:56:08 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Tue Apr 3 04:29:44 2018 +0000
----------------------------------------------------------------------
examples/java/insert-loadgen/README | 17 ---
examples/java/insert-loadgen/README.adoc | 45 +++++++
examples/java/insert-loadgen/pom.xml | 49 ++++---
.../org/apache/kudu/examples/InsertLoadgen.java | 132 +++++++++++++++++++
.../kududb/examples/loadgen/InsertLoadgen.java | 110 ----------------
5 files changed, 205 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/README
----------------------------------------------------------------------
diff --git a/examples/java/insert-loadgen/README b/examples/java/insert-loadgen/README
deleted file mode 100644
index 7c0628e..0000000
--- a/examples/java/insert-loadgen/README
+++ /dev/null
@@ -1,17 +0,0 @@
-Random insert load generator. This will insert as fast as it can using
-AUTO_BACKGROUND_FLUSH with the specified number of threads. All fields are
-randomized, and insert failures (including errors from collisions) are ignored.
-
-To build and run, do the following:
-
-$ mvn package
-$ java -jar target/kudu-insert-loadgen-0.1-SNAPSHOT.jar kudu_master_host kudu_table_name num_threads
-
-For example, if you are running the Quickstart VM with the host name
-"quickstart.cloudera", then you can use:
-
-$ java -jar target/kudu-insert-loadgen-0.1-SNAPSHOT.jar quickstart.cloudera test_table 16
-
-Note: This program will not create the "test_table" table. You must do that
-via other means, such as through impala-shell or using the create-demo-table
-program included in the Kudu source tree.
http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/README.adoc
----------------------------------------------------------------------
diff --git a/examples/java/insert-loadgen/README.adoc b/examples/java/insert-loadgen/README.adoc
new file mode 100644
index 0000000..e2e697f
--- /dev/null
+++ b/examples/java/insert-loadgen/README.adoc
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+= Kudu Java insert loadgen README
+:author: Kudu Team
+:homepage: https://kudu.apache.org/
+
+== Summary
+Random insert load generator. This will insert as fast as it can into a
+pre-existing table using AUTO_BACKGROUND_FLUSH mode. All fields are randomized.
+The load generator will continue inserting until it is stopped or it encounters
+an error. This load generator is single-threaded.
+
+To build and run, do the following:
+
+[source,bash]
+----
+$ mvn verify
+$ java -jar target/kudu-insert-loadgen-1.0-SNAPSHOT.jar master_addresses table_name
+----
+
+where `master_addresses` is a CSV of the master addresses for your Kudu cluster
+and `table_name` is the name of a pre-existing table. For example,
+
+[source,bash]
+----
+$ java -jar target/kudu-insert-loadgen-1.0-SNAPSHOT.jar localhost:7051 test_table
+----
+
+This program will not create the "test_table" table. You must do that
+via other means, such as through impala-shell.
http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/insert-loadgen/pom.xml b/examples/java/insert-loadgen/pom.xml
index 09bc0d1..b988465 100644
--- a/examples/java/insert-loadgen/pom.xml
+++ b/examples/java/insert-loadgen/pom.xml
@@ -1,10 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>kudu-examples</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-insert-loadgen</artifactId>
<packaging>jar</packaging>
- <version>0.1-SNAPSHOT</version>
+ <version>1.0-SNAPSHOT</version>
<name>Random Insert Load Generator for Kudu</name>
<build>
@@ -12,7 +32,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.1</version>
+ <version>3.6.2</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
@@ -21,7 +41,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>2.4</version>
+ <version>3.0.0</version>
<executions>
<execution>
<phase>package</phase>
@@ -31,7 +51,7 @@
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.kududb.examples.loadgen.InsertLoadgen</mainClass>
+ <mainClass>org.apache.kudu.examples.InsertLoadgen</mainClass>
</transformer>
</transformers>
</configuration>
@@ -41,32 +61,19 @@
</plugins>
</build>
- <repositories>
- <repository>
- <id>cdh.repo</id>
- <name>Cloudera Repositories</name>
- <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- </repositories>
-
<dependencies>
-
<dependency>
<groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
- <version>1.1.0</version>
+ <version>1.7.0</version>
</dependency>
- <!-- for logging messages -->
+ <!-- For logging messages. -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
- <version>1.7.12</version>
+ <version>1.7.25</version>
</dependency>
-
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java
----------------------------------------------------------------------
diff --git a/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java b/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java
new file mode 100644
index 0000000..923d5bc
--- /dev/null
+++ b/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.kudu.examples;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.Insert;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.PartialRow;
+import org.apache.kudu.client.SessionConfiguration;
+
+public class InsertLoadgen {
+ private static class RandomDataGenerator {
+ private final Random rng;
+ private final int index;
+ private final Type type;
+
+ /**
+ * Instantiate a random data generator for a specific field.
+ * @param index The numerical index of the column in the row schema
+ * @param type The type of the data at index {@code index}
+ */
+ public RandomDataGenerator(int index, Type type) {
+ this.rng = new Random();
+ this.index = index;
+ this.type = type;
+ }
+
+ /**
+ * Add random data to the given row for the column at index {@code index}
+ * of type {@code type}
+ * @param row The row to add the field to
+ */
+ void generateColumnData(PartialRow row) {
+ switch (type) {
+ case INT8:
+ row.addByte(index, (byte) rng.nextInt(Byte.MAX_VALUE));
+ return;
+ case INT16:
+ row.addShort(index, (short)rng.nextInt(Short.MAX_VALUE));
+ return;
+ case INT32:
+ row.addInt(index, rng.nextInt(Integer.MAX_VALUE));
+ return;
+ case INT64:
+ case UNIXTIME_MICROS:
+ row.addLong(index, rng.nextLong());
+ return;
+ case BINARY:
+ byte bytes[] = new byte[16];
+ rng.nextBytes(bytes);
+ row.addBinary(index, bytes);
+ return;
+ case STRING:
+ row.addString(index, UUID.randomUUID().toString());
+ return;
+ case BOOL:
+ row.addBoolean(index, rng.nextBoolean());
+ return;
+ case FLOAT:
+ row.addFloat(index, rng.nextFloat());
+ return;
+ case DOUBLE:
+ row.addDouble(index, rng.nextDouble());
+ return;
+ case DECIMAL:
+ row.addDecimal(index, new BigDecimal(rng.nextDouble()));
+ return;
+ default:
+ throw new UnsupportedOperationException("Unknown type " + type);
+ }
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ System.err.println("Usage: InsertLoadgen master_addresses table_name");
+ System.exit(1);
+ }
+
+ String masterAddrs = args[0];
+ String tableName = args[1];
+
+ try (KuduClient client = new KuduClient.KuduClientBuilder(masterAddrs).build()) {
+ KuduTable table = client.openTable(tableName);
+ Schema schema = table.getSchema();
+ List<RandomDataGenerator> generators = new ArrayList<>(schema.getColumnCount());
+ for (int i = 0; i < schema.getColumnCount(); i++) {
+ generators.add(new RandomDataGenerator(i, schema.getColumnByIndex(i).getType()));
+ }
+
+ KuduSession session = client.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+ for (int insertCount = 0; ; insertCount++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ for (int i = 0; i < schema.getColumnCount(); i++) {
+ generators.get(i).generateColumnData(row);
+ }
+ session.apply(insert);
+
+ // Check for errors. This is done periodically since inserts are batched.
+ if (insertCount % 1000 == 0 && session.countPendingErrors() > 0) {
+ throw new RuntimeException(session.getPendingErrors().getRowErrors()[0].toString());
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
----------------------------------------------------------------------
diff --git a/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java b/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
deleted file mode 100644
index 4b76c34..0000000
--- a/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java
+++ /dev/null
@@ -1,110 +0,0 @@
-package org.kududb.examples.loadgen;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-
-import org.apache.kudu.Schema;
-import org.apache.kudu.Type;
-import org.apache.kudu.client.Insert;
-import org.apache.kudu.client.KuduClient;
-import org.apache.kudu.client.KuduSession;
-import org.apache.kudu.client.KuduTable;
-import org.apache.kudu.client.PartialRow;
-import org.apache.kudu.client.SessionConfiguration;
-
-public class InsertLoadgen {
- private static class RandomDataGenerator {
- private final Random rng;
- private final int index;
- private final Type type;
-
- /**
- * Instantiate a random data generator for a specific field.
- * @param index The numerical index of the column in the row schema
- * @param type The type of the data at index {@code index}
- */
- public RandomDataGenerator(int index, Type type) {
- this.rng = new Random();
- this.index = index;
- this.type = type;
- }
-
- /**
- * Add random data to the given row for the column at index {@code index}
- * of type {@code type}
- * @param row The row to add the field to
- */
- void generateColumnData(PartialRow row) {
- switch (type) {
- case INT8:
- row.addByte(index, (byte) rng.nextInt(Byte.MAX_VALUE));
- return;
- case INT16:
- row.addShort(index, (short)rng.nextInt(Short.MAX_VALUE));
- return;
- case INT32:
- row.addInt(index, rng.nextInt(Integer.MAX_VALUE));
- return;
- case INT64:
- case UNIXTIME_MICROS:
- row.addLong(index, rng.nextLong());
- return;
- case BINARY:
- byte bytes[] = new byte[16];
- rng.nextBytes(bytes);
- row.addBinary(index, bytes);
- return;
- case STRING:
- row.addString(index, UUID.randomUUID().toString());
- return;
- case BOOL:
- row.addBoolean(index, rng.nextBoolean());
- return;
- case FLOAT:
- row.addFloat(index, rng.nextFloat());
- return;
- case DOUBLE:
- row.addDouble(index, rng.nextDouble());
- return;
- default:
- throw new UnsupportedOperationException("Unknown type " + type);
- }
- }
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- System.err.println("Usage: InsertLoadgen kudu_master_host kudu_table");
- System.exit(1);
- }
-
- String masterHost = args[0];
- String tableName = args[1];
-
- try (KuduClient client = new KuduClient.KuduClientBuilder(masterHost).build()) {
- KuduTable table = client.openTable(tableName);
- Schema schema = table.getSchema();
- List<RandomDataGenerator> generators = new ArrayList<>(schema.getColumnCount());
- for (int i = 0; i < schema.getColumnCount(); i++) {
- generators.add(new RandomDataGenerator(i, schema.getColumnByIndex(i).getType()));
- }
-
- KuduSession session = client.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
- for (int insertCount = 0; ; insertCount++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- for (int i = 0; i < schema.getColumnCount(); i++) {
- generators.get(i).generateColumnData(row);
- }
- session.apply(insert);
-
- if (insertCount % 1000 == 0 && session.countPendingErrors() > 0) {
- throw new RuntimeException(session.getPendingErrors().getRowErrors()[0].toString());
- }
- }
- }
- }
-}