You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by ah...@apache.org on 2019/03/16 22:23:34 UTC
[commons-rng] branch master updated (b007cb2 -> 3288206)
This is an automated email from the ASF dual-hosted git repository.
aherbert pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/commons-rng.git.
from b007cb2 Update the stress test c-code and documentation for TestU01 src install.
new e82f2d1 Use local SimpleDateFormat for each thread.
new b58fcf8 Add stdin2testu01 to .gitignore
new 3288206 Added a test program for the stress test java-to-c bridge.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 1 +
commons-rng-examples/examples-jmh/pom.xml | 2 -
commons-rng-examples/examples-stress/pom.xml | 38 ++++
.../examples-stress/src/main/c/stdin2testu01.c | 114 +++++++++++-
.../commons/rng/examples/stress/BridgeTester.java | 205 +++++++++++++++++++++
.../rng/examples/stress/RandomStressTester.java | 6 +-
commons-rng-examples/pom.xml | 6 +
7 files changed, 364 insertions(+), 8 deletions(-)
create mode 100644 commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTester.java
[commons-rng] 02/03: Add stdin2testu01 to .gitignore
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aherbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-rng.git
commit b58fcf89c42c113512aac8a7cb4ac1bdabc2d9c1
Author: Alex Herbert <ah...@apache.org>
AuthorDate: Sat Mar 16 20:09:38 2019 +0000
Add stdin2testu01 to .gitignore
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index 7410e55..7bb349e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,4 @@ site-content
*~
/.externalToolBuilders/
/maven-eclipse.xml
+stdin2testu01
[commons-rng] 03/03: Added a test program for the stress test
java-to-c bridge.
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aherbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-rng.git
commit 3288206ba2933544a1ae8e398af0e8d1c3c319db
Author: Alex Herbert <ah...@apache.org>
AuthorDate: Sat Mar 16 22:23:10 2019 +0000
Added a test program for the stress test java-to-c bridge.
---
commons-rng-examples/examples-jmh/pom.xml | 2 -
commons-rng-examples/examples-stress/pom.xml | 38 ++++
.../examples-stress/src/main/c/stdin2testu01.c | 114 +++++++++++-
.../commons/rng/examples/stress/BridgeTester.java | 205 +++++++++++++++++++++
.../rng/examples/stress/RandomStressTester.java | 4 +-
commons-rng-examples/pom.xml | 6 +
6 files changed, 361 insertions(+), 8 deletions(-)
diff --git a/commons-rng-examples/examples-jmh/pom.xml b/commons-rng-examples/examples-jmh/pom.xml
index 9d7e629..ead43af 100644
--- a/commons-rng-examples/examples-jmh/pom.xml
+++ b/commons-rng-examples/examples-jmh/pom.xml
@@ -70,7 +70,6 @@
<!-- JMH Benchmark related properties: version, name of the benchmarking uber jar. -->
<jmh.version>1.21</jmh.version>
<uberjar.name>benchmarks</uberjar.name>
- <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
</properties>
<profiles>
@@ -96,7 +95,6 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>${exec-maven-plugin.version}</version>
<executions>
<execution>
<id>benchmark</id>
diff --git a/commons-rng-examples/examples-stress/pom.xml b/commons-rng-examples/examples-stress/pom.xml
index ef2bae2..0cc55e2 100644
--- a/commons-rng-examples/examples-stress/pom.xml
+++ b/commons-rng-examples/examples-stress/pom.xml
@@ -102,6 +102,44 @@
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>bridge</id>
+ <properties>
+ <skipTests>true</skipTests>
+ <java.cmd>java</java.cmd>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>benchmark</id>
+ <phase>test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <classpathScope>test</classpathScope>
+ <executable>${java.cmd}</executable>
+ <arguments>
+ <argument>-classpath</argument>
+ <classpath/>
+ <argument>org.apache.commons.rng.examples.stress.BridgeTester</argument>
+ <argument>target/bridge</argument>
+ <argument>./stdin2testu01</argument>
+ <argument>stdout</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
diff --git a/commons-rng-examples/examples-stress/src/main/c/stdin2testu01.c b/commons-rng-examples/examples-stress/src/main/c/stdin2testu01.c
index f17a9cc..b54e375 100644
--- a/commons-rng-examples/examples-stress/src/main/c/stdin2testu01.c
+++ b/commons-rng-examples/examples-stress/src/main/c/stdin2testu01.c
@@ -50,25 +50,124 @@
#define TU_S "SmallCrush"
#define TU_C "Crush"
#define TU_B "BigCrush"
-#define BUFFER_LENGTH 256
+#define T_STDOUT "stdout"
+#define BUFFER_LENGTH 512
typedef struct {
- unsigned long buffer[BUFFER_LENGTH];
+ uint32_t buffer[BUFFER_LENGTH];
uint32_t index;
} StdinReader_state;
+/*
+ * Flag used to store if the system is little endian.
+ *
+ * Note: Java is big endian and the bytes passed on stdin by a
+ * Java DataOutputStream will be written most significant first.
+ * If the c compiler is little endian for uint32_t the bytes
+ * read from stdin must be reversed.
+ */
+static int littleEndian = 0;
+
+/* Lookup table for binary representation of bytes. */
+const char *bit_rep[16] = {
+ [ 0] = "0000", [ 1] = "0001", [ 2] = "0010", [ 3] = "0011",
+ [ 4] = "0100", [ 5] = "0101", [ 6] = "0110", [ 7] = "0111",
+ [ 8] = "1000", [ 9] = "1001", [10] = "1010", [11] = "1011",
+ [12] = "1100", [13] = "1101", [14] = "1110", [15] = "1111",
+};
+
+/*
+ * Print a binary string representation of the 8-bits of the byte to stdout.
+ *
+ * 01101101
+ */
+void printByte(uint8_t byte)
+{
+ printf("%s%s", bit_rep[byte >> 4], bit_rep[byte & 0x0F]);
+}
+
+/*
+ * Print a string representation of the 4 bytes of the unsigned int
+ * to stdout on a single line using: a binary string representation of
+ * the bytes; the unsigned integer; and the signed integer.
+ *
+ * 11001101 00100011 01101111 01110000 3441651568 -853315728
+ */
+void printInt(uint32_t value)
+{
+ /* Write out as 4 bytes with spaces between them, high byte first. */
+ printByte((uint8_t)((value >> 24) & 0xff));
+ putchar(' ');
+ printByte((uint8_t)((value >> 16) & 0xff));
+ putchar(' ');
+ printByte((uint8_t)((value >> 8) & 0xff));
+ putchar(' ');
+ printByte((uint8_t)( value & 0xff));
+ /* Write the unsigned and signed int value */
+ printf(" %11u %11d\n", value, (int) value);
+}
+
+/*
+ * Determine endianess.
+ */
+void detectEndianess() {
+ uint32_t val = 0x01;
+ /*
+ * Use a raw view of the bytes with a char* to determine if
+ * the first byte is unset (big endian) or set (little endian).
+ */
+ char * buff = (char *)&val;
+
+ littleEndian = (buff[0] != 0);
+}
+
+/*
+ * Reverse the 4 bytes in an unsigned int, effectively converting from big
+ * endian to small or vice versa.
+ */
+uint32_t reverseBytes(uint32_t value) {
+ return ((value >> 24) & 0x000000ff) | // move byte 3 to byte 0
+ ((value << 8) & 0x00ff0000) | // move byte 1 to byte 2
+ ((value >> 8) & 0x0000ff00) | // move byte 2 to byte 1
+ ((value << 24) & 0xff000000); // byte 0 to byte 3
+}
+
unsigned long nextInt(void *par,
void *sta) {
+ static size_t last_read = 0;
+
StdinReader_state *state = (StdinReader_state *) sta;
- if (state->index >= BUFFER_LENGTH) {
+ if (state->index >= last_read) {
/* Refill. */
- fread(state->buffer, sizeof(unsigned long), BUFFER_LENGTH, stdin);
+ last_read = fread(state->buffer, sizeof(uint32_t), BUFFER_LENGTH, stdin);
+ if (last_read != BUFFER_LENGTH) {
+ // Allow reading less than the buffer length, but not zero
+ if (last_read == 0) {
+ // Error handling
+ if (feof(stdin)) {
+ // End of stream, just exit. This is used for testing.
+ exit(0);
+ } else if (ferror(stdin)) {
+ // perror will contain a description of the error code
+ perror("[ERROR] Failed to read stdin");
+ exit(1);
+ } else {
+ printf("[ERROR] No data from stdin\n");
+ exit(1);
+ }
+ }
+ }
state->index = 0;
}
uint32_t random = state->buffer[state->index];
++state->index; /* Next request. */
+ if (littleEndian) {
+ /* Convert to machine representation from Java big endian. */
+ random = reverseBytes(random);
+ }
+
return random;
}
@@ -122,6 +221,8 @@ int main(int argc,
unif01_Gen *gen = createStdinReader();
char *spec = argv[1];
+ detectEndianess();
+
if (argc < 2) {
printf("[ERROR] Specify test suite: '%s', '%s' or '%s'\n", TU_S, TU_C, TU_B);
exit(1);
@@ -131,6 +232,11 @@ int main(int argc,
bbattery_Crush(gen);
} else if (strcmp(spec, TU_B) == 0) {
bbattery_BigCrush(gen);
+ } else if (strcmp(spec, T_STDOUT) == 0) {
+ // Print to stdout until stdin closes
+ while (1) {
+ printInt(nextInt(0, gen->state));
+ }
} else {
printf("[ERROR] Unknown specification: '%s'\n", spec);
exit(1);
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTester.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTester.java
new file mode 100644
index 0000000..bbfc560
--- /dev/null
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTester.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.rng.examples.stress;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Class that can be used for testing that {@code int} values can be piped to a
+ * program that reads {@code int} values from its standard input. Only
+ * a limited number of {@code int} values will be written.
+ *
+ * Example of command line, assuming that "examples.jar" specifies this
+ * class as the "main" class (see {@link #main(String[]) main} method):
+ * <pre><code>
+ * $ java -jar examples.jar \
+ * target/bridge \
+ * ./stdin2testu01 stdout
+ * </code></pre>
+ */
+public class BridgeTester {
+ /** Lookup table for binary representation of bytes. */
+ private static final String[] BIT_REP = {
+ "0000", "0001", "0010", "0011",
+ "0100", "0101", "0110", "0111",
+ "1000", "1001", "1010", "1011",
+ "1100", "1101", "1110", "1111",
+ };
+
+ /** Command line. */
+ private final List<String> cmdLine;
+ /** Output prefix. */
+ private final String fileOutputPrefix;
+
+ /**
+ * Creates the application.
+ *
+ * @param cmd Command line.
+ * @param outputPrefix Output prefix for file reports.
+ */
+ private BridgeTester(List<String> cmd,
+ String outputPrefix) {
+ final File exec = new File(cmd.get(0));
+ if (!exec.exists() ||
+ !exec.canExecute()) {
+ throw new IllegalArgumentException("Program is not executable: " + exec);
+ }
+
+ cmdLine = new ArrayList<>(cmd);
+ this.fileOutputPrefix = outputPrefix;
+
+ final File reportDir = new File(fileOutputPrefix).getParentFile();
+ if (!reportDir.exists() ||
+ !reportDir.isDirectory() ||
+ !reportDir.canWrite()) {
+ throw new IllegalArgumentException("Invalid output directory: " + reportDir);
+ }
+ }
+
+ /**
+ * Program's entry point.
+ *
+ * @param args Application's arguments.
+ * The order is as follows:
+ * <ol>
+ * <li>Output prefix: Filename prefix where the output will written to.
+ * The appended suffix is [.data] for the Java native {@code int} values,
+ * and [.out] and [.err] for the stdout/stderr from the called executable.</li>
+ * <li>Path to the executable: this is the software that reads 32-bit
+ * integers from stdin.</li>
+ * <li>All remaining arguments are passed to the executable.</li>
+ * </ol>
+ */
+ public static void main(String[] args) {
+ final String outputPrefix = args[0];
+
+ final List<String> cmdLine = new ArrayList<>();
+ cmdLine.addAll(Arrays.asList(Arrays.copyOfRange(args, 1, args.length)));
+
+ final BridgeTester app = new BridgeTester(cmdLine, outputPrefix);
+
+ // Throws runtime exceptions
+ app.run();
+ }
+
+ /**
+ * Starts the executable process and writes {@code int} values to a data file and the stdin
+ * of the executable. Captures stdout of the executable to a file.
+ */
+ private void run() {
+ try {
+ final File dataFile = new File(fileOutputPrefix + ".data");
+ final File outputFile = new File(fileOutputPrefix + ".out");
+ final File errorFile = new File(fileOutputPrefix + ".err");
+
+ // Start the application.
+ final ProcessBuilder builder = new ProcessBuilder(cmdLine);
+ builder.redirectOutput(ProcessBuilder.Redirect.to(outputFile));
+ builder.redirectError(ProcessBuilder.Redirect.to(errorFile));
+ final Process testingProcess = builder.start();
+
+ // Both resources will be closed automatically
+ try (
+ // Open the stdin of the process
+ DataOutputStream output = new DataOutputStream(
+ new BufferedOutputStream(testingProcess.getOutputStream()));
+ // Open the file for Java int data
+ BufferedWriter data = new BufferedWriter(new FileWriter(dataFile))) {
+
+ // Write int data using a single bit in all possible positions
+ int value = 1;
+ for (int i = 0; i < Integer.SIZE; i++) {
+ writeInt(data, output, value);
+ value <<= 1;
+ }
+
+ // Write random int data
+ final ThreadLocalRandom rng = ThreadLocalRandom.current();
+ for (int i = 0; i < Integer.SIZE; i++) {
+ writeInt(data, output, rng.nextInt());
+ }
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to run process: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Write an {@code int} value to the buffered writer and the output. The native
+ * Java value will be written to the buffered writer on a single line using: a
+ * binary string representation of the bytes; the unsigned integer; and the
+ * signed integer.
+ *
+ * <pre>
+ * 11001101 00100011 01101111 01110000 3441651568 -853315728
+ * </pre>
+ *
+ * @param data the native data buffered writer.
+ * @param output the output.
+ * @param value the value.
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private static void writeInt(BufferedWriter data,
+ DataOutputStream output,
+ int value) throws IOException {
+
+ // Write out as 4 bytes with spaces between them, high byte first.
+ writeByte(data, (value >>> 24) & 0xff);
+ data.write(' ');
+ writeByte(data, (value >>> 16) & 0xff);
+ data.write(' ');
+ writeByte(data, (value >>> 8) & 0xff);
+ data.write(' ');
+ writeByte(data, (value >>> 0) & 0xff);
+
+ // Write the unsigned and signed int value
+ data.write(String.format(" %11d %11d%n", value & 0xffffffffL, value));
+
+ // Write the raw binary data to the output
+ output.writeInt(value);
+ }
+
+ /**
+ * Write the lower 8 bits of an {@code int} value to the buffered writer using a binary
+ * string representation. This is left-filled with zeros if applicable.
+ *
+ * <pre>
+ * 11001101
+ * </pre>
+ *
+ * @param data the buffered writer.
+ * @param value the value.
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private static void writeByte(BufferedWriter data,
+ int value) throws IOException {
+ // This matches the functionality of:
+ // data.write(String.format("%8s", Integer.toBinaryString(value & 0xff)).replace(' ', '0'));
+ data.write(BIT_REP[value >>> 4]);
+ data.write(BIT_REP[value & 0x0F]);
+ }
+}
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java
index 5ace017..2b95afe 100644
--- a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java
@@ -58,12 +58,12 @@ public class RandomStressTester {
private static final String C = "# ";
/** New line. */
private static final String N = System.lineSeparator();
+ /** The date format. */
+ private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
/** Command line. */
private final List<String> cmdLine;
/** Output prefix. */
private final String fileOutputPrefix;
- /** The date format. */
- private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
/**
* Creates the application.
diff --git a/commons-rng-examples/pom.xml b/commons-rng-examples/pom.xml
index 491b377..bc19daa 100644
--- a/commons-rng-examples/pom.xml
+++ b/commons-rng-examples/pom.xml
@@ -69,6 +69,12 @@
<artifactId>commons-rng-client-api</artifactId>
<version>1.3-SNAPSHOT</version>
</dependency>
+
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.6.0</version>
+ </dependency>
</dependencies>
</dependencyManagement>
[commons-rng] 01/03: Use local SimpleDateFormat for each thread.
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aherbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-rng.git
commit e82f2d1fe4b0688da5d6d3d7d3f1b30b12272759
Author: Alex Herbert <ah...@apache.org>
AuthorDate: Sat Mar 16 20:07:50 2019 +0000
Use local SimpleDateFormat for each thread.
---
.../org/apache/commons/rng/examples/stress/RandomStressTester.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java
index 7438838..5ace017 100644
--- a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RandomStressTester.java
@@ -63,7 +63,7 @@ public class RandomStressTester {
/** Output prefix. */
private final String fileOutputPrefix;
/** The date format. */
- private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
/**
* Creates the application.
@@ -312,6 +312,8 @@ public class RandomStressTester {
*/
private void appendDate(StringBuilder sb,
String prefix) {
+ // Use local date format. It is not thread safe.
+ final SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
sb.append(C).append(prefix).append(": ").append(dateFormat.format(new Date())).append(N);
sb.append(C).append(N);
}