You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by ah...@apache.org on 2019/10/11 13:28:18 UTC
[commons-rng] 06/16: Updated stress test application for 64-bit
test applications.
This is an automated email from the ASF dual-hosted git repository.
aherbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-rng.git
commit bf2c3277c47259ad7b0d480653af7436dc69c5de
Author: aherbert <ah...@apache.org>
AuthorDate: Fri Oct 4 17:26:45 2019 +0100
Updated stress test application for 64-bit test applications.
64-bit support is required for PractRand which can test 64-bit
generators.
Added a custom RNG data output class to write byte data. This handles
endianess conversion when writing thus removing the requirement to do
this in code. It also removes synchronization of the
BufferedOutputStream. BigCrush runs approximately 2 times faster with
the modified application bridge code.
Fix stdout argument example to raw32.
Fix code documentation of 'bridge' command.
---
commons-rng-examples/examples-stress/endianness.md | 8 +-
.../rng/examples/stress/BridgeTestCommand.java | 71 ++--
.../commons/rng/examples/stress/OutputCommand.java | 156 ++++++--
.../commons/rng/examples/stress/RNGUtils.java | 392 +++++++++++++++------
.../commons/rng/examples/stress/RngDataOutput.java | 369 +++++++++++++++++++
.../rng/examples/stress/StressTestCommand.java | 71 ++--
.../rng/examples/stress/RngDataOutputTest.java | 215 +++++++++++
7 files changed, 1093 insertions(+), 189 deletions(-)
diff --git a/commons-rng-examples/examples-stress/endianness.md b/commons-rng-examples/examples-stress/endianness.md
index f33d0ee..be4e570 100644
--- a/commons-rng-examples/examples-stress/endianness.md
+++ b/commons-rng-examples/examples-stress/endianness.md
@@ -113,13 +113,13 @@ send a set of test integer data in binary format to the standard input (stdin) o
application.
The `stdin2testu01` application is able to read the input binary data and convert it to human
-readable format when run with the argument `stdout`. Details of how to build the `stdin2testu01`
+readable format when run with the argument `raw32`. Details of how to build the `stdin2testu01`
application are provided in the [stress test](./stress_test.md) page.
The platform endianess is auto-detected by the Java application. To run the bridge test
use the following command:
- > java -jar target/examples-stress.jar bridge ./stdin2testu01 stdout
+ > java -jar target/examples-stress.jar bridge ./stdin2testu01 raw32
This should produce the following output files:
@@ -144,12 +144,12 @@ integer value and the signed integer value. The contents will be similar to:
01011100 10001111 11110001 11000001 1552937409 1552937409
10110000 01110101 10010011 00011100 2960495388 -1334471908
-The `stdin2testu01` has been written to output the same format when using the `stdout` mode.
+The `stdin2testu01` has been written to output the same format when using the `raw32` mode.
If the data has been correctly read the `bridge.data` and `bridge.out` should match.
If the endianess is incorrect then the data sent by the Java application will not match the
data read by the sub-process. For example to swap the endianness use the `-b` option:
- > java -jar target/examples-stress.jar bridge -b BIG_ENDIAN ./stdin2testu01 stdout
+ > java -jar target/examples-stress.jar bridge -b BIG_ENDIAN ./stdin2testu01 raw32
In this case the little-endian plaform has been sent big-endian data and the contents of the
`bridge.out` file begin with the reverse byte order:
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTestCommand.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTestCommand.java
index b0cc26e..74e54f4 100644
--- a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTestCommand.java
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/BridgeTestCommand.java
@@ -16,18 +16,20 @@
*/
package org.apache.commons.rng.examples.stress;
+import org.apache.commons.rng.UniformRandomProvider;
+import org.apache.commons.rng.core.source32.IntProvider;
+
import picocli.CommandLine.Command;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;
-import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
-import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.Writer;
import java.nio.ByteOrder;
+import java.nio.IntBuffer;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
@@ -42,7 +44,7 @@ import java.util.concurrent.ThreadLocalRandom;
* a limited number of {@code int} values will be written.</p>
*/
@Command(name = "bridge",
- description = {"Transfer test data to a test application sub-process via standard input."})
+ description = {"Transfer test 32-bit data to a test application sub-process via standard input."})
class BridgeTestCommand implements Callable<Void> {
/** The standard options. */
@Mixin
@@ -94,33 +96,45 @@ class BridgeTestCommand implements Callable<Void> {
final File outputFile = new File(fileOutputPrefix + ".out");
final File errorFile = new File(fileOutputPrefix + ".err");
- // Start the application.
- final ProcessBuilder builder = new ProcessBuilder(command);
- builder.redirectOutput(ProcessBuilder.Redirect.to(outputFile));
- builder.redirectError(ProcessBuilder.Redirect.to(errorFile));
- final Process testingProcess = builder.start();
-
- // Both resources will be closed automatically
- try (
- // Open the stdin of the process
- DataOutputStream dataOutput = new DataOutputStream(
- new BufferedOutputStream(testingProcess.getOutputStream()));
- // Open the file for Java int data
- BufferedWriter textOutput = Files.newBufferedWriter(dataFile.toPath())) {
+ // Store int values
+ final IntBuffer buffer = IntBuffer.allocate(Integer.SIZE * 2);
- final boolean littleEndian = byteOrder == ByteOrder.LITTLE_ENDIAN;
+ try (BufferedWriter textOutput = Files.newBufferedWriter(dataFile.toPath())) {
// Write int data using a single bit in all possible positions
int value = 1;
for (int i = 0; i < Integer.SIZE; i++) {
- writeInt(textOutput, dataOutput, value, littleEndian);
+ writeInt(textOutput, buffer, value);
value <<= 1;
}
// Write random int data
- final ThreadLocalRandom rng = ThreadLocalRandom.current();
- for (int i = 0; i < Integer.SIZE; i++) {
- writeInt(textOutput, dataOutput, rng.nextInt(), littleEndian);
+ while (buffer.remaining() != 0) {
+ writeInt(textOutput, buffer, ThreadLocalRandom.current().nextInt());
+ }
+ }
+
+ // Pass the same values to the output application
+ buffer.flip();
+ UniformRandomProvider rng = new IntProvider() {
+ @Override
+ public int next() {
+ return buffer.get();
}
+ };
+
+ // Start the application.
+ final ProcessBuilder builder = new ProcessBuilder(command);
+ builder.redirectOutput(ProcessBuilder.Redirect.to(outputFile));
+ builder.redirectError(ProcessBuilder.Redirect.to(errorFile));
+ final Process testingProcess = builder.start();
+
+ // Open the stdin of the process and write to a custom data sink.
+ // Note: The 'bridge' command only supports 32-bit data in order to
+ // demonstrate passing suitable data for TestU01 BigCrush.
+ final boolean raw64 = false;
+ try (RngDataOutput sink = RNGUtils.createDataOutput(rng, raw64,
+ testingProcess.getOutputStream(), buffer.capacity() * 4, byteOrder)) {
+ sink.write(rng);
}
final Integer exitValue = ProcessUtils.getExitValue(testingProcess);
@@ -138,23 +152,20 @@ class BridgeTestCommand implements Callable<Void> {
}
/**
- * Write an {@code int} value to the writer and the binary output. The native
- * Java value will be written to the writer using the debugging output of the
+ * Write an {@code int} value to the writer and the buffer output. The native Java
+ * value will be written to the writer using the debugging output of the
* {@link OutputCommand}.
*
* @param textOutput the text data writer.
- * @param dataOutput the binary data output.
+ * @param buffer the buffer.
* @param value the value.
- * @param littleEndian Set to {@code true} to write the binary output using little-endian byte order.
* @throws IOException Signals that an I/O exception has occurred.
* @see OutputCommand#writeInt(java.io.Writer, int)
*/
private static void writeInt(Writer textOutput,
- DataOutputStream dataOutput,
- int value,
- boolean littleEndian) throws IOException {
+ IntBuffer buffer,
+ int value) throws IOException {
OutputCommand.writeInt(textOutput, value);
- final int binaryValue = littleEndian ? Integer.reverseBytes(value) : value;
- dataOutput.writeInt(binaryValue);
+ buffer.put(value);
}
}
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/OutputCommand.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/OutputCommand.java
index 6e15d3a..da5646b 100644
--- a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/OutputCommand.java
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/OutputCommand.java
@@ -17,6 +17,7 @@
package org.apache.commons.rng.examples.stress;
import org.apache.commons.rng.UniformRandomProvider;
+import org.apache.commons.rng.core.source64.RandomLongSource;
import org.apache.commons.rng.simple.RandomSource;
import picocli.CommandLine.Command;
@@ -24,9 +25,7 @@ import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;
-import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
-import java.io.DataOutputStream;
import java.io.File;
import java.io.FilterOutputStream;
import java.io.IOException;
@@ -101,6 +100,13 @@ class OutputCommand implements Callable<Void> {
"Use negative for an unlimited stream."})
private long count = 10;
+ /** The size of the byte buffer for the binary data. */
+ @Option(names = {"--buffer-size"},
+ description = {"Byte-buffer size for binary data (default: ${DEFAULT-VALUE}).",
+ "When outputing binary data the count parameter controls the " +
+ "number of buffers written."})
+ private int bufferSize = 8192;
+
/** The output byte order of the binary data. */
@Option(names = {"-b", "--byte-order"},
description = {"Byte-order of the output data (default: ${DEFAULT-VALUE}).",
@@ -113,6 +119,25 @@ class OutputCommand implements Callable<Void> {
description = {"Reverse the bits in the data (default: ${DEFAULT-VALUE})."})
private boolean reverseBits;
+ /** Flag to use the upper 32-bits from the 64-bit long output. */
+ @Option(names = {"--high-bits"},
+ description = {"Use the upper 32-bits from the 64-bit long output.",
+ "Takes precedent over --low-bits."})
+ private boolean longHighBits;
+
+ /** Flag to use the lower 32-bits from the 64-bit long output. */
+ @Option(names = {"--low-bits"},
+ description = {"Use the lower 32-bits from the 64-bit long output."})
+ private boolean longLowBits;
+
+ /** Flag to use 64-bit long output. */
+ @Option(names = {"--raw64"},
+ description = {"Use 64-bit output (default is 32-bit).",
+ "This is ignored if not a native 64-bit generator.",
+ "In 32-bit mode the output uses the upper then lower bits of 64-bit " +
+ "generators sequentially."})
+ private boolean raw64;
+
/**
* The output mode for existing files.
*/
@@ -132,22 +157,36 @@ class OutputCommand implements Callable<Void> {
public Void call() {
LogUtils.setLogLevel(reusableOptions.logLevel);
UniformRandomProvider rng = createRNG();
- if (byteOrder.equals(ByteOrder.LITTLE_ENDIAN)) {
- rng = RNGUtils.createReverseBytesIntProvider(rng);
+
+ // Upper or lower bits from 64-bit generators must be created first.
+ // This will throw if not a 64-bit generator.
+ if (longHighBits) {
+ rng = RNGUtils.createLongUpperBitsIntProvider(rng);
+ } else if (longLowBits) {
+ rng = RNGUtils.createLongLowerBitsIntProvider(rng);
}
if (reverseBits) {
- rng = RNGUtils.createReverseBitsIntProvider(rng);
+ rng = RNGUtils.createReverseBitsProvider(rng);
+ }
+
+ // -------
+ // Note: Manipulation of the byte order for the platform is done during output
+ // for the binary format. Otherwise do it in Java.
+ // -------
+ if (outputFormat != OutputFormat.BINARY) {
+ rng = toOutputFormat(rng);
}
+
try (OutputStream out = createOutputStream()) {
switch (outputFormat) {
case BINARY:
- writeBinaryIntData(rng, count, out);
+ writeBinaryData(rng, out);
break;
case DIEHARDER:
- writeDieharder(rng, seed == null ? "auto" : seed, count, out);
+ writeDieharder(rng, seed == null ? "auto" : seed, out);
break;
case BITS:
- writeBitIntData(rng, count, out);
+ writeBitData(rng, out);
break;
default:
throw new ApplicationException("Unknown output format: " + outputFormat);
@@ -213,6 +252,26 @@ class OutputCommand implements Callable<Void> {
}
/**
+ * Convert the native RNG to the requested output format. This will convert a 64-bit
+ * generator to a 32-bit generator unless the 64-bit mode is active. It then optionally
+ * reverses the byte order of the output.
+ *
+ * @param rng The random generator.
+ * @return the uniform random provider
+ */
+ private UniformRandomProvider toOutputFormat(UniformRandomProvider rng) {
+ UniformRandomProvider convertedRng = rng;
+ if (rng instanceof RandomLongSource && !raw64) {
+ // Convert to 32-bit generator
+ convertedRng = RNGUtils.createIntProvider(rng);
+ }
+ if (byteOrder == ByteOrder.LITTLE_ENDIAN) {
+ convertedRng = RNGUtils.createReverseBytesProvider(convertedRng);
+ }
+ return convertedRng;
+ }
+
+ /**
* Creates the output stream. This will not be buffered.
*
* @return the output stream
@@ -251,16 +310,14 @@ class OutputCommand implements Callable<Void> {
* Write int data to the specified output using the dieharder text format.
*
* @param rng The random generator.
- * @param seed The seed using to create the random source.
- * @param count The count of numbers to output.
+ * @param seedObject The seed using to create the random source.
* @param out The output.
* @throws IOException Signals that an I/O exception has occurred.
* @throws ApplicationException If the count is not positive.
*/
- static void writeDieharder(final UniformRandomProvider rng,
- final Object seed,
- long count,
- final OutputStream out) throws IOException {
+ private void writeDieharder(final UniformRandomProvider rng,
+ final Object seedObject,
+ final OutputStream out) throws IOException {
checkCount(count, OutputFormat.DIEHARDER);
// Use dieharder output, e.g.
@@ -276,7 +333,7 @@ class OutputCommand implements Callable<Void> {
output.write("# generator ");
output.write(rng.toString());
output.write(" seed = ");
- output.write(String.valueOf(seed));
+ output.write(String.valueOf(seedObject));
output.write(NEW_LINE);
writeHeaderLine(output);
output.write("type: d");
@@ -311,49 +368,88 @@ class OutputCommand implements Callable<Void> {
}
/**
- * Write raw binary int data to the specified file.
+ * Write raw binary data to the output.
*
* @param rng The random generator.
- * @param count The count of numbers to output.
* @param out The output.
* @throws IOException Signals that an I/O exception has occurred.
*/
- static void writeBinaryIntData(final UniformRandomProvider rng,
- long count,
- final OutputStream out) throws IOException {
+ private void writeBinaryData(final UniformRandomProvider rng,
+ final OutputStream out) throws IOException {
// If count is not positive use max value.
// This is effectively unlimited: program must be killed.
final long limit = (count < 1) ? Long.MAX_VALUE : count;
- try (DataOutputStream data = new DataOutputStream(
- new BufferedOutputStream(out))) {
+ try (RngDataOutput data = RNGUtils.createDataOutput(rng, raw64, out, bufferSize, byteOrder)) {
for (long c = 0; c < limit; c++) {
- data.writeInt(rng.nextInt());
+ data.write(rng);
}
}
}
/**
- * Write raw binary int data to the specified file.
+ * Write binary bit data to the specified file.
*
* @param rng The random generator.
- * @param count The count of numbers to output.
* @param out The output.
* @throws IOException Signals that an I/O exception has occurred.
* @throws ApplicationException If the count is not positive.
*/
- static void writeBitIntData(final UniformRandomProvider rng,
- long count,
- final OutputStream out) throws IOException {
+ private void writeBitData(final UniformRandomProvider rng,
+ final OutputStream out) throws IOException {
checkCount(count, OutputFormat.BITS);
+ boolean asLong = rng instanceof RandomLongSource;
+
try (BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
for (long c = 0; c < count; c++) {
- writeInt(output, rng.nextInt());
+ if (asLong) {
+ writeLong(output, rng.nextLong());
+ } else {
+ writeInt(output, rng.nextInt());
+ }
}
}
}
/**
+ * Write an {@code long} value to the the output. The native Java value will be
+ * written to the writer on a single line using: a binary string representation
+ * of the bytes; the unsigned integer; and the signed integer.
+ *
+ * <pre>
+ * 10011010 01010011 01011010 11100100 01000111 00010000 01000011 11000101 11120331841399178181 -7326412232310373435
+ * </pre>
+ *
+ * @param out The output.
+ * @param value The value.
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("resource")
+ static void writeLong(Writer out,
+ long value) throws IOException {
+
+ // Write out as 8 bytes with spaces between them, high byte first.
+ writeByte(out, (int)(value >>> 56) & 0xff);
+ out.write(' ');
+ writeByte(out, (int)(value >>> 48) & 0xff);
+ out.write(' ');
+ writeByte(out, (int)(value >>> 40) & 0xff);
+ out.write(' ');
+ writeByte(out, (int)(value >>> 32) & 0xff);
+ out.write(' ');
+ writeByte(out, (int)(value >>> 24) & 0xff);
+ out.write(' ');
+ writeByte(out, (int)(value >>> 16) & 0xff);
+ out.write(' ');
+ writeByte(out, (int)(value >>> 8) & 0xff);
+ out.write(' ');
+ writeByte(out, (int)(value >>> 0) & 0xff);
+
+ // Write the unsigned and signed int value
+ new Formatter(out).format(" %20s %20d%n", Long.toUnsignedString(value), value);
+ }
+
+ /**
* Write an {@code int} value to the the output. The native Java value will be
* written to the writer on a single line using: a binary string representation
* of the bytes; the unsigned integer; and the signed integer.
@@ -380,7 +476,7 @@ class OutputCommand implements Callable<Void> {
writeByte(out, (value >>> 0) & 0xff);
// Write the unsigned and signed int value
- new Formatter(out).format(" %11d %11d%n", value & 0xffffffffL, value);
+ new Formatter(out).format(" %10d %11d%n", value & 0xffffffffL, value);
}
/**
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RNGUtils.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RNGUtils.java
index fc8ac47..4c3056c 100644
--- a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RNGUtils.java
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RNGUtils.java
@@ -18,13 +18,34 @@ package org.apache.commons.rng.examples.stress;
import org.apache.commons.rng.UniformRandomProvider;
import org.apache.commons.rng.core.source32.IntProvider;
+import org.apache.commons.rng.core.source32.RandomIntSource;
+import org.apache.commons.rng.core.source64.RandomLongSource;
+import org.apache.commons.rng.core.util.NumberFactory;
+import org.apache.commons.rng.core.source64.LongProvider;
+import java.io.OutputStream;
+import java.nio.ByteOrder;
import java.util.concurrent.ThreadLocalRandom;
/**
* Utility methods for a {@link UniformRandomProvider}.
*/
final class RNGUtils {
+ /** Name prefix for bit-reversed RNGs. */
+ private static final String BYTE_REVERSED = "Byte-reversed ";
+ /** Name prefix for byte-reversed RNGs. */
+ private static final String BIT_REVERSED = "Bit-reversed ";
+ /** Name prefix for hashcode mixed RNGs. */
+ private static final String HASH_CODE = "HashCode ^ ";
+ /** Name prefix for ThreadLocalRandom xor mixed RNGs. */
+ private static final String TLR_MIXED = "ThreadLocalRandom ^ ";
+ /** Name of xor operator for xor mixed RNGs. */
+ private static final String XOR = " ^ ";
+ /** Message for an unrecognised native output type. */
+ private static final String UNRECOGNISED_NATIVE_TYPE = "Unrecognised native output type: ";
+ /** Message when not a RandomLongSource. */
+ private static final String NOT_LONG_SOURCE = "Not a 64-bit long generator: ";
+
/**
* Used to build 4-bit numbers as Hex.
*/
@@ -36,177 +57,346 @@ final class RNGUtils {
private RNGUtils() {}
/**
- * Wrap the random generator with an {@link IntProvider} that will reverse the byte order
- * of the {@code int}.
+ * Wrap the random generator with a new instance that will reverse the byte order of
+ * the native type. The input must be either a {@link RandomIntSource} or
+ * {@link RandomLongSource}.
*
* @param rng The random generator.
* @return the byte reversed random generator.
+ * @throws ApplicationException If the input source native type is not recognised.
* @see Integer#reverseBytes(int)
+ * @see Long#reverseBytes(long)
*/
- static UniformRandomProvider createReverseBytesIntProvider(final UniformRandomProvider rng) {
- // Note:
- // This always uses an IntProvider even if the underlying RNG is a LongProvider.
- // A LongProvider will produce 2 ints from 8 bytes of a long: 76543210 -> 7654 3210.
- // This will be reversed to output 2 ints as: 4567 0123.
- // This is a different output order than if reversing the entire long: 0123 4567.
- // The effect is to output the most significant bits from the long first, and
- // the least significant bits second. Thus the output of ints will be the same
- // on big-endian and little-endian platforms.
- return new IntProvider() {
- @Override
- public int next() {
- return Integer.reverseBytes(rng.nextInt());
- }
+ static UniformRandomProvider createReverseBytesProvider(final UniformRandomProvider rng) {
+ if (rng instanceof RandomIntSource) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return Integer.reverseBytes(rng.nextInt());
+ }
+
+ @Override
+ public String toString() {
+ return BYTE_REVERSED + rng.toString();
+ }
+ };
+ }
+ if (rng instanceof RandomLongSource) {
+ return new LongProvider() {
+ @Override
+ public long next() {
+ return Long.reverseBytes(rng.nextLong());
+ }
- @Override
- public String toString() {
- return "Byte-reversed " + rng.toString();
- }
- };
+ @Override
+ public String toString() {
+ return BYTE_REVERSED + rng.toString();
+ }
+ };
+ }
+ throw new ApplicationException(UNRECOGNISED_NATIVE_TYPE + rng);
}
/**
- * Wrap the random generator with an {@link IntProvider} that will reverse the bits
- * of the {@code int}.
+ * Wrap the random generator with a new instance that will reverse the bits of
+ * the native type. The input must be either a {@link RandomIntSource} or
+ * {@link RandomLongSource}.
*
* @param rng The random generator.
* @return the bit reversed random generator.
+ * @throws ApplicationException If the input source native type is not recognised.
* @see Integer#reverse(int)
+ * @see Long#reverse(long)
*/
- static UniformRandomProvider createReverseBitsIntProvider(final UniformRandomProvider rng) {
- return new IntProvider() {
- @Override
- public int next() {
- return Integer.reverse(rng.nextInt());
- }
+ static UniformRandomProvider createReverseBitsProvider(final UniformRandomProvider rng) {
+ if (rng instanceof RandomIntSource) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return Integer.reverse(rng.nextInt());
+ }
+
+ @Override
+ public String toString() {
+ return BIT_REVERSED + rng.toString();
+ }
+ };
+ }
+ if (rng instanceof RandomLongSource) {
+ return new LongProvider() {
+ @Override
+ public long next() {
+ return Long.reverse(rng.nextLong());
+ }
- @Override
- public String toString() {
- return "Bit-reversed " + rng.toString();
- }
- };
+ @Override
+ public String toString() {
+ return BIT_REVERSED + rng.toString();
+ }
+ };
+ }
+ throw new ApplicationException(UNRECOGNISED_NATIVE_TYPE + rng);
}
/**
- * Wrap the random generator with an {@link IntProvider} that will use the upper 32-bits
- * of the {@code long} from {@link UniformRandomProvider#nextLong()}.
+ * Wrap the random generator with an {@link IntProvider} that will use
+ * {@link UniformRandomProvider#nextInt()}.
+ * An input {@link RandomIntSource} is returned unmodified.
+ *
+ * @param rng The random generator.
+ * @return the int random generator.
+ */
+ static UniformRandomProvider createIntProvider(final UniformRandomProvider rng) {
+ if (!(rng instanceof RandomIntSource)) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return rng.nextInt();
+ }
+
+ @Override
+ public String toString() {
+ return "Int bits " + rng.toString();
+ }
+ };
+ }
+ return rng;
+ }
+
+ /**
+ * Wrap the random generator with an {@link IntProvider} that will use the upper
+ * 32-bits of the {@code long} from {@link UniformRandomProvider#nextLong()}.
+ * The input must be a {@link RandomLongSource}.
*
* @param rng The random generator.
* @return the upper bits random generator.
+ * @throws ApplicationException If the input source native type is not 64-bit.
*/
static UniformRandomProvider createLongUpperBitsIntProvider(final UniformRandomProvider rng) {
- return new IntProvider() {
- @Override
- public int next() {
- return (int) (rng.nextLong() >>> 32);
- }
+ if (rng instanceof RandomLongSource) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return (int) (rng.nextLong() >>> 32);
+ }
- @Override
- public String toString() {
- return "Long upper-bits " + rng.toString();
- }
- };
+ @Override
+ public String toString() {
+ return "Long upper-bits " + rng.toString();
+ }
+ };
+ }
+ throw new ApplicationException(NOT_LONG_SOURCE + rng);
}
/**
- * Wrap the random generator with an {@link IntProvider} that will use the lower 32-bits
- * of the {@code long} from {@link UniformRandomProvider#nextLong()}.
+ * Wrap the random generator with an {@link IntProvider} that will use the lower
+ * 32-bits of the {@code long} from {@link UniformRandomProvider#nextLong()}.
+ * The input must be a {@link RandomLongSource}.
*
* @param rng The random generator.
* @return the lower bits random generator.
+ * @throws ApplicationException If the input source native type is not 64-bit.
*/
static UniformRandomProvider createLongLowerBitsIntProvider(final UniformRandomProvider rng) {
- return new IntProvider() {
- @Override
- public int next() {
- return (int) rng.nextLong();
- }
+ if (rng instanceof RandomLongSource) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return (int) rng.nextLong();
+ }
- @Override
- public String toString() {
- return "Long lower-bits " + rng.toString();
- }
- };
+ @Override
+ public String toString() {
+ return "Long lower-bits " + rng.toString();
+ }
+ };
+ }
+ throw new ApplicationException(NOT_LONG_SOURCE + rng);
}
/**
- * Wrap the random generator with an {@link IntProvider} that will combine the bits
- * using a {@code xor} operation with a generated hash code.
+ * Wrap the random generator with a new instance that will combine the bits
+ * using a {@code xor} operation with a generated hash code. The input must be either
+ * a {@link RandomIntSource} or {@link RandomLongSource}.
*
- * <pre>{@code
+ * <pre>
+ * {@code
* System.identityHashCode(new Object()) ^ rng.nextInt()
- * }</pre>
+ * }
+ * </pre>
*
* Note: This generator will be slow.
*
* @param rng The random generator.
* @return the combined random generator.
+ * @throws ApplicationException If the input source native type is not recognised.
* @see System#identityHashCode(Object)
*/
- static UniformRandomProvider createHashCodeIntProvider(final UniformRandomProvider rng) {
- return new IntProvider() {
- @Override
- public int next() {
- return System.identityHashCode(new Object()) ^ rng.nextInt();
- }
+ static UniformRandomProvider createHashCodeProvider(final UniformRandomProvider rng) {
+ if (rng instanceof RandomIntSource) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return System.identityHashCode(new Object()) ^ rng.nextInt();
+ }
+
+ @Override
+ public String toString() {
+ return HASH_CODE + rng.toString();
+ }
+ };
+ }
+ if (rng instanceof RandomLongSource) {
+ return new LongProvider() {
+ @Override
+ public long next() {
+ final long mix = NumberFactory.makeLong(System.identityHashCode(new Object()),
+ System.identityHashCode(new Object()));
+ return mix ^ rng.nextLong();
+ }
- @Override
- public String toString() {
- return "HashCode ^ " + rng.toString();
- }
- };
+ @Override
+ public String toString() {
+ return HASH_CODE + rng.toString();
+ }
+ };
+ }
+ throw new ApplicationException(UNRECOGNISED_NATIVE_TYPE + rng);
}
/**
- * Wrap the random generator with an {@link IntProvider} that will combine the bits
+ * Wrap the random generator with a new instance that will combine the bits
* using a {@code xor} operation with the output from {@link ThreadLocalRandom}.
+ * The input must be either a {@link RandomIntSource} or {@link RandomLongSource}.
*
- * <pre>{@code
+ * <pre>
+ * {@code
* ThreadLocalRandom.current().nextInt() ^ rng.nextInt()
- * }</pre>
+ * }
+ * </pre>
*
* @param rng The random generator.
* @return the combined random generator.
+ * @throws ApplicationException If the input source native type is not recognised.
*/
- static UniformRandomProvider createThreadLocalRandomIntProvider(final UniformRandomProvider rng) {
- return new IntProvider() {
- @Override
- public int next() {
- return ThreadLocalRandom.current().nextInt() ^ rng.nextInt();
- }
+ static UniformRandomProvider createThreadLocalRandomProvider(final UniformRandomProvider rng) {
+ if (rng instanceof RandomIntSource) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return ThreadLocalRandom.current().nextInt() ^ rng.nextInt();
+ }
+
+ @Override
+ public String toString() {
+ return TLR_MIXED + rng.toString();
+ }
+ };
+ }
+ if (rng instanceof RandomLongSource) {
+ return new LongProvider() {
+ @Override
+ public long next() {
+ return ThreadLocalRandom.current().nextLong() ^ rng.nextLong();
+ }
- @Override
- public String toString() {
- return "ThreadLocalRandom ^ " + rng.toString();
- }
- };
+ @Override
+ public String toString() {
+ return TLR_MIXED + rng.toString();
+ }
+ };
+ }
+ throw new ApplicationException(UNRECOGNISED_NATIVE_TYPE + rng);
}
/**
* Combine the two random generators using a {@code xor} operations.
+ * The input must be either a {@link RandomIntSource} or {@link RandomLongSource}.
+ * The returned type will match the native output type of {@code rng1}.
*
- * <pre>{@code
+ * <pre>
+ * {@code
* rng1.nextInt() ^ rng2.nextInt()
- * }</pre>
+ * }
+ * </pre>
*
* @param rng1 The first random generator.
* @param rng2 The second random generator.
* @return the combined random generator.
+ * @throws ApplicationException If the input source native type is not recognised.
*/
- static UniformRandomProvider createXorIntProvider(final UniformRandomProvider rng1,
- final UniformRandomProvider rng2) {
- return new IntProvider() {
- @Override
- public int next() {
- return rng1.nextInt() ^ rng2.nextInt();
- }
+ static UniformRandomProvider createXorProvider(final UniformRandomProvider rng1,
+ final UniformRandomProvider rng2) {
+ if (rng1 instanceof RandomIntSource) {
+ return new IntProvider() {
+ @Override
+ public int next() {
+ return rng1.nextInt() ^ rng2.nextInt();
+ }
+
+ @Override
+ public String toString() {
+ return rng1.toString() + XOR + rng2.toString();
+ }
+ };
+ }
+ if (rng1 instanceof RandomLongSource) {
+ return new LongProvider() {
+ @Override
+ public long next() {
+ return rng1.nextLong() ^ rng2.nextLong();
+ }
- @Override
- public String toString() {
- return rng1.toString() + " ^ " + rng2.toString();
- }
- };
+ @Override
+ public String toString() {
+ return rng1.toString() + XOR + rng2.toString();
+ }
+ };
+ }
+ throw new ApplicationException(UNRECOGNISED_NATIVE_TYPE + rng1);
}
+
+ /**
+ * Create a new instance to write batches of byte data from the specified RNG to the
+ * specified output stream.
+ *
+ * <p>This will detect the native output type of the RNG and create an appropriate
+ * data output for the raw bytes. The input must be either a {@link RandomIntSource} or
+ * {@link RandomLongSource}.</p>
+ *
+ * <p>If the RNG is a {@link RandomLongSource} then the byte output can be 32-bit or 64-bit.
+ * If 32-bit then the 64-bit output will be written as if 2 {@code int} values were generated
+ * sequentially from the upper then lower 32-bits of the {@code long}. This setting is
+ * significant depending on the byte order. If using the Java standard big-endian
+ * representation the flag has no effect and the output will be the same. If using little
+ * endian the output bytes will be written as:</p>
+ *
+ * <pre>
+ * 76543210 -> 4567 0123
+ * </pre>
+ *
+ * @param rng The random generator.
+ * @param raw64 Set to true for 64-bit byte output.
+ * @param out Output stream.
+ * @param byteSize Number of bytes values to write.
+ * @param byteOrder Byte order.
+ * @return the data output
+ * @throws ApplicationException If the input source native type is not recognised.
+ */
+ static RngDataOutput createDataOutput(final UniformRandomProvider rng, boolean raw64,
+ OutputStream out, int byteSize, ByteOrder byteOrder) {
+ if (rng instanceof RandomIntSource) {
+ return RngDataOutput.ofInt(out, byteSize / 4, byteOrder);
+ }
+ if (rng instanceof RandomLongSource) {
+ return raw64 ?
+ RngDataOutput.ofLong(out, byteSize / 8, byteOrder) :
+ RngDataOutput.ofLongAsInt(out, byteSize / 8, byteOrder);
+ }
+ throw new ApplicationException(UNRECOGNISED_NATIVE_TYPE + rng);
+ }
+
/**
* Parses the argument into an object suitable for the RandomSource constructor. Supports:
*
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RngDataOutput.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RngDataOutput.java
new file mode 100644
index 0000000..90972d6
--- /dev/null
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/RngDataOutput.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.rng.examples.stress;
+
+import org.apache.commons.rng.UniformRandomProvider;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteOrder;
+
+/**
+ * A specialised data output class that combines the functionality of
+ * {@link java.io.DataOutputStream DataOutputStream} and
+ * {@link java.io.BufferedOutputStream BufferedOutputStream} to write byte data from a RNG
+ * to an OutputStream. Large blocks of byte data are written in a single operation for efficiency.
+ * The byte data endianness can be configured.
+ *
+ * <p>This class is the functional equivalent of:</p>
+ *
+ * <pre>
+ * <code>
+ * OutputStream out = ...
+ * UniformRandomProvider rng = ...
+ * int size = 2048;
+ * DataOutputStream sink = new DataOutputStream(new BufferedOutputStream(out, size * 4));
+ * for (int i = 0; i < size; i++) {
+ * sink.writeInt(rng.nextInt());
+ * }
+ *
+ * // Replaced with
+ * RngDataOutput output = RngDataOutput.ofInt(out, size, ByteOrder.BIG_ENDIAN);
+ * output.write(rng);
+ * </code>
+ * </pre>
+ *
+ * <p>Use of this class avoids the synchronized write operations in
+ * {@link java.io.BufferedOutputStream BufferedOutputStream}. In particular it avoids the
+ * 4 synchronized write operations to
+ * {@link java.io.BufferedOutputStream#write(int) BufferedOutputStream#write(int)} that
+ * occur for each {@code int} value that is written to
+ * {@link java.io.DataOutputStream#writeInt(int) DataOutputStream#writeInt(int)}.</p>
+ */
+abstract class RngDataOutput implements Closeable {
+ /** The data buffer. */
+ protected final byte[] buffer;
+
+ /** The underlying output stream. */
+ private final OutputStream out;
+
+ /**
+ * Write big-endian {@code int} data.
+ * <pre>
+ * 3210 -> 3210
+ * </pre>
+ */
+ private static class BIntRngDataOutput extends RngDataOutput {
+ /**
+ * @param out Output stream.
+ * @param size Buffer size.
+ */
+ BIntRngDataOutput(OutputStream out, int size) {
+ super(out, size);
+ }
+
+ @Override
+ public void fillBuffer(UniformRandomProvider rng) {
+ for (int i = 0; i < buffer.length; i += 4) {
+ writeIntBE(i, rng.nextInt());
+ }
+ }
+ }
+
+ /**
+ * Write little-endian {@code int} data.
+ * <pre>
+ * 3210 -> 0123
+ * </pre>
+ */
+ private static class LIntRngDataOutput extends RngDataOutput {
+ /**
+ * @param out Output stream.
+ * @param size Buffer size.
+ */
+ LIntRngDataOutput(OutputStream out, int size) {
+ super(out, size);
+ }
+
+ @Override
+ public void fillBuffer(UniformRandomProvider rng) {
+ for (int i = 0; i < buffer.length; i += 4) {
+ writeIntLE(i, rng.nextInt());
+ }
+ }
+ }
+
+ /**
+ * Write big-endian {@code long} data.
+ * <pre>
+ * 76543210 -> 76543210
+ * </pre>
+ */
+ private static class BLongRngDataOutput extends RngDataOutput {
+ /**
+ * @param out Output stream.
+ * @param size Buffer size.
+ */
+ BLongRngDataOutput(OutputStream out, int size) {
+ super(out, size);
+ }
+
+ @Override
+ public void fillBuffer(UniformRandomProvider rng) {
+ for (int i = 0; i < buffer.length; i += 8) {
+ writeLongBE(i, rng.nextLong());
+ }
+ }
+ }
+
+ /**
+ * Write little-endian {@code long} data.
+ * <pre>
+ * 76543210 -> 01234567
+ * </pre>
+ */
+ private static class LLongRngDataOutput extends RngDataOutput {
+ /**
+ * @param out Output stream.
+ * @param size Buffer size.
+ */
+ LLongRngDataOutput(OutputStream out, int size) {
+ super(out, size);
+ }
+
+ @Override
+ public void fillBuffer(UniformRandomProvider rng) {
+ for (int i = 0; i < buffer.length; i += 8) {
+ writeLongLE(i, rng.nextLong());
+ }
+ }
+ }
+
+ /**
+ * Write {@code long} data as two little-endian {@code int} values.
+ * <pre>
+ * 76543210 -> 4567 0123
+ * </pre>
+ *
+ * <p>This is a specialisation that allows the Java big-endian representation to be split
+ * into two little-endian values in the original order of upper then lower bits. In
+ * comparison the {@link LLongRngDataOutput} will output the same data as:
+ *
+ * <pre>
+ * 76543210 -> 0123 4567
+ * </pre>
+ */
+ private static class LLongAsIntRngDataOutput extends RngDataOutput {
+ /**
+ * @param out Output stream.
+ * @param size Buffer size.
+ */
+ LLongAsIntRngDataOutput(OutputStream out, int size) {
+ super(out, size);
+ }
+
+ @Override
+ public void fillBuffer(UniformRandomProvider rng) {
+ for (int i = 0; i < buffer.length; i += 8) {
+ writeLongAsIntLE(i, rng.nextLong());
+ }
+ }
+ }
+
+ /**
+ * Create a new instance.
+ *
+ * @param out Output stream.
+ * @param size Buffer size.
+ */
+ RngDataOutput(OutputStream out, int size) {
+ this.out = out;
+ buffer = new byte[size];
+ }
+
+ /**
+ * Write the configured amount of byte data from the specified RNG to the output.
+ *
+ * @param rng Source of randomness.
+ * @exception IOException if an I/O error occurs.
+ */
+ public void write(UniformRandomProvider rng) throws IOException {
+ fillBuffer(rng);
+ out.write(buffer);
+ }
+
+ /**
+ * Fill the buffer from the specified RNG.
+ *
+ * @param rng Source of randomness.
+ */
+ public abstract void fillBuffer(UniformRandomProvider rng);
+
+ /**
+ * Writes an {@code int} to the buffer as four bytes, high byte first (big-endian).
+ *
+ * @param index the index to start writing.
+ * @param value an {@code int} to be written.
+ */
+ final void writeIntBE(int index, int value) {
+ buffer[index ] = (byte) (value >>> 24);
+ buffer[index + 1] = (byte) (value >>> 16);
+ buffer[index + 2] = (byte) (value >>> 8);
+ buffer[index + 3] = (byte) value;
+ }
+
+ /**
+ * Writes an {@code int} to the buffer as four bytes, low byte first (little-endian).
+ *
+ * @param index the index to start writing.
+ * @param value an {@code int} to be written.
+ */
+ final void writeIntLE(int index, int value) {
+ buffer[index ] = (byte) value;
+ buffer[index + 1] = (byte) (value >>> 8);
+ buffer[index + 2] = (byte) (value >>> 16);
+ buffer[index + 3] = (byte) (value >>> 24);
+ }
+
+ /**
+ * Writes an {@code long} to the buffer as eight bytes, high byte first (big-endian).
+ *
+ * @param index the index to start writing.
+ * @param value an {@code long} to be written.
+ */
+ final void writeLongBE(int index, long value) {
+ buffer[index ] = (byte) (value >>> 56);
+ buffer[index + 1] = (byte) (value >>> 48);
+ buffer[index + 2] = (byte) (value >>> 40);
+ buffer[index + 3] = (byte) (value >>> 32);
+ buffer[index + 4] = (byte) (value >>> 24);
+ buffer[index + 5] = (byte) (value >>> 16);
+ buffer[index + 6] = (byte) (value >>> 8);
+ buffer[index + 7] = (byte) value;
+ }
+
+ /**
+ * Writes an {@code long} to the buffer as eight bytes, low byte first (big-endian).
+ *
+ * @param index the index to start writing.
+ * @param value an {@code long} to be written.
+ */
+ final void writeLongLE(int index, long value) {
+ buffer[index ] = (byte) value;
+ buffer[index + 1] = (byte) (value >>> 8);
+ buffer[index + 2] = (byte) (value >>> 16);
+ buffer[index + 3] = (byte) (value >>> 24);
+ buffer[index + 4] = (byte) (value >>> 32);
+ buffer[index + 5] = (byte) (value >>> 40);
+ buffer[index + 6] = (byte) (value >>> 48);
+ buffer[index + 7] = (byte) (value >>> 56);
+ }
+
+ /**
+ * Writes an {@code long} to the buffer as two integers of four bytes, each
+ * low byte first (big-endian).
+ *
+ * @param index the index to start writing.
+ * @param value an {@code long} to be written.
+ */
+ final void writeLongAsIntLE(int index, long value) {
+ buffer[index ] = (byte) (value >>> 32);
+ buffer[index + 1] = (byte) (value >>> 40);
+ buffer[index + 2] = (byte) (value >>> 48);
+ buffer[index + 3] = (byte) (value >>> 56);
+ buffer[index + 4] = (byte) value;
+ buffer[index + 5] = (byte) (value >>> 8);
+ buffer[index + 6] = (byte) (value >>> 16);
+ buffer[index + 7] = (byte) (value >>> 24);
+ }
+
+ @Override
+ public void close() throws IOException {
+ try (OutputStream ostream = out) {
+ ostream.flush();
+ }
+ }
+
+ /**
+ * Create a new instance to write batches of data from
+ * {@link UniformRandomProvider#nextInt()} to the specified output.
+ *
+ * @param out Output stream.
+ * @param size Number of values to write.
+ * @param byteOrder Byte order.
+ * @return the data output
+ */
+ @SuppressWarnings("resource")
+ static RngDataOutput ofInt(OutputStream out, int size, ByteOrder byteOrder) {
+ // Ensure the buffer is positive and a factor of 4
+ final int bytes = Math.max(size * 4, 4);
+ return byteOrder == ByteOrder.LITTLE_ENDIAN ?
+ new LIntRngDataOutput(out, bytes) :
+ new BIntRngDataOutput(out, bytes);
+ }
+
+ /**
+ * Create a new instance to write batches of data from
+ * {@link UniformRandomProvider#nextLong()} to the specified output.
+ *
+ * @param out Output stream.
+ * @param size Number of values to write.
+ * @param byteOrder Byte order.
+ * @return the data output
+ */
+ @SuppressWarnings("resource")
+ static RngDataOutput ofLong(OutputStream out, int size, ByteOrder byteOrder) {
+ // Ensure the buffer is positive and a factor of 8
+ final int bytes = Math.max(size * 8, 8);
+ return byteOrder == ByteOrder.LITTLE_ENDIAN ?
+ new LLongRngDataOutput(out, bytes) :
+ new BLongRngDataOutput(out, bytes);
+ }
+
+ /**
+ * Create a new instance to write batches of data from
+ * {@link UniformRandomProvider#nextLong()} to the specified output as two sequential
+ * {@code int} values.
+ *
+ * <p>This will output the following bytes:</p>
+ *
+ * <pre>
+ * // Little-endian
+ * 76543210 -> 4567 0123
+ *
+ * // Big-endian
+ * 76543210 -> 7654 3210
+ * </pre>
+ *
+ * <p>This ensures the output from the generator is the original upper then lower order bits
+ * for each endianess.
+ *
+ * @param out Output stream.
+ * @param size Number of values to write.
+ * @param byteOrder Byte order.
+ * @return the data output
+ */
+ @SuppressWarnings("resource")
+ static RngDataOutput ofLongAsInt(OutputStream out, int size, ByteOrder byteOrder) {
+ // Ensure the buffer is positive and a factor of 8
+ final int bytes = Math.max(size * 8, 8);
+ return byteOrder == ByteOrder.LITTLE_ENDIAN ?
+ new LLongAsIntRngDataOutput(out, bytes) :
+ new BLongRngDataOutput(out, bytes);
+ }
+}
diff --git a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/StressTestCommand.java b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/StressTestCommand.java
index aa02d34..b8b2c5d 100644
--- a/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/StressTestCommand.java
+++ b/commons-rng-examples/examples-stress/src/main/java/org/apache/commons/rng/examples/stress/StressTestCommand.java
@@ -17,6 +17,7 @@
package org.apache.commons.rng.examples.stress;
import org.apache.commons.rng.UniformRandomProvider;
+import org.apache.commons.rng.core.source64.RandomLongSource;
import org.apache.commons.rng.simple.RandomSource;
import picocli.CommandLine.Command;
@@ -24,10 +25,8 @@ import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;
-import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
-import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.ByteOrder;
@@ -119,6 +118,11 @@ class StressTestCommand implements Callable<Void> {
"Two threads are required per task."})
private int taskCount = Math.max(1, Runtime.getRuntime().availableProcessors() / 2);
+ /** The size of the byte buffer for the binary data. */
+ @Option(names = {"--buffer-size"},
+ description = {"Byte-buffer size for the transferred data (default: ${DEFAULT-VALUE})."})
+ private int bufferSize = 8192;
+
/** The output byte order of the binary data. */
@Option(names = {"-b", "--byte-order"},
description = {"Byte-order of the transferred data (default: ${DEFAULT-VALUE}).",
@@ -143,6 +147,14 @@ class StressTestCommand implements Callable<Void> {
description = {"Use the lower 32-bits from the 64-bit long output."})
private boolean longLowBits;
+ /** Flag to use 64-bit long output. */
+ @Option(names = {"--raw64"},
+ description = {"Use 64-bit output (default is 32-bit).",
+ "This requires a 64-bit testing application and native 64-bit generators.",
+ "In 32-bit mode the output uses the upper then lower bits of 64-bit " +
+ "generators sequentially, each appropriately byte reversed for the platform."})
+ private boolean raw64;
+
/**
* Flag to indicate the output should be combined with a hashcode from a new object.
* This is a method previously used in the
@@ -456,30 +468,36 @@ class StressTestCommand implements Callable<Void> {
// Create the generator. Explicitly create a seed so it can be recorded.
final byte[] seed = testData.getRandomSource().createSeed();
UniformRandomProvider rng = testData.createRNG(seed);
- // Combined generators must be created first
+
+ // Upper or lower bits from 64-bit generators must be created first.
+ // This will throw if not a 64-bit generator.
if (longHighBits) {
rng = RNGUtils.createLongUpperBitsIntProvider(rng);
} else if (longLowBits) {
rng = RNGUtils.createLongLowerBitsIntProvider(rng);
}
+
+ // Combination generators. Mainly used for testing.
+ // These operations maintain the native output type (int/long).
if (xorHashCode) {
- rng = RNGUtils.createHashCodeIntProvider(rng);
+ rng = RNGUtils.createHashCodeProvider(rng);
}
if (xorThreadLocalRandom) {
- rng = RNGUtils.createThreadLocalRandomIntProvider(rng);
+ rng = RNGUtils.createThreadLocalRandomProvider(rng);
}
if (xorRandomSource != null) {
- rng = RNGUtils.createXorIntProvider(
+ rng = RNGUtils.createXorProvider(
RandomSource.create(xorRandomSource),
rng);
}
if (reverseBits) {
- rng = RNGUtils.createReverseBitsIntProvider(rng);
- }
- // Manipulation of the bytes for the platform is done on the entire generator
- if (byteOrder.equals(ByteOrder.LITTLE_ENDIAN)) {
- rng = RNGUtils.createReverseBytesIntProvider(rng);
+ rng = RNGUtils.createReverseBitsProvider(rng);
}
+
+ // -------
+ // Note: Manipulation of the byte order for the platform is done during output.
+ // -------
+
// Run the test
final Runnable r = new StressTestTask(testData.getRandomSource(), rng, seed,
output, command, this, progressTracker);
@@ -568,8 +586,8 @@ class StressTestCommand implements Callable<Void> {
/** The progress tracker. */
private final ProgressTracker progressTracker;
- /** The count of numbers used by the sub-process. */
- private long numbersUsed;
+ /** The count of bytes used by the sub-process. */
+ private long bytesUsed;
/**
* Creates the task.
@@ -643,19 +661,20 @@ class StressTestCommand implements Callable<Void> {
builder.redirectOutput(ProcessBuilder.Redirect.appendTo(output));
builder.redirectErrorStream(true);
final Process testingProcess = builder.start();
- final DataOutputStream sink = new DataOutputStream(
- new BufferedOutputStream(testingProcess.getOutputStream()));
-
- try {
- while (true) {
- sink.writeInt(rng.nextInt());
- numbersUsed++;
+ // Use a custom data output to write the RNG.
+ try (RngDataOutput sink = RNGUtils.createDataOutput(rng, cmd.raw64,
+ testingProcess.getOutputStream(), cmd.bufferSize, cmd.byteOrder)) {
+ for (;;) {
+ sink.write(rng);
+ bytesUsed++;
}
} catch (final IOException ignored) {
// Hopefully getting here when the analyzing software terminates.
}
+ bytesUsed *= cmd.bufferSize;
+
// Get the exit value
return ProcessUtils.getExitValue(testingProcess);
}
@@ -686,7 +705,11 @@ class StressTestCommand implements Callable<Void> {
.append(' ').append(System.getProperty("os.version"))
.append(' ').append(System.getProperty("os.arch")).append(N)
.append(C).append("Native byte-order: ").append(ByteOrder.nativeOrder()).append(N)
- .append(C).append(N)
+ .append(C).append("Output byte-order: ").append(cmd.byteOrder).append(N);
+ if (rng instanceof RandomLongSource) {
+ sb.append(C).append("64-bit output: ").append(cmd.raw64).append(N);
+ }
+ sb.append(C).append(N)
.append(C).append("Analyzer: ");
for (final String s : command) {
sb.append(s).append(' ');
@@ -715,9 +738,9 @@ class StressTestCommand implements Callable<Void> {
appendDate(sb, "End").append(C).append(N);
sb.append(C).append("Exit value: ").append(exitValue).append(N)
- .append(C).append("Numbers used: ").append(numbersUsed)
- .append(" >= 2^").append(log2(numbersUsed))
- .append(" (").append(bytesToString(numbersUsed * 4)).append(')').append(N)
+ .append(C).append("Bytes used: ").append(bytesUsed)
+ .append(" >= 2^").append(log2(bytesUsed))
+ .append(" (").append(bytesToString(bytesUsed)).append(')').append(N)
.append(C).append(N);
final double duration = nanoTime * 1e-9 / 60;
diff --git a/commons-rng-examples/examples-stress/src/test/java/org/apache/commons/rng/examples/stress/RngDataOutputTest.java b/commons-rng-examples/examples-stress/src/test/java/org/apache/commons/rng/examples/stress/RngDataOutputTest.java
new file mode 100644
index 0000000..a2e50ed
--- /dev/null
+++ b/commons-rng-examples/examples-stress/src/test/java/org/apache/commons/rng/examples/stress/RngDataOutputTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.rng.examples.stress;
+
+import org.apache.commons.rng.UniformRandomProvider;
+import org.apache.commons.rng.core.source32.IntProvider;
+import org.apache.commons.rng.core.source32.RandomIntSource;
+import org.apache.commons.rng.core.source64.RandomLongSource;
+import org.apache.commons.rng.simple.RandomSource;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteOrder;
+import java.util.function.BiConsumer;
+import java.util.function.UnaryOperator;
+
+/**
+ * Tests for {@link RngDataOutput}.
+ */
+public class RngDataOutputTest {
+ /**
+ * A factory for creating RngDataOutput objects.
+ */
+ interface RngDataOutputFactory {
+ /**
+ * Create a new instance.
+ *
+ * @param out Output stream.
+ * @param size Number of values to write.
+ * @param byteOrder Byte order.
+ * @return the data output
+ */
+ RngDataOutput create(OutputStream out, int size, ByteOrder byteOrder);
+ }
+
+ @Test
+ public void testIntBigEndian() throws IOException {
+ assertRngOutput(RandomSource.PCG_MCG_XSH_RS_32,
+ UnaryOperator.identity(),
+ RngDataOutputTest::writeInt,
+ RngDataOutput::ofInt, ByteOrder.BIG_ENDIAN);
+ }
+
+ @Test
+ public void testIntLittleEndian() throws IOException {
+ assertRngOutput(RandomSource.PCG_MCG_XSH_RS_32,
+ RNGUtils::createReverseBytesProvider,
+ RngDataOutputTest::writeInt,
+ RngDataOutput::ofInt, ByteOrder.LITTLE_ENDIAN);
+ }
+
+ @Test
+ public void testLongBigEndian() throws IOException {
+ assertRngOutput(RandomSource.SPLIT_MIX_64,
+ UnaryOperator.identity(),
+ RngDataOutputTest::writeLong,
+ RngDataOutput::ofLong, ByteOrder.BIG_ENDIAN);
+ }
+
+ @Test
+ public void testLongLittleEndian() throws IOException {
+ assertRngOutput(RandomSource.SPLIT_MIX_64,
+ RNGUtils::createReverseBytesProvider,
+ RngDataOutputTest::writeLong,
+ RngDataOutput::ofLong, ByteOrder.LITTLE_ENDIAN);
+ }
+
+ @Test
+ public void testLongAsIntBigEndian() throws IOException {
+ assertRngOutput(RandomSource.SPLIT_MIX_64,
+ // Convert SplitMix64 to an int provider so it is detected as requiring double the
+ // length output.
+ rng -> new IntProvider() {
+ @Override
+ public int next() {
+ return rng.nextInt();
+ }
+ },
+ RngDataOutputTest::writeInt,
+ RngDataOutput::ofLongAsInt, ByteOrder.BIG_ENDIAN);
+ }
+
+ @Test
+ public void testLongAsIntLittleEndian() throws IOException {
+ assertRngOutput(RandomSource.SPLIT_MIX_64,
+ // Convert SplitMix64 to an int provider so it is detected as requiring double the
+ // length output. Then reverse the bytes.
+ rng -> new IntProvider() {
+ @Override
+ public int next() {
+ return Integer.reverseBytes(rng.nextInt());
+ }
+ },
+ RngDataOutputTest::writeInt,
+ RngDataOutput::ofLongAsInt, ByteOrder.LITTLE_ENDIAN);
+ }
+
+ private static void writeInt(DataOutputStream sink, UniformRandomProvider rng) {
+ try {
+ sink.writeInt(rng.nextInt());
+ } catch (IOException e) {
+ Assert.fail();
+ }
+ }
+
+ private static void writeLong(DataOutputStream sink, UniformRandomProvider rng) {
+ try {
+ sink.writeLong(rng.nextLong());
+ } catch (IOException e) {
+ Assert.fail();
+ }
+ }
+
+ /**
+ * Assert the byte output from the source is the same. Creates two instances of the same
+ * RNG with the same seed. The first is converted to a new RNG using the {@code rngConverter}.
+ * This is used to output raw bytes using a {@link DataOutputStream} via the specified
+ * {@code pipe}. The second is used to output raw bytes via the {@link RngDataOutput} class
+ * created using the {@code factory}.
+ *
+ * <p>The random source should output native {@code int} or {@code long} values.
+ *
+ * @param source Random source.
+ * @param rngConverter Converter for the raw RNG.
+ * @param pipe Pipe to send data from the RNG to the DataOutputStream.
+ * @param factory Factory for the RngDataOutput.
+ * @param byteOrder Byte order for the RngDataOutput.
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private static void assertRngOutput(RandomSource source,
+ UnaryOperator<UniformRandomProvider> rngConverter,
+ BiConsumer<DataOutputStream, UniformRandomProvider> pipe,
+ RngDataOutputFactory factory,
+ ByteOrder byteOrder) throws IOException {
+ final long seed = RandomSource.createLong();
+ UniformRandomProvider rng1 = RandomSource.create(source, seed);
+ UniformRandomProvider rng2 = RandomSource.create(source, seed);
+ final int size = 37;
+ for (int repeats = 1; repeats <= 2; repeats++) {
+ byte[] expected = createBytes(rng1, size, repeats, rngConverter, pipe);
+ byte[] actual = writeRngOutput(rng2, size, repeats, byteOrder, factory);
+ Assert.assertArrayEquals(expected, actual);
+ }
+ }
+
+ /**
+ * Convert the RNG and then creates bytes from the RNG using the pipe.
+ *
+ * @param rng RNG.
+ * @param size The number of values to send to the pipe.
+ * @param repeats The number of repeat iterations.
+ * @param rngConverter Converter for the raw RNG.
+ * @param pipe Pipe to send data from the RNG to the DataOutputStream.
+ * @return the bytes
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private static byte[] createBytes(UniformRandomProvider rng, int size, int repeats,
+ UnaryOperator<UniformRandomProvider> rngConverter,
+ BiConsumer<DataOutputStream, UniformRandomProvider> pipe) throws IOException {
+ UniformRandomProvider rng2 = rngConverter.apply(rng);
+ // If the factory converts to an IntProvider then output twice the size
+ if (rng instanceof RandomLongSource && rng2 instanceof RandomIntSource) {
+ size *= 2;
+ }
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream sink = new DataOutputStream(out)) {
+ for (int j = 0; j < repeats; j++) {
+ for (int i = 0; i < size; i++) {
+ pipe.accept(sink, rng2);
+ }
+ }
+ }
+ return out.toByteArray();
+ }
+
+ /**
+ * Write the RNG to the RngDataOutput built by the factory.
+ *
+ * @param rng RNG.
+ * @param size The number of values to send to the RngDataOutput.
+ * @param repeats The number of repeat iterations.
+ * @param byteOrder Byte order for the RngDataOutput.
+ * @param factory Factory for the RngDataOutput.
+ * @return the bytes
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private static byte[] writeRngOutput(UniformRandomProvider rng, int size, int repeats,
+ ByteOrder byteOrder, RngDataOutputFactory factory) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (RngDataOutput sink = factory.create(out, size, byteOrder)) {
+ for (int j = 0; j < repeats; j++) {
+ sink.write(rng);
+ }
+ }
+ return out.toByteArray();
+ }
+}