You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/12/14 17:48:41 UTC
[4/8] impala git commit: IMPALA-5754: Improve randomness of
rand()/random()
IMPALA-5754: Improve randomness of rand()/random()
Currently implementation of rand/random built-in functions
use rand_r of C library. We recognized its randomness was poor.
pcg32 of third party library shows better randomness than rand_r.
Testing:
Revise unit test in expr-test
Add E2E test to random.test
Change-Id: Idafdd5fe7502ff242c76a91a815c565146108684
Reviewed-on: http://gerrit.cloudera.org:8080/8355
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4feb4f3a
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4feb4f3a
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4feb4f3a
Branch: refs/heads/master
Commit: 4feb4f3a549a79646b52f6aabb75c3f529d5cfe8
Parents: 9a6e5fa
Author: Jinchul <ji...@gmail.com>
Authored: Mon Oct 23 15:24:12 2017 +0900
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Dec 13 10:04:40 2017 +0000
----------------------------------------------------------------------
LICENSE.txt | 1 +
be/src/exprs/expr-test.cc | 25 +-
be/src/exprs/math-functions-ir.cc | 36 +-
be/src/thirdparty/pcg-cpp-0.98/LICENSE.txt | 201 ++
be/src/thirdparty/pcg-cpp-0.98/README.md | 52 +
.../pcg-cpp-0.98/include/pcg_extras.hpp | 637 +++++++
.../pcg-cpp-0.98/include/pcg_random.hpp | 1751 ++++++++++++++++++
.../pcg-cpp-0.98/include/pcg_uint128.hpp | 750 ++++++++
bin/rat_exclude_files.txt | 2 +
bin/run_clang_tidy.sh | 3 +-
.../queries/QueryTest/alloc-fail-init.test | 2 +-
.../queries/QueryTest/random.test | 24 +
tests/query_test/test_queries.py | 6 +
13 files changed, 3458 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/4feb4f3a/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index aa211ad..d4c4307 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -669,6 +669,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
be/src/thirdparty/mustache: Apache 2.0 license
+be/src/thirdparty/pcg-cpp-0.98: Apache 2.0 license
be/src/expr/hll-bias.h: Apache 2.0 license
shell/ext-py/sasl-0.1.1: Apache 2.0 license
http://git-wip-us.apache.org/repos/asf/impala/blob/4feb4f3a/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 5670bcf..35599c0 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -1132,6 +1132,12 @@ int64_t ExprTest::ConvertValue<int64_t>(const string& value) {
}
template <>
+double ExprTest::ConvertValue<double>(const string& value) {
+ StringParser::ParseResult result;
+ return StringParser::StringToFloat<double>(value.data(), value.size(), &result);
+}
+
+template <>
TimestampValue ExprTest::ConvertValue<TimestampValue>(const string& value) {
return TimestampValue::Parse(value.data(), value.size());
}
@@ -4895,16 +4901,15 @@ TEST_F(ExprTest, MathFunctions) {
TestValue("dsqrt(81.0)", TYPE_DOUBLE, 9);
// Run twice to test deterministic behavior.
- uint32_t seed = 0;
- double expected = static_cast<double>(rand_r(&seed)) / static_cast<double>(RAND_MAX);
- TestValue("rand()", TYPE_DOUBLE, expected);
- TestValue("rand()", TYPE_DOUBLE, expected);
- TestValue("random()", TYPE_DOUBLE, expected); // Test alias
- seed = 1234;
- expected = static_cast<double>(rand_r(&seed)) / static_cast<double>(RAND_MAX);
- TestValue("rand(1234)", TYPE_DOUBLE, expected);
- TestValue("rand(1234)", TYPE_DOUBLE, expected);
- TestValue("random(1234)", TYPE_DOUBLE, expected); // Test alias
+ for (uint32_t seed : {0, 1234}) {
+ stringstream rand, random;
+ rand << "rand(" << seed << ")";
+ random << "random(" << seed << ")";
+ const double expected_result = ConvertValue<double>(GetValue(rand.str(),
+ TYPE_DOUBLE));
+ TestValue(rand.str(), TYPE_DOUBLE, expected_result);
+ TestValue(random.str(), TYPE_DOUBLE, expected_result); // Test alias
+ }
// Test bigint param.
TestValue("pmod(10, 3)", TYPE_BIGINT, 1);
http://git-wip-us.apache.org/repos/asf/impala/blob/4feb4f3a/be/src/exprs/math-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/math-functions-ir.cc b/be/src/exprs/math-functions-ir.cc
index cc86d5b..531bf24 100644
--- a/be/src/exprs/math-functions-ir.cc
+++ b/be/src/exprs/math-functions-ir.cc
@@ -18,6 +18,7 @@
#include "exprs/math-functions.h"
#include <iomanip>
+#include <random>
#include <sstream>
#include <math.h>
@@ -27,6 +28,7 @@
#include "util/string-parser.h"
#include "runtime/runtime-state.h"
#include "runtime/string-value.inline.h"
+#include "thirdparty/pcg-cpp-0.98/include/pcg_random.hpp"
#include "common/names.h"
@@ -148,9 +150,7 @@ DoubleVal MathFunctions::Pow(FunctionContext* ctx, const DoubleVal& base,
void MathFunctions::RandPrepare(
FunctionContext* ctx, FunctionContext::FunctionStateScope scope) {
if (scope == FunctionContext::THREAD_LOCAL) {
- uint32_t* seed = ctx->Allocate<uint32_t>();
- RETURN_IF_NULL(ctx, seed);
- ctx->SetFunctionState(scope, seed);
+ uint32_t seed = 0;
if (ctx->GetNumArgs() == 1) {
// This is a call to RandSeed, initialize the seed
// TODO: should we support non-constant seed?
@@ -160,26 +160,22 @@ void MathFunctions::RandPrepare(
}
DCHECK_EQ(ctx->GetArgType(0)->type, FunctionContext::TYPE_BIGINT);
BigIntVal* seed_arg = static_cast<BigIntVal*>(ctx->GetConstantArg(0));
- if (seed_arg->is_null) {
- *seed = 0;
- } else {
- *seed = seed_arg->val;
- }
- } else {
- // This is a call to Rand, initialize seed to 0
- // TODO: can we change this behavior? This is stupid.
- *seed = 0;
+ if (!seed_arg->is_null) seed = seed_arg->val;
}
+ pcg32* generator = ctx->Allocate<pcg32>();
+ RETURN_IF_NULL(ctx, generator);
+ ctx->SetFunctionState(scope, generator);
+ new (generator) pcg32(seed);
}
}
DoubleVal MathFunctions::Rand(FunctionContext* ctx) {
- uint32_t* seed =
- reinterpret_cast<uint32_t*>(ctx->GetFunctionState(FunctionContext::THREAD_LOCAL));
- DCHECK(seed != NULL);
- *seed = rand_r(seed);
- // Normalize to [0,1].
- return DoubleVal(static_cast<double>(*seed) / RAND_MAX);
+ pcg32* const generator =
+ reinterpret_cast<pcg32*>(ctx->GetFunctionState(FunctionContext::THREAD_LOCAL));
+ DCHECK(generator != nullptr);
+ static const double min = 0, max = 1;
+ std::uniform_real_distribution<double> distribution(min, max);
+ return DoubleVal(distribution(*generator));
}
DoubleVal MathFunctions::RandSeed(FunctionContext* ctx, const BigIntVal& seed) {
@@ -190,9 +186,9 @@ DoubleVal MathFunctions::RandSeed(FunctionContext* ctx, const BigIntVal& seed) {
void MathFunctions::RandClose(FunctionContext* ctx,
FunctionContext::FunctionStateScope scope) {
if (scope == FunctionContext::THREAD_LOCAL) {
- uint8_t* seed = reinterpret_cast<uint8_t*>(
+ uint8_t* generator = reinterpret_cast<uint8_t*>(
ctx->GetFunctionState(FunctionContext::THREAD_LOCAL));
- ctx->Free(seed);
+ ctx->Free(generator);
ctx->SetFunctionState(FunctionContext::THREAD_LOCAL, nullptr);
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/4feb4f3a/be/src/thirdparty/pcg-cpp-0.98/LICENSE.txt
----------------------------------------------------------------------
diff --git a/be/src/thirdparty/pcg-cpp-0.98/LICENSE.txt b/be/src/thirdparty/pcg-cpp-0.98/LICENSE.txt
new file mode 100644
index 0000000..8dada3e
--- /dev/null
+++ b/be/src/thirdparty/pcg-cpp-0.98/LICENSE.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/impala/blob/4feb4f3a/be/src/thirdparty/pcg-cpp-0.98/README.md
----------------------------------------------------------------------
diff --git a/be/src/thirdparty/pcg-cpp-0.98/README.md b/be/src/thirdparty/pcg-cpp-0.98/README.md
new file mode 100644
index 0000000..f2858db
--- /dev/null
+++ b/be/src/thirdparty/pcg-cpp-0.98/README.md
@@ -0,0 +1,52 @@
+# PCG Random Number Generation, C++ Edition
+
+[PCG-Random website]: http://www.pcg-random.org
+
+This code provides an implementation of the PCG family of random number
+generators, which are fast, statistically excellent, and offer a number of
+useful features.
+
+Full details can be found at the [PCG-Random website]. This version
+of the code provides many family members -- if you just want one
+simple generator, you may prefer the minimal C version of the library.
+
+There are two kinds of generator, normal generators and extended generators.
+Extended generators provide *k* dimensional equidistribution and can perform
+party tricks, but generally speaking most people only need the normal
+generators.
+
+There are two ways to access the generators, using a convenience typedef
+or by using the underlying templates directly (similar to C++11's `std::mt19937` typedef vs its `std::mersenne_twister_engine` template). For most users, the convenience typedef is what you want, and probably you're fine with `pcg32` for 32-bit numbers. If you want 64-bit numbers, either use `pcg64` (or, if you're on a 32-bit system, making 64 bits from two calls to `pcg32_k2` may be faster).
+
+## Documentation and Examples
+
+Visit [PCG-Random website] for information on how to use this library, or look
+at the sample code in the `sample` directory -- hopefully it should be fairly
+self explanatory.
+
+## Building
+
+The code is written in C++11, as an include-only library (i.e., there is
+nothing you need to build). There are some provided demo programs and tests
+however. On a Unix-style system (e.g., Linux, Mac OS X) you should be able
+to just type type
+
+ make
+
+To build the demo programs.
+
+## Testing
+
+Run
+
+ make test
+
+## Directory Structure
+
+The directories are arranged as follows:
+
+* `include` -- contains `pcg_random.hpp` and supporting include files
+* `test-high` -- test code for the high-level API where the functions have
+ shorter, less scary-looking names.
+* `sample` -- sample code, some similar to the code in `test-high` but more
+ human readable, some other examples too
http://git-wip-us.apache.org/repos/asf/impala/blob/4feb4f3a/be/src/thirdparty/pcg-cpp-0.98/include/pcg_extras.hpp
----------------------------------------------------------------------
diff --git a/be/src/thirdparty/pcg-cpp-0.98/include/pcg_extras.hpp b/be/src/thirdparty/pcg-cpp-0.98/include/pcg_extras.hpp
new file mode 100644
index 0000000..ec3e569
--- /dev/null
+++ b/be/src/thirdparty/pcg-cpp-0.98/include/pcg_extras.hpp
@@ -0,0 +1,637 @@
+/*
+ * PCG Random Number Generation for C++
+ *
+ * Copyright 2014 Melissa O'Neill <on...@pcg-random.org>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * For additional information about the PCG random number generation scheme,
+ * including its license and other licensing options, visit
+ *
+ * http://www.pcg-random.org
+ */
+
+/*
+ * This file provides support code that is useful for random-number generation
+ * but not specific to the PCG generation scheme, including:
+ * - 128-bit int support for platforms where it isn't available natively
+ * - bit twiddling operations
+ * - I/O of 128-bit and 8-bit integers
+ * - Handling the evilness of SeedSeq
+ * - Support for efficiently producing random numbers less than a given
+ * bound
+ */
+
+#ifndef PCG_EXTRAS_HPP_INCLUDED
+#define PCG_EXTRAS_HPP_INCLUDED 1
+
+#include <cinttypes>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <cassert>
+#include <limits>
+#include <iostream>
+#include <type_traits>
+#include <utility>
+#include <locale>
+#include <iterator>
+#include <utility>
+
+#ifdef __GNUC__
+ #include <cxxabi.h>
+#endif
+
+/*
+ * Abstractions for compiler-specific directives
+ */
+
+#ifdef __GNUC__
+ #define PCG_NOINLINE __attribute__((noinline))
+#else
+ #define PCG_NOINLINE
+#endif
+
+/*
+ * Some members of the PCG library use 128-bit math. When compiling on 64-bit
+ * platforms, both GCC and Clang provide 128-bit integer types that are ideal
+ * for the job.
+ *
+ * On 32-bit platforms (or with other compilers), we fall back to a C++
+ * class that provides 128-bit unsigned integers instead. It may seem
+ * like we're reinventing the wheel here, because libraries already exist
+ * that support large integers, but most existing libraries provide a very
+ * generic multiprecision code, but here we're operating at a fixed size.
+ * Also, most other libraries are fairly heavyweight. So we use a direct
+ * implementation. Sadly, it's much slower than hand-coded assembly or
+ * direct CPU support.
+ *
+ */
+#if __SIZEOF_INT128__
+ namespace pcg_extras {
+ typedef __uint128_t pcg128_t;
+ }
+ #define PCG_128BIT_CONSTANT(high,low) \
+ ((pcg128_t(high) << 64) + low)
+#else
+ #include "pcg_uint128.hpp"
+ namespace pcg_extras {
+ typedef pcg_extras::uint_x4<uint32_t,uint64_t> pcg128_t;
+ }
+ #define PCG_128BIT_CONSTANT(high,low) \
+ pcg128_t(high,low)
+ #define PCG_EMULATED_128BIT_MATH 1
+#endif
+
+
+namespace pcg_extras {
+
+/*
+ * We often need to represent a "number of bits". When used normally, these
+ * numbers are never greater than 128, so an unsigned char is plenty.
+ * If you're using a nonstandard generator of a larger size, you can set
+ * PCG_BITCOUNT_T to have it define it as a larger size. (Some compilers
+ * might produce faster code if you set it to an unsigned int.)
+ */
+
+#ifndef PCG_BITCOUNT_T
+ typedef uint8_t bitcount_t;
+#else
+ typedef PCG_BITCOUNT_T bitcount_t;
+#endif
+
+/*
+ * C++ requires us to be able to serialize RNG state by printing or reading
+ * it from a stream. Because we use 128-bit ints, we also need to be able
+ * ot print them, so here is code to do so.
+ *
+ * This code provides enough functionality to print 128-bit ints in decimal
+ * and zero-padded in hex. It's not a full-featured implementation.
+ */
+
+template <typename CharT, typename Traits>
+std::basic_ostream<CharT,Traits>&
+operator<<(std::basic_ostream<CharT,Traits>& out, pcg128_t value)
+{
+ auto desired_base = out.flags() & out.basefield;
+ bool want_hex = desired_base == out.hex;
+
+ if (want_hex) {
+ uint64_t highpart = uint64_t(value >> 64);
+ uint64_t lowpart = uint64_t(value);
+ auto desired_width = out.width();
+ if (desired_width > 16) {
+ out.width(desired_width - 16);
+ }
+ if (highpart != 0 || desired_width > 16)
+ out << highpart;
+ CharT oldfill;
+ if (highpart != 0) {
+ out.width(16);
+ oldfill = out.fill('0');
+ }
+ auto oldflags = out.setf(decltype(desired_base){}, out.showbase);
+ out << lowpart;
+ out.setf(oldflags);
+ if (highpart != 0) {
+ out.fill(oldfill);
+ }
+ return out;
+ }
+ constexpr size_t MAX_CHARS_128BIT = 40;
+
+ char buffer[MAX_CHARS_128BIT];
+ char* pos = buffer+sizeof(buffer);
+ *(--pos) = '\0';
+ constexpr auto BASE = pcg128_t(10ULL);
+ do {
+ auto div = value / BASE;
+ auto mod = uint32_t(value - (div * BASE));
+ *(--pos) = '0' + mod;
+ value = div;
+ } while(value != pcg128_t(0ULL));
+ return out << pos;
+}
+
+template <typename CharT, typename Traits>
+std::basic_istream<CharT,Traits>&
+operator>>(std::basic_istream<CharT,Traits>& in, pcg128_t& value)
+{
+ typename std::basic_istream<CharT,Traits>::sentry s(in);
+
+ if (!s)
+ return in;
+
+ constexpr auto BASE = pcg128_t(10ULL);
+ pcg128_t current(0ULL);
+ bool did_nothing = true;
+ bool overflow = false;
+ for(;;) {
+ CharT wide_ch = in.get();
+ if (!in.good())
+ break;
+ auto ch = in.narrow(wide_ch, '\0');
+ if (ch < '0' || ch > '9') {
+ in.unget();
+ break;
+ }
+ did_nothing = false;
+ pcg128_t digit(uint32_t(ch - '0'));
+ pcg128_t timesbase = current*BASE;
+ overflow = overflow || timesbase < current;
+ current = timesbase + digit;
+ overflow = overflow || current < digit;
+ }
+
+ if (did_nothing || overflow) {
+ in.setstate(std::ios::failbit);
+ if (overflow)
+ current = ~pcg128_t(0ULL);
+ }
+
+ value = current;
+
+ return in;
+}
+
+/*
+ * Likewise, if people use tiny rngs, we'll be serializing uint8_t.
+ * If we just used the provided IO operators, they'd read/write chars,
+ * not ints, so we need to define our own. We *can* redefine this operator
+ * here because we're in our own namespace.
+ */
+
+template <typename CharT, typename Traits>
+std::basic_ostream<CharT,Traits>&
+operator<<(std::basic_ostream<CharT,Traits>&out, uint8_t value)
+{
+ return out << uint32_t(value);
+}
+
+template <typename CharT, typename Traits>
+std::basic_istream<CharT,Traits>&
+operator>>(std::basic_istream<CharT,Traits>& in, uint8_t target)
+{
+ uint32_t value = 0xdecea5edU;
+ in >> value;
+ if (!in && value == 0xdecea5edU)
+ return in;
+ if (value > uint8_t(~0)) {
+ in.setstate(std::ios::failbit);
+ value = ~0U;
+ }
+ target = uint8_t(value);
+ return in;
+}
+
+/* Unfortunately, the above functions don't get found in preference to the
+ * built in ones, so we create some more specific overloads that will.
+ * Ugh.
+ */
+
+inline std::ostream& operator<<(std::ostream& out, uint8_t value)
+{
+ return pcg_extras::operator<< <char>(out, value);
+}
+
+inline std::istream& operator>>(std::istream& in, uint8_t& value)
+{
+ return pcg_extras::operator>> <char>(in, value);
+}
+
+
+
+/*
+ * Useful bitwise operations.
+ */
+
+/*
+ * XorShifts are invertable, but they are someting of a pain to invert.
+ * This function backs them out. It's used by the whacky "inside out"
+ * generator defined later.
+ */
+
+template <typename itype>
+inline itype unxorshift(itype x, bitcount_t bits, bitcount_t shift)
+{
+ if (2*shift >= bits) {
+ return x ^ (x >> shift);
+ }
+ itype lowmask1 = (itype(1U) << (bits - shift*2)) - 1;
+ itype highmask1 = ~lowmask1;
+ itype top1 = x;
+ itype bottom1 = x & lowmask1;
+ top1 ^= top1 >> shift;
+ top1 &= highmask1;
+ x = top1 | bottom1;
+ itype lowmask2 = (itype(1U) << (bits - shift)) - 1;
+ itype bottom2 = x & lowmask2;
+ bottom2 = unxorshift(bottom2, bits - shift, shift);
+ bottom2 &= lowmask1;
+ return top1 | bottom2;
+}
+
+/*
+ * Rotate left and right.
+ *
+ * In ideal world, compilers would spot idiomatic rotate code and convert it
+ * to a rotate instruction. Of course, opinions vary on what the correct
+ * idiom is and how to spot it. For clang, sometimes it generates better
+ * (but still crappy) code if you define PCG_USE_ZEROCHECK_ROTATE_IDIOM.
+ */
+
+template <typename itype>
+inline itype rotl(itype value, bitcount_t rot)
+{
+ constexpr bitcount_t bits = sizeof(itype) * 8;
+ constexpr bitcount_t mask = bits - 1;
+#if PCG_USE_ZEROCHECK_ROTATE_IDIOM
+ return rot ? (value << rot) | (value >> (bits - rot)) : value;
+#else
+ return (value << rot) | (value >> ((- rot) & mask));
+#endif
+}
+
+template <typename itype>
+inline itype rotr(itype value, bitcount_t rot)
+{
+ constexpr bitcount_t bits = sizeof(itype) * 8;
+ constexpr bitcount_t mask = bits - 1;
+#if PCG_USE_ZEROCHECK_ROTATE_IDIOM
+ return rot ? (value >> rot) | (value << (bits - rot)) : value;
+#else
+ return (value >> rot) | (value << ((- rot) & mask));
+#endif
+}
+
+/* Unfortunately, both Clang and GCC sometimes perform poorly when it comes
+ * to properly recognizing idiomatic rotate code, so for we also provide
+ * assembler directives (enabled with PCG_USE_INLINE_ASM). Boo, hiss.
+ * (I hope that these compilers get better so that this code can die.)
+ *
+ * These overloads will be preferred over the general template code above.
+ */
+#if PCG_USE_INLINE_ASM && __GNUC__ && (__x86_64__ || __i386__)
+
+inline uint8_t rotr(uint8_t value, bitcount_t rot)
+{
+ asm ("rorb %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
+ return value;
+}
+
+inline uint16_t rotr(uint16_t value, bitcount_t rot)
+{
+ asm ("rorw %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
+ return value;
+}
+
+inline uint32_t rotr(uint32_t value, bitcount_t rot)
+{
+ asm ("rorl %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
+ return value;
+}
+
+#if __x86_64__
+inline uint64_t rotr(uint64_t value, bitcount_t rot)
+{
+ asm ("rorq %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
+ return value;
+}
+#endif // __x86_64__
+
+#endif // PCG_USE_INLINE_ASM
+
+
+/*
+ * The C++ SeedSeq concept (modelled by seed_seq) can fill an array of
+ * 32-bit integers with seed data, but sometimes we want to produce
+ * larger or smaller integers.
+ *
+ * The following code handles this annoyance.
+ *
+ * uneven_copy will copy an array of 32-bit ints to an array of larger or
+ * smaller ints (actually, the code is general it only needing forward
+ * iterators). The copy is identical to the one that would be performed if
+ * we just did memcpy on a standard little-endian machine, but works
+ * regardless of the endian of the machine (or the weirdness of the ints
+ * involved).
+ *
+ * generate_to initializes an array of integers using a SeedSeq
+ * object. It is given the size as a static constant at compile time and
+ * tries to avoid memory allocation. If we're filling in 32-bit constants
+ * we just do it directly. If we need a separate buffer and it's small,
+ * we allocate it on the stack. Otherwise, we fall back to heap allocation.
+ * Ugh.
+ *
+ * generate_one produces a single value of some integral type using a
+ * SeedSeq object.
+ */
+
+ /* uneven_copy helper, case where destination ints are less than 32 bit. */
+
+template<class SrcIter, class DestIter>
+SrcIter uneven_copy_impl(
+ SrcIter src_first, DestIter dest_first, DestIter dest_last,
+ std::true_type)
+{
+ typedef typename std::iterator_traits<SrcIter>::value_type src_t;
+ typedef typename std::iterator_traits<DestIter>::value_type dest_t;
+
+ constexpr bitcount_t SRC_SIZE = sizeof(src_t);
+ constexpr bitcount_t DEST_SIZE = sizeof(dest_t);
+ constexpr bitcount_t DEST_BITS = DEST_SIZE * 8;
+ constexpr bitcount_t SCALE = SRC_SIZE / DEST_SIZE;
+
+ size_t count = 0;
+ src_t value;
+
+ while (dest_first != dest_last) {
+ if ((count++ % SCALE) == 0)
+ value = *src_first++; // Get more bits
+ else
+ value >>= DEST_BITS; // Move down bits
+
+ *dest_first++ = dest_t(value); // Truncates, ignores high bits.
+ }
+ return src_first;
+}
+
+ /* uneven_copy helper, case where destination ints are more than 32 bit. */
+
+template<class SrcIter, class DestIter>
+SrcIter uneven_copy_impl(
+ SrcIter src_first, DestIter dest_first, DestIter dest_last,
+ std::false_type)
+{
+ typedef typename std::iterator_traits<SrcIter>::value_type src_t;
+ typedef typename std::iterator_traits<DestIter>::value_type dest_t;
+
+ constexpr auto SRC_SIZE = sizeof(src_t);
+ constexpr auto SRC_BITS = SRC_SIZE * 8;
+ constexpr auto DEST_SIZE = sizeof(dest_t);
+ constexpr auto SCALE = (DEST_SIZE+SRC_SIZE-1) / SRC_SIZE;
+
+ while (dest_first != dest_last) {
+ dest_t value(0UL);
+ unsigned int shift = 0;
+
+ for (size_t i = 0; i < SCALE; ++i) {
+ value |= dest_t(*src_first++) << shift;
+ shift += SRC_BITS;
+ }
+
+ *dest_first++ = value;
+ }
+ return src_first;
+}
+
+/* uneven_copy, call the right code for larger vs. smaller */
+
+template<class SrcIter, class DestIter>
+inline SrcIter uneven_copy(SrcIter src_first,
+ DestIter dest_first, DestIter dest_last)
+{
+ typedef typename std::iterator_traits<SrcIter>::value_type src_t;
+ typedef typename std::iterator_traits<DestIter>::value_type dest_t;
+
+ constexpr bool DEST_IS_SMALLER = sizeof(dest_t) < sizeof(src_t);
+
+ return uneven_copy_impl(src_first, dest_first, dest_last,
+ std::integral_constant<bool, DEST_IS_SMALLER>{});
+}
+
+/* generate_to, fill in a fixed-size array of integral type using a SeedSeq
+ * (actually works for any random-access iterator)
+ */
+
+template <size_t size, typename SeedSeq, typename DestIter>
+inline void generate_to_impl(SeedSeq&& generator, DestIter dest,
+ std::true_type)
+{
+ generator.generate(dest, dest+size);
+}
+
+template <size_t size, typename SeedSeq, typename DestIter>
+void generate_to_impl(SeedSeq&& generator, DestIter dest,
+ std::false_type)
+{
+ typedef typename std::iterator_traits<DestIter>::value_type dest_t;
+ constexpr auto DEST_SIZE = sizeof(dest_t);
+ constexpr auto GEN_SIZE = sizeof(uint32_t);
+
+ constexpr bool GEN_IS_SMALLER = GEN_SIZE < DEST_SIZE;
+ constexpr size_t FROM_ELEMS =
+ GEN_IS_SMALLER
+ ? size * ((DEST_SIZE+GEN_SIZE-1) / GEN_SIZE)
+ : (size + (GEN_SIZE / DEST_SIZE) - 1)
+ / ((GEN_SIZE / DEST_SIZE) + GEN_IS_SMALLER);
+ // this odd code ^^^^^^^^^^^^^^^^^ is work-around for
+ // a bug: http://llvm.org/bugs/show_bug.cgi?id=21287
+
+ if (FROM_ELEMS <= 1024) {
+ uint32_t buffer[FROM_ELEMS];
+ generator.generate(buffer, buffer+FROM_ELEMS);
+ uneven_copy(buffer, dest, dest+size);
+ } else {
+ uint32_t* buffer = (uint32_t*) malloc(GEN_SIZE * FROM_ELEMS);
+ generator.generate(buffer, buffer+FROM_ELEMS);
+ uneven_copy(buffer, dest, dest+size);
+ free(buffer);
+ }
+}
+
+template <size_t size, typename SeedSeq, typename DestIter>
+inline void generate_to(SeedSeq&& generator, DestIter dest)
+{
+ typedef typename std::iterator_traits<DestIter>::value_type dest_t;
+ constexpr bool IS_32BIT = sizeof(dest_t) == sizeof(uint32_t);
+
+ generate_to_impl<size>(std::forward<SeedSeq>(generator), dest,
+ std::integral_constant<bool, IS_32BIT>{});
+}
+
+/* generate_one, produce a value of integral type using a SeedSeq
+ * (optionally, we can have it produce more than one and pick which one
+ * we want)
+ */
+
+template <typename UInt, size_t i = 0UL, size_t N = i+1UL, typename SeedSeq>
+inline UInt generate_one(SeedSeq&& generator)
+{
+ UInt result[N];
+ generate_to<N>(std::forward<SeedSeq>(generator), result);
+ return result[i];
+}
+
+template <typename RngType>
+auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
+ -> typename RngType::result_type
+{
+ typedef typename RngType::result_type rtype;
+ rtype threshold = (RngType::max() - RngType::min() + rtype(1) - upper_bound)
+ % upper_bound;
+ for (;;) {
+ rtype r = rng() - RngType::min();
+ if (r >= threshold)
+ return r % upper_bound;
+ }
+}
+
+template <typename Iter, typename RandType>
+void shuffle(Iter from, Iter to, RandType&& rng)
+{
+ typedef typename std::iterator_traits<Iter>::difference_type delta_t;
+ auto count = to - from;
+ while (count > 1) {
+ delta_t chosen(bounded_rand(rng, count));
+ --count;
+ --to;
+ using std::swap;
+ swap(*(from+chosen), *to);
+ }
+}
+
+/*
+ * Although std::seed_seq is useful, it isn't everything. Often we want to
+ * initialize a random-number generator some other way, such as from a random
+ * device.
+ *
+ * Technically, it does not meet the requirements of a SeedSequence because
+ * it lacks some of the rarely-used member functions (some of which would
+ * be impossible to provide). However the C++ standard is quite specific
+ * that actual engines only called the generate method, so it ought not to be
+ * a problem in practice.
+ */
+
+template <typename RngType>
+class seed_seq_from {
+private:
+ RngType rng_;
+
+ typedef uint_least32_t result_type;
+
+public:
+ template<typename... Args>
+ seed_seq_from(Args&&... args) :
+ rng_(std::forward<Args>(args)...)
+ {
+ // Nothing (else) to do...
+ }
+
+ template<typename Iter>
+ void generate(Iter start, Iter finish)
+ {
+ for (auto i = start; i != finish; ++i)
+ *i = result_type(rng_());
+ }
+
+ constexpr size_t size() const
+ {
+ return (sizeof(typename RngType::result_type) > sizeof(result_type)
+ && RngType::max() > ~size_t(0UL))
+ ? ~size_t(0UL)
+ : size_t(RngType::max());
+ }
+};
+
+/*
+ * Sometimes you might want a distinct seed based on when the program
+ * was compiled. That way, a particular instance of the program will
+ * behave the same way, but when recompiled it'll produce a different
+ * value.
+ */
+
+template <typename IntType>
+struct static_arbitrary_seed {
+private:
+ static constexpr IntType fnv(IntType hash, const char* pos) {
+ return *pos == '\0'
+ ? hash
+ : fnv((hash * IntType(16777619U)) ^ *pos, (pos+1));
+ }
+
+public:
+ static constexpr IntType value = fnv(IntType(2166136261U ^ sizeof(IntType)),
+ __DATE__ __TIME__ __FILE__);
+};
+
+// Sometimes, when debugging or testing, it's handy to be able print the name
+// of a (in human-readable form). This code allows the idiom:
+//
+// cout << printable_typename<my_foo_type_t>()
+//
+// to print out my_foo_type_t (or its concrete type if it is a synonym)
+
+template <typename T>
+struct printable_typename {};
+
+template <typename T>
+std::ostream& operator<<(std::ostream& out, printable_typename<T>) {
+ const char *implementation_typename = typeid(T).name();
+#ifdef __GNUC__
+ int status;
+ const char* pretty_name =
+ abi::__cxa_demangle(implementation_typename, NULL, NULL, &status);
+ if (status == 0)
+ out << pretty_name;
+ free((void*) pretty_name);
+ if (status == 0)
+ return out;
+#endif
+ out << implementation_typename;
+ return out;
+}
+
+} // namespace pcg_extras
+
+#endif // PCG_EXTRAS_HPP_INCLUDED