You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sn...@apache.org on 2015/05/14 10:41:57 UTC
[2/6] cassandra git commit: token-generator - generated tokens too
long
token-generator - generated tokens too long
Patch by Robert Stupp; Reviewed by Stefania for CASSANDRA-9300
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1611ef3d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1611ef3d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1611ef3d
Branch: refs/heads/cassandra-2.1
Commit: 1611ef3d1995337998b9d7514c7ea5c35fa59895
Parents: 418deaf
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu May 14 10:35:35 2015 +0200
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu May 14 10:35:35 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 3 +++
tools/bin/token-generator | 53 +++++++++++++++++++++++++++-----------
tools/bin/token-generator.bat | 34 ++++++++++++++++++++++++
3 files changed, 75 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d7d01cf..cee28bc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,6 @@
+2.0.16:
+ * token-generator - generated tokens too long (CASSANDRA-9300)
+
2.0.15:
* Fix counting of tombstones for TombstoneOverwhelmingException (CASSANDRA-9299)
* Fix ReconnectableSnitch reconnecting to peers during upgrade (CASSANDRA-6702)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/tools/bin/token-generator
----------------------------------------------------------------------
diff --git a/tools/bin/token-generator b/tools/bin/token-generator
index c958529..b70866d 100755
--- a/tools/bin/token-generator
+++ b/tools/bin/token-generator
@@ -30,24 +30,38 @@ from tempfile import NamedTemporaryFile
description = '''Given a list of numbers indicating the number of nodes
in each separate datacenter, outputs a recommended list of tokens to use
-with RandomPartitioner: one for each node in each datacenter.
+with Murmur3Partitioner (by default) or with RandomPartitioner.
+The list contains one token for each node in each datacenter.
'''
usage = "%prog <nodes_in_dc1> [<nodes_in_dc2> [...]]"
parser = optparse.OptionParser(description=description, usage=usage)
+
+def part_murmur3(option, opt, value, parser):
+ parser.values.ringoffset=-(1<<63)
+ parser.values.ringrange=(1<<64)
+ return
+def part_random(option, opt, value, parser):
+ parser.values.ringoffset=0
+ parser.values.ringrange=(1<<127)
+ return
+parser.add_option('--murmur3', action='callback', callback=part_murmur3,
+ help='Generate tokens for Murmur3Partitioner (default).')
+parser.add_option('--random', action='callback', callback=part_random,
+ help='Generate tokens for RandomPartitioner.')
+parser.add_option('--ringoffset', type='int',
+ help=optparse.SUPPRESS_HELP)
parser.add_option('--ringrange', type='int',
- help='Specify a numeric maximum token value for your ring, '
- 'different from the default value of 2^127.')
+ help=optparse.SUPPRESS_HELP)
+
parser.add_option('--graph', action='store_true',
help='Show a rendering of the generated tokens as line '
'segments in a circle, colored according to datacenter')
parser.add_option('-n', '--nts', action='store_const', dest='strat', const='nts',
- help='Optimize multi-cluster distribution for '
- 'NetworkTopologyStrategy [default]')
+ help=optparse.SUPPRESS_HELP)
parser.add_option('-o', '--onts', action='store_const', dest='strat', const='onts',
- help='Optimize multi-cluster distribution for '
- 'OldNetworkTopologyStrategy')
+ help=optparse.SUPPRESS_HELP)
parser.add_option('--test', action='store_true',
help='Run in test mode, outputting an HTML file to display '
@@ -58,8 +72,11 @@ parser.add_option('--browser-wait-time', type='float', help=optparse.SUPPRESS_HE
parser.add_option('--test-colors', help=optparse.SUPPRESS_HELP)
parser.add_option('--test-graphsize', type='int', help=optparse.SUPPRESS_HELP)
+
parser.set_defaults(
- ringrange=(1<<127),
+ # default is Murmur3
+ ringoffset=-(1<<63),
+ ringrange=(1<<64),
# whether to create (and try to display) graph output
graph=False,
@@ -94,7 +111,8 @@ class Ring:
MIN_DC_OFFSET_DIVIDER = 235
offset_spacer = 2
- def __init__(self, dc_counts, ringrange, strategy='nts'):
+ def __init__(self, dc_counts, ringoffset, ringrange, strategy='nts'):
+ self.ringoffset = ringoffset
self.ringrange = ringrange
self.dc_counts = dc_counts
self.calculate_offset_tokens = getattr(self, 'calc_offset_tokens_' + strategy)
@@ -110,13 +128,18 @@ class Ring:
division = max(lowest_division, self.MIN_DC_OFFSET_DIVIDER)
return -self.ringrange // division
+ def bound_token(self, tok):
+ if tok < self.ringoffset:
+ tok += self.ringrange
+ return tok
+
def calc_offset_tokens_nts(self):
dc_offset = self.best_per_dc_offset()
dcs = []
for (dcnum, dccount) in enumerate(self.dc_counts):
offset = dcnum * dc_offset
arcsize = self.ringrange // (dccount or 1)
- dcs.append([(n * arcsize + offset) % self.ringrange for n in xrange(dccount)])
+ dcs.append(sorted([self.bound_token((n * arcsize + offset) - self.ringoffset % self.ringrange) for n in xrange(dccount)]))
return dcs
def calc_offset_tokens_onts(self):
@@ -127,7 +150,7 @@ class Ring:
final = [[] for x in dcs_by_count]
for pos, dc in enumerate(layout):
- final[dc].append(pos * self.ringrange // len(layout))
+ final[dc].append(self.ringoffset + pos * self.ringrange // len(layout))
return final
@@ -139,8 +162,8 @@ def print_tokens(tokens, tokenwidth, indent=0):
for tnum, tok in enumerate(toklist):
print "%s Node #%0*d: % *d" % (indentstr, nwidth, tnum + 1, tokenwidth, tok)
-def calculate_ideal_tokens(datacenters, ringrange, strategy):
- return Ring(datacenters, ringrange, strategy).calculate_offset_tokens()
+def calculate_ideal_tokens(datacenters, ringoffset, ringrange, strategy):
+ return Ring(datacenters, ringoffset, ringrange, strategy).calculate_offset_tokens()
def file_to_url(path):
path = os.path.abspath(path)
@@ -248,7 +271,7 @@ def run_tests(opts):
tokensets = []
for test in tests:
print "Test %r" % (test,)
- tokens = calculate_ideal_tokens(test, opts.ringrange, opts.strat)
+ tokens = calculate_ideal_tokens(test, opts.ringoffset, opts.ringrange, opts.strat)
print_tokens(tokens, len(str(opts.ringrange)) + 1, indent=2)
tokensets.append(tokens)
return tokensets
@@ -315,7 +338,7 @@ def main(opts, args):
parser.error('Arguments should be integers.')
renderer = RingRenderer(ringrange=opts.ringrange, graphsize=opts.graphsize,
colors=opts.colorlist)
- tokens = calculate_ideal_tokens(datacenters, opts.ringrange, opts.strat)
+ tokens = calculate_ideal_tokens(datacenters, opts.ringoffset, opts.ringrange, opts.strat)
print_tokens(tokens, len(str(opts.ringrange)) + 1)
tokensets = [tokens]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/tools/bin/token-generator.bat
----------------------------------------------------------------------
diff --git a/tools/bin/token-generator.bat b/tools/bin/token-generator.bat
new file mode 100644
index 0000000..a7188db
--- /dev/null
+++ b/tools/bin/token-generator.bat
@@ -0,0 +1,34 @@
+@ECHO OFF
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+
+@echo off
+
+if "%OS%" == "Windows_NT" setlocal
+
+python -V >nul 2>&1
+if ERRORLEVEL 1 goto err
+
+python "%~dp0\token-generator" %*
+goto finally
+
+:err
+echo Can't detect Python version!
+
+:finally
+
+ENDLOCAL
+