You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2016/01/06 18:03:27 UTC
[30/50] [abbrv] cassandra git commit: Match cassandra-loader options
in COPY FROM (3.0 version)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2883879/pylib/cqlshlib/formatting.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py
index 62ecd10..2e219c8 100644
--- a/pylib/cqlshlib/formatting.py
+++ b/pylib/cqlshlib/formatting.py
@@ -60,7 +60,8 @@ empty_colormap = defaultdict(lambda: '')
def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False,
- nullval=None, date_time_format=None, float_precision=None):
+ nullval=None, date_time_format=None, float_precision=None,
+ decimal_sep=None, thousands_sep=None, boolean_styles=None):
if nullval is None:
nullval = default_null_placeholder
if val is None:
@@ -75,7 +76,8 @@ def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False,
float_precision = default_float_precision
return format_value(cqltype, val, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
- nullval=nullval)
+ nullval=nullval, decimal_sep=decimal_sep, thousands_sep=thousands_sep,
+ boolean_styles=boolean_styles)
def color_text(bval, colormap, displaywidth=None):
@@ -155,7 +157,9 @@ def format_python_formatted_type(val, colormap, color, quote=False):
@formatter_for('Decimal')
-def format_value_decimal(val, colormap, **_):
+def format_value_decimal(val, float_precision, colormap, decimal_sep=None, thousands_sep=None, **_):
+ if (decimal_sep and decimal_sep != '.') or thousands_sep:
+ return format_floating_point_type(val, colormap, float_precision, decimal_sep, thousands_sep)
return format_python_formatted_type(val, colormap, 'decimal')
@@ -170,34 +174,60 @@ def formatter_value_inet(val, colormap, quote=False, **_):
@formatter_for('bool')
-def format_value_boolean(val, colormap, **_):
+def format_value_boolean(val, colormap, boolean_styles=None, **_):
+ if boolean_styles:
+ val = boolean_styles[0] if val else boolean_styles[1]
return format_python_formatted_type(val, colormap, 'boolean')
-def format_floating_point_type(val, colormap, float_precision, **_):
+def format_floating_point_type(val, colormap, float_precision, decimal_sep=None, thousands_sep=None, **_):
if math.isnan(val):
bval = 'NaN'
elif math.isinf(val):
bval = 'Infinity' if val > 0 else '-Infinity'
else:
- exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1
- if -4 <= exponent < float_precision:
- # when this is true %g will not use scientific notation,
- # increasing precision should not change this decision
- # so we increase the precision to take into account the
- # digits to the left of the decimal point
- float_precision = float_precision + exponent + 1
- bval = '%.*g' % (float_precision, val)
+ if thousands_sep:
+ dpart, ipart = math.modf(val)
+ bval = format_integer_with_thousands_sep(ipart, thousands_sep)
+ dpart_str = ('%.*f' % (float_precision, math.fabs(dpart)))[2:].rstrip('0')
+ if dpart_str:
+ bval += '%s%s' % ('.' if not decimal_sep else decimal_sep, dpart_str)
+ else:
+ exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1
+ if -4 <= exponent < float_precision:
+ # when this is true %g will not use scientific notation,
+ # increasing precision should not change this decision
+ # so we increase the precision to take into account the
+ # digits to the left of the decimal point
+ float_precision = float_precision + exponent + 1
+ bval = '%.*g' % (float_precision, val)
+ if decimal_sep:
+ bval = bval.replace('.', decimal_sep)
+
return colorme(bval, colormap, 'float')
formatter_for('float')(format_floating_point_type)
-def format_integer_type(val, colormap, **_):
+def format_integer_type(val, colormap, thousands_sep=None, **_):
# base-10 only for now; support others?
- bval = str(val)
+ bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val)
return colorme(bval, colormap, 'int')
+# We can get rid of this in cassandra-2.2
+if sys.version_info >= (2, 7):
+ def format_integer_with_thousands_sep(val, thousands_sep=','):
+ return "{:,.0f}".format(val).replace(',', thousands_sep)
+else:
+ def format_integer_with_thousands_sep(val, thousands_sep=','):
+ if val < 0:
+ return '-' + format_integer_with_thousands_sep(-val, thousands_sep)
+ result = ''
+ while val >= 1000:
+ val, r = divmod(val, 1000)
+ result = "%s%03d%s" % (thousands_sep, r, result)
+ return "%d%s" % (val, result)
+
formatter_for('long')(format_integer_type)
formatter_for('int')(format_integer_type)
@@ -242,10 +272,12 @@ formatter_for('unicode')(format_value_text)
def format_simple_collection(val, lbracket, rbracket, encoding,
- colormap, date_time_format, float_precision, nullval):
+ colormap, date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles):
subs = [format_value(type(sval), sval, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
- nullval=nullval, quote=True)
+ nullval=nullval, quote=True, decimal_sep=decimal_sep,
+ thousands_sep=thousands_sep, boolean_styles=boolean_styles)
for sval in val]
bval = lbracket + ', '.join(get_str(sval) for sval in subs) + rbracket
if colormap is NO_COLOR_MAP:
@@ -259,32 +291,40 @@ def format_simple_collection(val, lbracket, rbracket, encoding,
@formatter_for('list')
-def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
+def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles, **_):
return format_simple_collection(val, '[', ']', encoding, colormap,
- date_time_format, float_precision, nullval)
+ date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles)
@formatter_for('tuple')
-def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
+def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles, **_):
return format_simple_collection(val, '(', ')', encoding, colormap,
- date_time_format, float_precision, nullval)
+ date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles)
@formatter_for('set')
-def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
+def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles, **_):
return format_simple_collection(sorted(val), '{', '}', encoding, colormap,
- date_time_format, float_precision, nullval)
+ date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles)
formatter_for('frozenset')(format_value_set)
formatter_for('sortedset')(format_value_set)
formatter_for('SortedSet')(format_value_set)
@formatter_for('dict')
-def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
+def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles, **_):
def subformat(v):
return format_value(type(v), v, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
- nullval=nullval, quote=True)
+ nullval=nullval, quote=True, decimal_sep=decimal_sep,
+ thousands_sep=thousands_sep, boolean_styles=boolean_styles)
subs = [(subformat(k), subformat(v)) for (k, v) in sorted(val.items())]
bval = '{' + ', '.join(get_str(k) + ': ' + get_str(v) for (k, v) in subs) + '}'
@@ -303,13 +343,15 @@ formatter_for('OrderedMap')(format_value_map)
formatter_for('OrderedMapSerializedKey')(format_value_map)
-def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
+def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval,
+ decimal_sep, thousands_sep, boolean_styles, **_):
def format_field_value(v):
if v is None:
return colorme(nullval, colormap, 'error')
return format_value(type(v), v, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
- nullval=nullval, quote=True)
+ nullval=nullval, quote=True, decimal_sep=decimal_sep,
+ thousands_sep=thousands_sep, boolean_styles=boolean_styles)
def format_field_name(name):
return format_value_text(name, encoding=encoding, colormap=colormap, quote=False)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2883879/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
index dc7ef73..3979597 100644
--- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
@@ -34,6 +34,8 @@ import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.*;
import org.apache.cassandra.tracing.Tracing;
@@ -48,7 +50,7 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse
*/
public class BatchStatement implements CQLStatement
{
- public static enum Type
+ public enum Type
{
LOGGED, UNLOGGED, COUNTER
}
@@ -297,12 +299,22 @@ public class BatchStatement implements CQLStatement
Set<DecoratedKey> keySet = new HashSet<>();
Set<String> tableNames = new HashSet<>();
+ Map<String, Collection<Range<Token>>> localTokensByKs = new HashMap<>();
+ boolean localPartitionsOnly = true;
for (PartitionUpdate update : updates)
{
keySet.add(update.partitionKey());
tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName));
+
+ if (localPartitionsOnly)
+ localPartitionsOnly &= isPartitionLocal(localTokensByKs, update);
}
+ // CASSANDRA-9303: If we only have local mutations we do not warn
+ if (localPartitionsOnly)
+ return;
+
+
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, UNLOGGED_BATCH_WARNING,
keySet.size(), keySet.size() == 1 ? "" : "s",
tableNames.size() == 1 ? "" : "s", tableNames);
@@ -313,6 +325,18 @@ public class BatchStatement implements CQLStatement
}
}
+ private boolean isPartitionLocal(Map<String, Collection<Range<Token>>> localTokensByKs, PartitionUpdate update)
+ {
+ Collection<Range<Token>> localRanges = localTokensByKs.get(update.metadata().ksName);
+ if (localRanges == null)
+ {
+ localRanges = StorageService.instance.getLocalRanges(update.metadata().ksName);
+ localTokensByKs.put(update.metadata().ksName, localRanges);
+ }
+
+ return Range.isInRanges(update.partitionKey().getToken(), localRanges);
+ }
+
public ResultMessage execute(QueryState queryState, QueryOptions options) throws RequestExecutionException, RequestValidationException
{
return execute(queryState, BatchQueryOptions.withoutPerStatementVariables(options));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2883879/test/unit/org/apache/cassandra/service/ClientWarningsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java
index 5cdeb78..c3f2629 100644
--- a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java
+++ b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java
@@ -31,6 +31,7 @@ import org.apache.cassandra.transport.SimpleClient;
import org.apache.cassandra.transport.messages.QueryMessage;
import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
public class ClientWarningsTest extends CQLTester
{
@@ -52,12 +53,11 @@ public class ClientWarningsTest extends CQLTester
QueryMessage query = new QueryMessage(createBatchStatement2(1), QueryOptions.DEFAULT);
Message.Response resp = client.execute(query);
- assertEquals(1, resp.getWarnings().size());
+ assertNull(resp.getWarnings());
query = new QueryMessage(createBatchStatement2(DatabaseDescriptor.getBatchSizeWarnThreshold()), QueryOptions.DEFAULT);
resp = client.execute(query);
- assertEquals(2, resp.getWarnings().size());
-
+ assertEquals(1, resp.getWarnings().size());
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2883879/tools/bin/cassandra-stress.bat
----------------------------------------------------------------------
diff --git a/tools/bin/cassandra-stress.bat b/tools/bin/cassandra-stress.bat
index f1bbcc9..5c2ecfa 100644
--- a/tools/bin/cassandra-stress.bat
+++ b/tools/bin/cassandra-stress.bat
@@ -19,4 +19,4 @@ if "%OS%" == "Windows_NT" setlocal
pushd "%~dp0"
call cassandra.in.bat
if NOT DEFINED STRESS_HOME set STRESS_HOME=%CD%\..
-"%JAVA_HOME%\bin\java" -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %*
+"%JAVA_HOME%\bin\java" %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %*