You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2021/06/25 16:19:26 UTC
[ignite-3] branch ignite-14743-row-formats updated (9ce3e73 ->
1181d7c)
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a change to branch ignite-14743-row-formats
in repository https://gitbox.apache.org/repos/asf/ignite-3.git.
from 9ce3e73 Minor.
new 1316302 Fix benchmarks
new 1181d7c Minor optimizations.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../main/java/org/apache/ignite/table/Tuple.java | 5 +-
.../ignite/internal/table/RowChunkAdapter.java | 11 +++
.../org/apache/ignite/internal/table/TableRow.java | 23 +-----
.../ignite/internal/table/TupleBuilderImpl.java | 6 +-
.../ignite/internal/table/TupleMarshallerImpl.java | 91 +++++++++++-----------
.../TupleMarshallerVarlenOnlyBenchmark.java | 30 +++++--
.../internal/table/impl/TestTupleBuilder.java | 20 ++---
7 files changed, 99 insertions(+), 87 deletions(-)
[ignite-3] 01/02: Fix benchmarks
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch ignite-14743-row-formats
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 1316302314a70932734ec8d4fcda2a9a81117a89
Author: Andrew Mashenkov <an...@gmail.com>
AuthorDate: Fri Jun 25 18:14:59 2021 +0300
Fix benchmarks
---
.../TupleMarshallerVarlenOnlyBenchmark.java | 32 ++++++++++++++++------
1 file changed, 24 insertions(+), 8 deletions(-)
diff --git a/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java b/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java
index 80f33ac..2d25935 100644
--- a/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java
+++ b/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java
@@ -48,6 +48,7 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
import static org.apache.ignite.internal.schema.NativeTypes.BYTES;
import static org.apache.ignite.internal.schema.NativeTypes.LONG;
+import static org.apache.ignite.internal.schema.NativeTypes.STRING;
/**
* Serializer benchmark.
@@ -71,18 +72,22 @@ public class TupleMarshallerVarlenOnlyBenchmark {
public int fieldsCount;
/** Payload size. */
- @Param({"250", "64000", "65000"})
+ @Param({"250", "64000", "66000"})
public int dataSize;
/** Nullable cols. */
@Param({"true", "false"})
- public boolean nullable;
+ public boolean nullable ;
+
+ /** Column types. */
+ @Param({"string", "bytes"})
+ public String type;
/** Schema descriptor. */
private SchemaDescriptor schema;
- /** Values. */
- private byte[] val;
+ /** Value. */
+ private Object val;
/**
* Runner.
@@ -100,7 +105,8 @@ public class TupleMarshallerVarlenOnlyBenchmark {
*/
@Setup
public void init() {
- long seed = System.currentTimeMillis();
+ final long seed = System.currentTimeMillis();
+ final boolean useString = "string".equals(type);
rnd = new Random(seed);
@@ -109,7 +115,7 @@ public class TupleMarshallerVarlenOnlyBenchmark {
42,
new Column[] {new Column("key", LONG, false)},
IntStream.range(0, fieldsCount).boxed()
- .map(i -> new Column("col" + i, BYTES, nullable))
+ .map(i -> new Column("col" + i, useString ? STRING : BYTES, nullable))
.toArray(Column[]::new)
);
@@ -123,9 +129,19 @@ public class TupleMarshallerVarlenOnlyBenchmark {
}
});
- val = new byte[dataSize / fieldsCount];
+ if (useString) {
+ final byte[] data = new byte[dataSize / fieldsCount];
+
+ for (int i = 0; i < data.length; i++) {
+ final byte b = (byte)rnd.nextInt();
+
+ data[i] = b < 0 ? (byte)(-b) : b;
+ }
- rnd.nextBytes(val);
+ val = new String(data); // Latin1 string.
+ }
+ else
+ rnd.nextBytes((byte[])(val = new byte[dataSize / fieldsCount]));
}
/**
[ignite-3] 02/02: Minor optimizations.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch ignite-14743-row-formats
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 1181d7c2eaed00b02ce56576fdfdad41776ce9f6
Author: Andrew Mashenkov <an...@gmail.com>
AuthorDate: Fri Jun 25 18:21:02 2021 +0300
Minor optimizations.
---
.../main/java/org/apache/ignite/table/Tuple.java | 5 +-
.../ignite/internal/table/RowChunkAdapter.java | 11 +++
.../org/apache/ignite/internal/table/TableRow.java | 23 +-----
.../ignite/internal/table/TupleBuilderImpl.java | 6 +-
.../ignite/internal/table/TupleMarshallerImpl.java | 91 +++++++++++-----------
.../TupleMarshallerVarlenOnlyBenchmark.java | 2 +-
.../internal/table/impl/TestTupleBuilder.java | 20 ++---
7 files changed, 77 insertions(+), 81 deletions(-)
diff --git a/modules/api/src/main/java/org/apache/ignite/table/Tuple.java b/modules/api/src/main/java/org/apache/ignite/table/Tuple.java
index c9e0374..93605d5 100644
--- a/modules/api/src/main/java/org/apache/ignite/table/Tuple.java
+++ b/modules/api/src/main/java/org/apache/ignite/table/Tuple.java
@@ -31,9 +31,10 @@ public interface Tuple {
* Returns {@code true} if this tuple contains a column with the specified name.
*
* @param colName Column name.
- * @return {@code true} if this tuple contains a column with the specified name. Otherwise returns {@code false}.
+ * @param def Default value.
+ * @return Column value if this tuple contains a column with the specified name. Otherwise returns {@code default}.
*/
- boolean contains(String colName);
+ <T> T valueOrDefault(String colName, T def);
/**
* Gets column value for given column name.
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/RowChunkAdapter.java b/modules/table/src/main/java/org/apache/ignite/internal/table/RowChunkAdapter.java
index 2e213f1..dc9ab74 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/RowChunkAdapter.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/RowChunkAdapter.java
@@ -33,6 +33,7 @@ public abstract class RowChunkAdapter implements Tuple {
/**
* @param colName Column name.
* @return Column.
+ * @throws ColumnNotFoundException If column wasn't found.
*/
@NotNull protected abstract Column columnByName(@NotNull String colName);
@@ -42,6 +43,16 @@ public abstract class RowChunkAdapter implements Tuple {
protected abstract Row row();
/** {@inheritDoc} */
+ @Override public <T> T valueOrDefault(String colName, T def) {
+ try {
+ return value(colName);
+ }
+ catch (ColumnNotFoundException ex) {
+ return def;
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public <T> T value(String colName) {
final Column col = columnByName(colName);
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/TableRow.java b/modules/table/src/main/java/org/apache/ignite/internal/table/TableRow.java
index 31ab606..de111e2 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/TableRow.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/TableRow.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.table;
import java.util.Objects;
import org.apache.ignite.internal.schema.Column;
-import org.apache.ignite.internal.schema.row.Row;
import org.apache.ignite.internal.schema.SchemaDescriptor;
+import org.apache.ignite.internal.schema.row.Row;
import org.apache.ignite.table.Tuple;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -57,7 +57,7 @@ public class TableRow extends RowChunkAdapter {
final Column col = schema.column(colName);
if (col == null)
- throw new IllegalArgumentException("Invalid column name: columnName=" + colName + ", schemaVersion=" + schema.version());
+ throw new ColumnNotFoundException("Invalid column name: columnName=" + colName + ", schemaVersion=" + schema.version());
return col;
}
@@ -81,11 +81,6 @@ public class TableRow extends RowChunkAdapter {
return row;
}
- /** */
- @Override public boolean contains(String colName) {
- return schema.column(colName) != null;
- }
-
/** Key column chunk. */
private class KeyRowChunk extends RowChunkAdapter {
/** {@inheritDoc} */
@@ -100,15 +95,10 @@ public class TableRow extends RowChunkAdapter {
final Column col = schema.column(colName);
if (col == null || !schema.isKeyColumn(col.schemaIndex()))
- throw new IllegalArgumentException("Invalid key column name: columnName=" + colName + ", schemaVersion=" + schema.version());
+ throw new ColumnNotFoundException("Invalid key column name: columnName=" + colName + ", schemaVersion=" + schema.version());
return col;
}
-
- /** */
- @Override public boolean contains(String colName) {
- return schema.column(colName) != null;
- }
}
/** Value column chunk. */
@@ -125,14 +115,9 @@ public class TableRow extends RowChunkAdapter {
final Column col = schema.column(colName);
if (col == null || schema.isKeyColumn(col.schemaIndex()))
- throw new IllegalArgumentException("Invalid key column name: columnName=" + colName + ", schemaVersion=" + schema.version());
+ throw new ColumnNotFoundException("Invalid key column name: columnName=" + colName + ", schemaVersion=" + schema.version());
return col;
}
-
- /** */
- @Override public boolean contains(String colName) {
- return schema.column(colName) != null;
- }
}
}
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/TupleBuilderImpl.java b/modules/table/src/main/java/org/apache/ignite/internal/table/TupleBuilderImpl.java
index 66e93aa..10dcfcd 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/TupleBuilderImpl.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/TupleBuilderImpl.java
@@ -48,7 +48,7 @@ public class TupleBuilderImpl implements TupleBuilder, Tuple {
Objects.requireNonNull(schemaDesc);
this.schemaDesc = schemaDesc;
- map = new HashMap<>();
+ map = new HashMap<>(schemaDesc.length());
}
/** {@inheritDoc} */
@@ -71,8 +71,8 @@ public class TupleBuilderImpl implements TupleBuilder, Tuple {
}
/** {@inheritDoc} */
- @Override public boolean contains(String colName) {
- return map.containsKey(colName);
+ @Override public <T> T valueOrDefault(String colName, T def) {
+ return (T)map.getOrDefault(colName, def);
}
@Override public <T> T value(String colName) {
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/TupleMarshallerImpl.java b/modules/table/src/main/java/org/apache/ignite/internal/table/TupleMarshallerImpl.java
index 0406bd6..1407b0f 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/TupleMarshallerImpl.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/TupleMarshallerImpl.java
@@ -62,10 +62,7 @@ public class TupleMarshallerImpl implements TupleMarshaller {
validate(keyTuple, schema.keyColumns());
- TupleStatistics keyStat = tupleStatistics(schema.keyColumns(), keyTuple);
- TupleStatistics valStat = tupleStatistics(schema.valueColumns(), valTuple);
-
- final RowAssembler rowBuilder = createAssembler(schema, keyStat, valStat);
+ final RowAssembler rowBuilder = createAssembler(schema, keyTuple, valTuple);
for (int i = 0; i < schema.keyColumns().length(); i++) {
final Column col = schema.keyColumns().column(i);
@@ -87,38 +84,6 @@ public class TupleMarshallerImpl implements TupleMarshaller {
}
/**
- * Analyze given tuple and gather statistics.
- *
- * @param cols Columns which statistics is calculated for.
- * @param tup Tuple to analyze.
- * @return Tuple statistics.
- */
- private TupleStatistics tupleStatistics(Columns cols, Tuple tup) {
- if (tup == null)
- return new TupleStatistics();
-
- TupleStatistics chunk = new TupleStatistics();
-
- for (int i = 0; i < cols.length(); i++) {
- Column col = cols.column(i);
-
- Object val = (tup.contains(col.name())) ? tup.value(col.name()) : col.defaultValue();
-
- if (val == null)
- chunk.hasNulls = true;
- else if (col.type().spec().fixedLength())
- chunk.payloadLen += col.type().sizeInBytes();
- else {
- chunk.nonNullVarlen++;
-
- chunk.payloadLen += getValueSize(val, col.type());
- }
- }
-
- return chunk;
- }
-
- /**
* Validates columns values.
*
* @param tuple Tuple to validate.
@@ -142,11 +107,14 @@ public class TupleMarshallerImpl implements TupleMarshaller {
/**
* Creates {@link RowAssembler} for key-value tuples.
*
- * @param keyStat Key tuple statistics.
- * @param valStat Value tuple statistics.
+ * @param keyTuple Key tuple.
+ * @param valTuple Value tuple.
* @return Row assembler.
*/
- private RowAssembler createAssembler(SchemaDescriptor schema, TupleStatistics keyStat, TupleStatistics valStat) {
+ private RowAssembler createAssembler(SchemaDescriptor schema, Tuple keyTuple, Tuple valTuple) {
+ TupleStatistics keyStat = tupleStatistics(schema.keyColumns(), keyTuple);
+ TupleStatistics valStat = tupleStatistics(schema.valueColumns(), valTuple);
+
return new RowAssembler(
schema,
keyStat.payloadLen,
@@ -163,7 +131,7 @@ public class TupleMarshallerImpl implements TupleMarshaller {
* @param tup Tuple.
*/
private void writeColumn(RowAssembler rowAsm, Column col, Tuple tup) {
- Object val = tup.contains(col.name()) ? tup.value(col.name()) : col.defaultValue();
+ Object val = tup.valueOrDefault(col.name(), col.defaultValue());
if (val == null) {
rowAsm.appendNull();
@@ -228,16 +196,51 @@ public class TupleMarshallerImpl implements TupleMarshaller {
}
/**
+ * Analyze given tuple and gather statistics.
+ *
+ * @param cols Columns which statistics is calculated for.
+ * @param tup Tuple to analyze.
+ * @return Tuple statistics.
+ */
+ private TupleStatistics tupleStatistics(Columns cols, Tuple tup) {
+ if (tup == null)
+ return TupleStatistics.ZERO_VARLEN_STATISTICS;
+
+ TupleStatistics chunk = new TupleStatistics();
+
+ for (int i = 0; i < cols.length(); i++) {
+ Column col = cols.column(i);
+
+ final Object val = tup.valueOrDefault(col.name(), col.defaultValue());
+
+ if (val == null)
+ chunk.hasNulls = true;
+ else if (col.type().spec().fixedLength())
+ chunk.payloadLen += col.type().sizeInBytes();
+ else {
+ chunk.nonNullVarlen++;
+
+ chunk.payloadLen += getValueSize(val, col.type());
+ }
+ }
+
+ return chunk;
+ }
+
+ /**
* Tuple statistics record.
*/
private static class TupleStatistics {
- /** Tuple has nulls. */
- boolean hasNulls;
+ /** Cached zero statistics. */
+ static final TupleStatistics ZERO_VARLEN_STATISTICS = new TupleStatistics();
+
+ /** Number of non-null varlen columns. */
+ int nonNullVarlen;
/** Payload length in bytes. */
int payloadLen;
- /** Number of non-null varlen columns. */
- int nonNullVarlen;
+ /** Tuple has nulls. */
+ boolean hasNulls;
}
}
diff --git a/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java b/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java
index 2d25935..de5b063 100644
--- a/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java
+++ b/modules/table/src/test/java/org/apache/ignite/internal/benchmarks/TupleMarshallerVarlenOnlyBenchmark.java
@@ -77,7 +77,7 @@ public class TupleMarshallerVarlenOnlyBenchmark {
/** Nullable cols. */
@Param({"true", "false"})
- public boolean nullable ;
+ public boolean nullable;
/** Column types. */
@Param({"string", "bytes"})
diff --git a/modules/table/src/test/java/org/apache/ignite/internal/table/impl/TestTupleBuilder.java b/modules/table/src/test/java/org/apache/ignite/internal/table/impl/TestTupleBuilder.java
index 00450cf..f8520dc 100644
--- a/modules/table/src/test/java/org/apache/ignite/internal/table/impl/TestTupleBuilder.java
+++ b/modules/table/src/test/java/org/apache/ignite/internal/table/impl/TestTupleBuilder.java
@@ -24,34 +24,30 @@ import java.util.UUID;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.binary.BinaryObjects;
import org.apache.ignite.table.Tuple;
+import org.apache.ignite.table.TupleBuilder;
/**
* Dummy table storage implementation.
*/
-public class TestTupleBuilder implements Tuple {
+public class TestTupleBuilder implements TupleBuilder, Tuple {
/** Columns values. */
private final Map<String, Object> map = new HashMap<>();
- /**
- * Constructor.
- */
- public TestTupleBuilder() {
- }
-
- public TestTupleBuilder set(String colName, Object value) {
+ /** {@inheritDoc} */
+ @Override public TestTupleBuilder set(String colName, Object value) {
map.put(colName, value);
return this;
}
- /** */
- public Tuple build() {
+ /** {@inheritDoc} */
+ @Override public Tuple build() {
return this;
}
/** {@inheritDoc} */
- @Override public boolean contains(String colName) {
- return map.containsKey(colName);
+ @Override public <T> T valueOrDefault(String colName, T def) {
+ return (T)map.getOrDefault(colName, def);
}
/** {@inheritDoc} */