You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/01/10 14:05:11 UTC
(incubator-paimon) 02/06: [flink] Refactor FileStoreLookupFunction to abstract more into LookupTable
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit b13d99f4bfb43034ca86459010e34da6926a0df2
Author: Jingsong <ji...@gmail.com>
AuthorDate: Tue Jan 9 14:53:46 2024 +0800
[flink] Refactor FileStoreLookupFunction to abstract more into LookupTable
---
.../generated/flink_connector_configuration.html | 4 +-
.../apache/paimon/predicate/PredicateFilter.java | 5 +-
.../apache/paimon/flink/FlinkConnectorOptions.java | 6 +-
.../flink/lookup/FileStoreLookupFunction.java | 130 +++---------
...ractor.java => FixedBucketFromPkExtractor.java} | 4 +-
.../paimon/flink/lookup/FullCacheLookupTable.java | 172 +++++++++++++++-
.../apache/paimon/flink/lookup/LookupTable.java | 82 +-------
.../flink/lookup/NoPrimaryKeyLookupTable.java | 35 +---
.../flink/lookup/PartialCacheLookupTable.java | 30 ---
.../paimon/flink/lookup/PrimaryKeyLookupTable.java | 59 ++----
.../flink/lookup/PrimaryKeyPartialLookupTable.java | 42 +++-
.../flink/lookup/SecondaryIndexLookupTable.java | 49 ++---
.../paimon/flink/source/DataTableSource.java | 8 -
.../org/apache/paimon/flink/LookupJoinITCase.java | 87 ++------
.../paimon/flink/lookup/LookupTableTest.java | 223 ++++++++++++---------
15 files changed, 432 insertions(+), 504 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/flink_connector_configuration.html b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
index 5e407fa53..099d9127c 100644
--- a/docs/layouts/shortcodes/generated/flink_connector_configuration.html
+++ b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
@@ -52,9 +52,9 @@ under the License.
</tr>
<tr>
<td><h5>lookup.cache</h5></td>
- <td style="word-wrap: break-word;">FULL</td>
+ <td style="word-wrap: break-word;">AUTO</td>
<td><p>Enum</p></td>
- <td>The cache mode of lookup join.<br /><br />Possible values:<ul><li>"PARTIAL"</li><li>"FULL"</li></ul></td>
+ <td>The cache mode of lookup join.<br /><br />Possible values:<ul><li>"AUTO"</li><li>"FULL"</li></ul></td>
</tr>
<tr>
<td><h5>scan.infer-parallelism</h5></td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java
index 900214c42..f88bc1445 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java
@@ -20,14 +20,15 @@ package org.apache.paimon.predicate;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Filter;
import org.apache.paimon.utils.RowDataToObjectArrayConverter;
import javax.annotation.Nullable;
import java.util.List;
-/** A {@link java.util.function.Predicate} to filter {@link InternalRow}. */
-public class PredicateFilter implements java.util.function.Predicate<InternalRow> {
+/** A {@link Filter} to filter {@link InternalRow}. */
+public class PredicateFilter implements Filter<InternalRow> {
private final RowDataToObjectArrayConverter arrayConverter;
@Nullable private final Predicate predicate;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
index ea41831e7..ae6451916 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
@@ -290,7 +290,7 @@ public class FlinkConnectorOptions {
public static final ConfigOption<LookupCacheMode> LOOKUP_CACHE_MODE =
ConfigOptions.key("lookup.cache")
.enumType(LookupCacheMode.class)
- .defaultValue(LookupCacheMode.FULL)
+ .defaultValue(LookupCacheMode.AUTO)
.withDescription("The cache mode of lookup join.");
public static final ConfigOption<Boolean> SINK_AUTO_TAG_FOR_SAVEPOINT =
@@ -331,8 +331,8 @@ public class FlinkConnectorOptions {
/** The mode of lookup cache. */
public enum LookupCacheMode {
- /** Use partial caching mode. */
- PARTIAL,
+ /** Auto mode, try to use partial mode. */
+ AUTO,
/** Use full caching mode. */
FULL
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
index 27fd0b2dd..96b7b0b47 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
@@ -18,32 +18,18 @@
package org.apache.paimon.flink.lookup;
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.disk.IOManager;
-import org.apache.paimon.flink.FlinkConnectorOptions;
import org.apache.paimon.flink.FlinkConnectorOptions.LookupCacheMode;
import org.apache.paimon.flink.FlinkRowData;
import org.apache.paimon.flink.FlinkRowWrapper;
-import org.apache.paimon.flink.lookup.FullCacheLookupTable.TableBulkLoader;
import org.apache.paimon.flink.utils.TableScanUtils;
-import org.apache.paimon.lookup.BulkLoader;
-import org.apache.paimon.lookup.RocksDBState;
-import org.apache.paimon.lookup.RocksDBStateFactory;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateFilter;
-import org.apache.paimon.reader.RecordReaderIterator;
-import org.apache.paimon.sort.BinaryExternalSortBuffer;
+import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.source.OutOfRangeException;
-import org.apache.paimon.types.DataTypes;
-import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FileIOUtils;
-import org.apache.paimon.utils.MutableObjectIterator;
-import org.apache.paimon.utils.PartialRow;
import org.apache.paimon.utils.TypeUtils;
import org.apache.paimon.shade.guava30.com.google.common.primitives.Ints;
@@ -66,6 +52,7 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
@@ -73,10 +60,10 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.apache.paimon.CoreOptions.CONTINUOUS_DISCOVERY_INTERVAL;
+import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_CACHE_MODE;
import static org.apache.paimon.lookup.RocksDBOptions.LOOKUP_CACHE_ROWS;
import static org.apache.paimon.lookup.RocksDBOptions.LOOKUP_CONTINUOUS_DISCOVERY_INTERVAL;
import static org.apache.paimon.predicate.PredicateBuilder.transformFieldMapping;
-import static org.apache.paimon.schema.SystemColumns.SEQUENCE_NUMBER;
/** A lookup {@link TableFunction} for file store. */
public class FileStoreLookupFunction implements Serializable, Closeable {
@@ -92,16 +79,10 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
private transient Duration refreshInterval;
private transient File path;
- private transient RocksDBStateFactory stateFactory;
private transient LookupTable lookupTable;
// timestamp when cache expires
private transient long nextLoadTime;
- private transient TableStreamingReader streamingReader;
-
- private final boolean sequenceFieldEnabled;
-
- private transient TableFileMonitor fileMonitor;
public FileStoreLookupFunction(
Table table, int[] projection, int[] joinKeyIndex, @Nullable Predicate predicate) {
@@ -128,9 +109,6 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
}
this.predicate = predicate;
- this.sequenceFieldEnabled =
- table.primaryKeys().size() > 0
- && new CoreOptions(table.options()).sequenceField().isPresent();
}
public void open(FunctionContext context) throws Exception {
@@ -148,79 +126,39 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
}
private void open() throws Exception {
+ this.nextLoadTime = -1;
+
Options options = Options.fromMap(table.options());
this.refreshInterval =
options.getOptional(LOOKUP_CONTINUOUS_DISCOVERY_INTERVAL)
.orElse(options.get(CONTINUOUS_DISCOVERY_INTERVAL));
- LookupCacheMode cacheMode = options.get(FlinkConnectorOptions.LOOKUP_CACHE_MODE);
List<String> fieldNames = table.rowType().getFieldNames();
int[] projection = projectFields.stream().mapToInt(fieldNames::indexOf).toArray();
- RowType rowType = TypeUtils.project(table.rowType(), projection);
+ FileStoreTable storeTable = (FileStoreTable) table;
- PredicateFilter recordFilter = createRecordFilter(projection);
-
- switch (cacheMode) {
- case FULL:
- this.stateFactory = new RocksDBStateFactory(path.toString(), options, null);
+ if (options.get(LOOKUP_CACHE_MODE) == LookupCacheMode.AUTO
+ && new HashSet<>(table.primaryKeys()).equals(new HashSet<>(joinKeys))) {
+ try {
this.lookupTable =
- LookupTable.create(
- stateFactory,
- sequenceFieldEnabled
- ? rowType.appendDataField(
- SEQUENCE_NUMBER, DataTypes.BIGINT())
- : rowType,
- table.primaryKeys(),
- joinKeys,
- recordFilter,
- options.get(LOOKUP_CACHE_ROWS));
- this.streamingReader = new TableStreamingReader(table, projection, this.predicate);
- bulkLoad(options);
- break;
- case PARTIAL:
- this.lookupTable = LookupTable.create(table, projection, joinKeys, path);
- this.fileMonitor = new TableFileMonitor(table, this.predicate);
- ((PartialCacheLookupTable) lookupTable).refresh(fileMonitor.readChanges());
- break;
- default:
- throw new IllegalArgumentException("Unsupported lookup cache mode: " + cacheMode);
- }
- this.nextLoadTime = -1;
- }
-
- private void bulkLoad(Options options) throws Exception {
- BinaryExternalSortBuffer bulkLoadSorter =
- RocksDBState.createBulkLoadSorter(
- IOManager.create(path.toString()), new CoreOptions(options));
- FullCacheLookupTable lookupTable = (FullCacheLookupTable) this.lookupTable;
- try (RecordReaderIterator<InternalRow> batch =
- new RecordReaderIterator<>(streamingReader.nextBatch(true, sequenceFieldEnabled))) {
- while (batch.hasNext()) {
- InternalRow row = batch.next();
- if (lookupTable.recordFilter().test(row)) {
- bulkLoadSorter.write(
- GenericRow.of(
- lookupTable.toKeyBytes(row), lookupTable.toValueBytes(row)));
- }
+ new PrimaryKeyPartialLookupTable(storeTable, predicate, projection, path);
+ } catch (UnsupportedOperationException ignore) {
}
}
- MutableObjectIterator<BinaryRow> keyIterator = bulkLoadSorter.sortedIterator();
- BinaryRow row = new BinaryRow(2);
- TableBulkLoader bulkLoader = lookupTable.createBulkLoader();
- try {
- while ((row = keyIterator.next(row)) != null) {
- bulkLoader.write(row.getBinary(0), row.getBinary(1));
- }
- } catch (BulkLoader.WriteException e) {
- throw new RuntimeException(
- "Exception in bulkLoad, the most suspicious reason is that "
- + "your data contains duplicates, please check your lookup table. ",
- e.getCause());
+ if (lookupTable == null) {
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ projection,
+ predicate,
+ path,
+ createRecordFilter(projection),
+ joinKeys);
+ this.lookupTable = FullCacheLookupTable.create(context, options.get(LOOKUP_CACHE_ROWS));
}
- bulkLoader.finish();
- bulkLoadSorter.clear();
+ lookupTable.open();
}
private PredicateFilter createRecordFilter(int[] projection) {
@@ -245,9 +183,6 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
List<InternalRow> results = lookupTable.get(new FlinkRowWrapper(keyRow));
List<RowData> rows = new ArrayList<>(results.size());
for (InternalRow matchedRow : results) {
- if (sequenceFieldEnabled) {
- matchedRow = new PartialRow(matchedRow.getFieldCount() - 1, matchedRow);
- }
rows.add(new FlinkRowData(matchedRow));
}
return rows;
@@ -285,27 +220,14 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
}
private void refresh() throws Exception {
- if (lookupTable instanceof FullCacheLookupTable) {
- while (true) {
- try (RecordReaderIterator<InternalRow> batch =
- new RecordReaderIterator<>(
- streamingReader.nextBatch(false, sequenceFieldEnabled))) {
- if (!batch.hasNext()) {
- return;
- }
- ((FullCacheLookupTable) this.lookupTable).refresh(batch, sequenceFieldEnabled);
- }
- }
- } else {
- ((PartialCacheLookupTable) lookupTable).refresh(fileMonitor.readChanges());
- }
+ lookupTable.refresh();
}
@Override
public void close() throws IOException {
- if (stateFactory != null) {
- stateFactory.close();
- stateFactory = null;
+ if (lookupTable != null) {
+ lookupTable.close();
+ lookupTable = null;
}
if (path != null) {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketKeyExtractor.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketFromPkExtractor.java
similarity index 96%
rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketKeyExtractor.java
rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketFromPkExtractor.java
index 07e524a26..2c296992a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketKeyExtractor.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketFromPkExtractor.java
@@ -29,7 +29,7 @@ import org.apache.paimon.table.sink.KeyAndBucketExtractor;
import static org.apache.paimon.utils.Preconditions.checkArgument;
/** Extractor to extract bucket from the primary key. */
-public class FixedBucketKeyExtractor implements KeyAndBucketExtractor<InternalRow> {
+public class FixedBucketFromPkExtractor implements KeyAndBucketExtractor<InternalRow> {
private transient InternalRow primaryKey;
@@ -47,7 +47,7 @@ public class FixedBucketKeyExtractor implements KeyAndBucketExtractor<InternalRo
private final Projection logPrimaryKeyProjection;
- public FixedBucketKeyExtractor(TableSchema schema) {
+ public FixedBucketFromPkExtractor(TableSchema schema) {
this.numBuckets = new CoreOptions(schema.options()).bucket();
checkArgument(numBuckets > 0, "Num bucket is illegal: " + numBuckets);
this.sameBucketKeyAndTrimmedPrimaryKey =
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
index b9988e1ca..ed4451053 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
@@ -20,31 +20,187 @@
package org.apache.paimon.flink.lookup;
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.disk.IOManager;
import org.apache.paimon.lookup.BulkLoader;
+import org.apache.paimon.lookup.RocksDBState;
+import org.apache.paimon.lookup.RocksDBStateFactory;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.reader.RecordReaderIterator;
+import org.apache.paimon.sort.BinaryExternalSortBuffer;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Filter;
+import org.apache.paimon.utils.MutableObjectIterator;
+import org.apache.paimon.utils.PartialRow;
+import org.apache.paimon.utils.TypeUtils;
+import javax.annotation.Nullable;
+
+import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.Iterator;
-import java.util.function.Predicate;
+import java.util.List;
+
+import static org.apache.paimon.schema.SystemColumns.SEQUENCE_NUMBER;
/** Lookup table of full cache. */
-public interface FullCacheLookupTable extends LookupTable {
+public abstract class FullCacheLookupTable implements LookupTable {
+
+ protected final Context context;
+ protected final RocksDBStateFactory stateFactory;
+ protected final RowType projectedType;
+ private final TableStreamingReader reader;
+ private final boolean sequenceFieldEnabled;
+
+ public FullCacheLookupTable(Context context) throws IOException {
+ this.context = context;
+ this.stateFactory =
+ new RocksDBStateFactory(
+ context.tempPath.toString(),
+ context.table.coreOptions().toConfiguration(),
+ null);
+ FileStoreTable table = context.table;
+ this.reader = new TableStreamingReader(table, context.projection, context.predicate);
+ this.sequenceFieldEnabled =
+ table.primaryKeys().size() > 0
+ && new CoreOptions(table.options()).sequenceField().isPresent();
+ RowType projectedType = TypeUtils.project(table.rowType(), context.projection);
+ if (sequenceFieldEnabled) {
+ projectedType = projectedType.appendDataField(SEQUENCE_NUMBER, DataTypes.BIGINT());
+ }
+ this.projectedType = projectedType;
+ }
+
+ @Override
+ public void open() throws Exception {
+ BinaryExternalSortBuffer bulkLoadSorter =
+ RocksDBState.createBulkLoadSorter(
+ IOManager.create(context.tempPath.toString()), context.table.coreOptions());
+ try (RecordReaderIterator<InternalRow> batch =
+ new RecordReaderIterator<>(reader.nextBatch(true, sequenceFieldEnabled))) {
+ while (batch.hasNext()) {
+ InternalRow row = batch.next();
+ if (recordFilter().test(row)) {
+ bulkLoadSorter.write(GenericRow.of(toKeyBytes(row), toValueBytes(row)));
+ }
+ }
+ }
+
+ MutableObjectIterator<BinaryRow> keyIterator = bulkLoadSorter.sortedIterator();
+ BinaryRow row = new BinaryRow(2);
+ TableBulkLoader bulkLoader = createBulkLoader();
+ try {
+ while ((row = keyIterator.next(row)) != null) {
+ bulkLoader.write(row.getBinary(0), row.getBinary(1));
+ }
+ } catch (BulkLoader.WriteException e) {
+ throw new RuntimeException(
+ "Exception in bulkLoad, the most suspicious reason is that "
+ + "your data contains duplicates, please check your lookup table. ",
+ e.getCause());
+ }
+
+ bulkLoader.finish();
+ bulkLoadSorter.clear();
+ }
+
+ public void refresh() throws Exception {
+ while (true) {
+ try (RecordReaderIterator<InternalRow> batch =
+ new RecordReaderIterator<>(reader.nextBatch(false, sequenceFieldEnabled))) {
+ if (!batch.hasNext()) {
+ return;
+ }
+ refresh(batch, sequenceFieldEnabled);
+ }
+ }
+ }
+
+ @Override
+ public final List<InternalRow> get(InternalRow key) throws IOException {
+ List<InternalRow> values = innerGet(key);
+ if (!sequenceFieldEnabled) {
+ return values;
+ }
- void refresh(Iterator<InternalRow> input, boolean orderByLastField) throws IOException;
+ List<InternalRow> dropSequence = new ArrayList<>(values.size());
+ for (InternalRow matchedRow : values) {
+ dropSequence.add(new PartialRow(matchedRow.getFieldCount() - 1, matchedRow));
+ }
+ return dropSequence;
+ }
+
+ public abstract List<InternalRow> innerGet(InternalRow key) throws IOException;
+
+ public abstract void refresh(Iterator<InternalRow> input, boolean orderByLastField)
+ throws IOException;
- Predicate<InternalRow> recordFilter();
+ public Filter<InternalRow> recordFilter() {
+ return context.recordFilter;
+ }
+
+ public abstract byte[] toKeyBytes(InternalRow row) throws IOException;
- byte[] toKeyBytes(InternalRow row) throws IOException;
+ public abstract byte[] toValueBytes(InternalRow row) throws IOException;
- byte[] toValueBytes(InternalRow row) throws IOException;
+ public abstract TableBulkLoader createBulkLoader();
- TableBulkLoader createBulkLoader();
+ @Override
+ public void close() throws IOException {
+ stateFactory.close();
+ }
/** Bulk loader for the table. */
- interface TableBulkLoader {
+ public interface TableBulkLoader {
void write(byte[] key, byte[] value) throws BulkLoader.WriteException, IOException;
void finish() throws IOException;
}
+
+ static FullCacheLookupTable create(Context context, long lruCacheSize) throws IOException {
+ List<String> primaryKeys = context.table.primaryKeys();
+ if (primaryKeys.isEmpty()) {
+ return new NoPrimaryKeyLookupTable(context, lruCacheSize);
+ } else {
+ if (new HashSet<>(primaryKeys).equals(new HashSet<>(context.joinKey))) {
+ return new PrimaryKeyLookupTable(context, lruCacheSize);
+ } else {
+ return new SecondaryIndexLookupTable(context, lruCacheSize);
+ }
+ }
+ }
+
+ /** Context for {@link LookupTable}. */
+ public static class Context {
+
+ public final FileStoreTable table;
+ public final int[] projection;
+ public final @Nullable Predicate predicate;
+ public final File tempPath;
+ public final Filter<InternalRow> recordFilter;
+ public final List<String> joinKey;
+
+ public Context(
+ FileStoreTable table,
+ int[] projection,
+ @Nullable Predicate predicate,
+ File tempPath,
+ Filter<InternalRow> recordFilter,
+ List<String> joinKey) {
+ this.table = table;
+ this.projection = projection;
+ this.predicate = predicate;
+ this.tempPath = tempPath;
+ this.recordFilter = recordFilter;
+ this.joinKey = joinKey;
+ }
+ }
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java
index 50bca16aa..e89259073 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java
@@ -18,90 +18,18 @@
package org.apache.paimon.flink.lookup;
-import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.flink.FlinkConnectorOptions.LookupCacheMode;
-import org.apache.paimon.lookup.RocksDBStateFactory;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.Table;
-import org.apache.paimon.types.RowType;
-import java.io.File;
+import java.io.Closeable;
import java.io.IOException;
-import java.util.HashSet;
import java.util.List;
-import java.util.function.Predicate;
-
-import static org.apache.paimon.CoreOptions.MergeEngine.DEDUPLICATE;
-import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_CACHE_MODE;
/** A lookup table which provides get and refresh. */
-public interface LookupTable {
-
- List<InternalRow> get(InternalRow key) throws IOException;
-
- static FullCacheLookupTable create(
- RocksDBStateFactory stateFactory,
- RowType rowType,
- List<String> primaryKey,
- List<String> joinKey,
- Predicate<InternalRow> recordFilter,
- long lruCacheSize)
- throws IOException {
- if (primaryKey.isEmpty()) {
- return new NoPrimaryKeyLookupTable(
- stateFactory, rowType, joinKey, recordFilter, lruCacheSize);
- } else {
- if (new HashSet<>(primaryKey).equals(new HashSet<>(joinKey))) {
- return new PrimaryKeyLookupTable(
- stateFactory, rowType, joinKey, recordFilter, lruCacheSize);
- } else {
- return new SecondaryIndexLookupTable(
- stateFactory, rowType, primaryKey, joinKey, recordFilter, lruCacheSize);
- }
- }
- }
+public interface LookupTable extends Closeable {
- static PartialCacheLookupTable create(
- Table table, int[] projection, List<String> joinKey, File tempPath) {
- validatePartialCacheMode(table, joinKey);
- return new PrimaryKeyPartialLookupTable((FileStoreTable) table, projection, tempPath);
- }
+ void open() throws Exception;
- static void validatePartialCacheMode(Table table, List<String> joinKey) {
- CoreOptions options = new CoreOptions(table.options());
- if (options.toConfiguration().get(LOOKUP_CACHE_MODE) != LookupCacheMode.PARTIAL) {
- return;
- }
- List<String> primaryKey = table.primaryKeys();
- if (!new HashSet<>(primaryKey).equals(new HashSet<>(joinKey))) {
- throw new UnsupportedOperationException(
- String.format(
- "For partial cache mode, the join key: %s must equal to the primary key: %s",
- joinKey, primaryKey));
- }
-
- if (table.partitionKeys().size() > 0) {
- throw new UnsupportedOperationException(
- "The partitioned table are not supported in partial cache mode.");
- }
-
- if (new CoreOptions(table.options()).bucket() <= 0) {
- throw new UnsupportedOperationException(
- "Only support fixed bucket mode in partial cache mode.");
- }
-
- if (options.changelogProducer() != CoreOptions.ChangelogProducer.LOOKUP) {
- if (options.sequenceField().isPresent()) {
- throw new UnsupportedOperationException(
- "Not support sequence field definition, but is: "
- + options.sequenceField().get());
- }
+ List<InternalRow> get(InternalRow key) throws IOException;
- if (options.mergeEngine() != DEDUPLICATE) {
- throw new UnsupportedOperationException(
- "Only support deduplicate merge engine, but is: " + options.mergeEngine());
- }
- }
- }
+ void refresh() throws Exception;
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java
index d6979e9b7..329cd0d6a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java
@@ -24,9 +24,7 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalSerializers;
import org.apache.paimon.lookup.BulkLoader;
import org.apache.paimon.lookup.RocksDBListState;
-import org.apache.paimon.lookup.RocksDBStateFactory;
import org.apache.paimon.types.RowKind;
-import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.KeyProjectedRow;
import org.apache.paimon.utils.TypeUtils;
@@ -35,38 +33,30 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import java.util.function.Predicate;
/** A {@link LookupTable} for table without primary key. */
-public class NoPrimaryKeyLookupTable implements FullCacheLookupTable {
+public class NoPrimaryKeyLookupTable extends FullCacheLookupTable {
private final RocksDBListState<InternalRow, InternalRow> state;
- private final Predicate<InternalRow> recordFilter;
-
private final KeyProjectedRow joinKeyRow;
- public NoPrimaryKeyLookupTable(
- RocksDBStateFactory stateFactory,
- RowType rowType,
- List<String> joinKeys,
- Predicate<InternalRow> recordFilter,
- long lruCacheSize)
- throws IOException {
- List<String> fieldNames = rowType.getFieldNames();
- int[] joinKeyMapping = joinKeys.stream().mapToInt(fieldNames::indexOf).toArray();
+ public NoPrimaryKeyLookupTable(Context context, long lruCacheSize) throws IOException {
+ super(context);
+ List<String> fieldNames = projectedType.getFieldNames();
+ int[] joinKeyMapping = context.joinKey.stream().mapToInt(fieldNames::indexOf).toArray();
this.joinKeyRow = new KeyProjectedRow(joinKeyMapping);
this.state =
stateFactory.listState(
"join-key-index",
- InternalSerializers.create(TypeUtils.project(rowType, joinKeyMapping)),
- InternalSerializers.create(rowType),
+ InternalSerializers.create(
+ TypeUtils.project(projectedType, joinKeyMapping)),
+ InternalSerializers.create(projectedType),
lruCacheSize);
- this.recordFilter = recordFilter;
}
@Override
- public List<InternalRow> get(InternalRow key) throws IOException {
+ public List<InternalRow> innerGet(InternalRow key) throws IOException {
return state.get(key);
}
@@ -82,7 +72,7 @@ public class NoPrimaryKeyLookupTable implements FullCacheLookupTable {
InternalRow row = incremental.next();
joinKeyRow.replaceRow(row);
if (row.getRowKind() == RowKind.INSERT || row.getRowKind() == RowKind.UPDATE_AFTER) {
- if (recordFilter.test(row)) {
+ if (recordFilter().test(row)) {
state.add(joinKeyRow, row);
}
} else {
@@ -94,11 +84,6 @@ public class NoPrimaryKeyLookupTable implements FullCacheLookupTable {
}
}
- @Override
- public Predicate<InternalRow> recordFilter() {
- return recordFilter;
- }
-
@Override
public byte[] toKeyBytes(InternalRow row) throws IOException {
joinKeyRow.replaceRow(row);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartialCacheLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartialCacheLookupTable.java
deleted file mode 100644
index a82507ffa..000000000
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartialCacheLookupTable.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.paimon.flink.lookup;
-
-import org.apache.paimon.table.source.Split;
-
-import java.util.List;
-
-/** Lookup table of partial cache. */
-public interface PartialCacheLookupTable extends LookupTable {
- void refresh(List<Split> splits);
-}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java
index d8707a4ab..9ef35bea8 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java
@@ -21,10 +21,8 @@ package org.apache.paimon.flink.lookup;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalSerializers;
import org.apache.paimon.lookup.BulkLoader;
-import org.apache.paimon.lookup.RocksDBStateFactory;
import org.apache.paimon.lookup.RocksDBValueState;
import org.apache.paimon.types.RowKind;
-import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.KeyProjectedRow;
import org.apache.paimon.utils.TypeUtils;
@@ -32,44 +30,34 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.function.Predicate;
/** A {@link LookupTable} for primary key table. */
-public class PrimaryKeyLookupTable implements FullCacheLookupTable {
+public class PrimaryKeyLookupTable extends FullCacheLookupTable {
protected final RocksDBValueState<InternalRow, InternalRow> tableState;
- protected final RowType rowType;
-
- protected final Predicate<InternalRow> recordFilter;
-
protected int[] primaryKeyMapping;
- protected final KeyProjectedRow primaryKey;
+ protected final KeyProjectedRow primaryKeyRow;
- public PrimaryKeyLookupTable(
- RocksDBStateFactory stateFactory,
- RowType rowType,
- List<String> primaryKey,
- Predicate<InternalRow> recordFilter,
- long lruCacheSize)
- throws IOException {
- this.rowType = rowType;
- List<String> fieldNames = rowType.getFieldNames();
- this.primaryKeyMapping = primaryKey.stream().mapToInt(fieldNames::indexOf).toArray();
- this.primaryKey = new KeyProjectedRow(primaryKeyMapping);
+ public PrimaryKeyLookupTable(Context context, long lruCacheSize) throws IOException {
+ super(context);
+ List<String> fieldNames = projectedType.getFieldNames();
+ this.primaryKeyMapping =
+ context.table.primaryKeys().stream().mapToInt(fieldNames::indexOf).toArray();
+ this.primaryKeyRow = new KeyProjectedRow(primaryKeyMapping);
this.tableState =
stateFactory.valueState(
"table",
- InternalSerializers.create(TypeUtils.project(rowType, primaryKeyMapping)),
- InternalSerializers.create(rowType),
+ InternalSerializers.create(
+ TypeUtils.project(projectedType, primaryKeyMapping)),
+ InternalSerializers.create(projectedType),
lruCacheSize);
- this.recordFilter = recordFilter;
}
@Override
- public List<InternalRow> get(InternalRow key) throws IOException {
+ public List<InternalRow> innerGet(InternalRow key) throws IOException {
InternalRow value = tableState.get(key);
return value == null ? Collections.emptyList() : Collections.singletonList(value);
}
@@ -79,38 +67,33 @@ public class PrimaryKeyLookupTable implements FullCacheLookupTable {
throws IOException {
while (incremental.hasNext()) {
InternalRow row = incremental.next();
- primaryKey.replaceRow(row);
+ primaryKeyRow.replaceRow(row);
if (orderByLastField) {
- InternalRow previous = tableState.get(primaryKey);
- int orderIndex = rowType.getFieldCount() - 1;
+ InternalRow previous = tableState.get(primaryKeyRow);
+ int orderIndex = projectedType.getFieldCount() - 1;
if (previous != null && previous.getLong(orderIndex) > row.getLong(orderIndex)) {
continue;
}
}
if (row.getRowKind() == RowKind.INSERT || row.getRowKind() == RowKind.UPDATE_AFTER) {
- if (recordFilter.test(row)) {
- tableState.put(primaryKey, row);
+ if (recordFilter().test(row)) {
+ tableState.put(primaryKeyRow, row);
} else {
// The new record under primary key is filtered
// We need to delete this primary key as it no longer exists.
- tableState.delete(primaryKey);
+ tableState.delete(primaryKeyRow);
}
} else {
- tableState.delete(primaryKey);
+ tableState.delete(primaryKeyRow);
}
}
}
- @Override
- public Predicate<InternalRow> recordFilter() {
- return recordFilter;
- }
-
@Override
public byte[] toKeyBytes(InternalRow row) throws IOException {
- primaryKey.replaceRow(row);
- return tableState.serializeKey(primaryKey);
+ primaryKeyRow.replaceRow(row);
+ return tableState.serializeKey(primaryKeyRow);
}
@Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java
index 66aa965b4..57a274e11 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java
@@ -22,14 +22,15 @@ import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.disk.IOManagerImpl;
import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.table.BucketMode;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.query.TableQuery;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.utils.Projection;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
@@ -37,20 +38,37 @@ import java.util.Collections;
import java.util.List;
/** Lookup table for primary key which supports to read the LSM tree directly. */
-public class PrimaryKeyPartialLookupTable implements PartialCacheLookupTable {
+public class PrimaryKeyPartialLookupTable implements LookupTable {
- private static final Logger LOG = LoggerFactory.getLogger(PrimaryKeyPartialLookupTable.class);
-
- private final FixedBucketKeyExtractor extractor;
+ private final FixedBucketFromPkExtractor extractor;
private final TableQuery tableQuery;
- public PrimaryKeyPartialLookupTable(FileStoreTable table, int[] projection, File tempPath) {
+ private final TableFileMonitor fileMonitor;
+
+ public PrimaryKeyPartialLookupTable(
+ FileStoreTable table, @Nullable Predicate predicate, int[] projection, File tempPath) {
+ if (table.partitionKeys().size() > 0) {
+ throw new UnsupportedOperationException(
+ "The partitioned table are not supported in partial cache mode.");
+ }
+
+ if (table.bucketMode() != BucketMode.FIXED) {
+ throw new UnsupportedOperationException(
+ "Unsupported mode for partial lookup: " + table.bucketMode());
+ }
+
this.tableQuery =
table.newTableQuery()
.withValueProjection(Projection.of(projection).toNestedIndexes())
.withIOManager(new IOManagerImpl(tempPath.toString()));
- this.extractor = new FixedBucketKeyExtractor(table.schema());
+ this.extractor = new FixedBucketFromPkExtractor(table.schema());
+ this.fileMonitor = new TableFileMonitor(table, predicate);
+ }
+
+ @Override
+ public void open() throws Exception {
+ refresh();
}
@Override
@@ -68,7 +86,8 @@ public class PrimaryKeyPartialLookupTable implements PartialCacheLookupTable {
}
@Override
- public void refresh(List<Split> splits) {
+ public void refresh() {
+ List<Split> splits = fileMonitor.readChanges();
for (Split split : splits) {
if (!(split instanceof DataSplit)) {
throw new IllegalArgumentException("Unsupported split: " + split.getClass());
@@ -81,4 +100,9 @@ public class PrimaryKeyPartialLookupTable implements PartialCacheLookupTable {
tableQuery.refreshFiles(partition, bucket, before, after);
}
}
+
+ @Override
+ public void close() throws IOException {
+ tableQuery.close();
+ }
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java
index 8de3fa34d..793b99c5b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java
@@ -21,9 +21,7 @@ package org.apache.paimon.flink.lookup;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalSerializers;
import org.apache.paimon.lookup.RocksDBSetState;
-import org.apache.paimon.lookup.RocksDBStateFactory;
import org.apache.paimon.types.RowKind;
-import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.KeyProjectedRow;
import org.apache.paimon.utils.TypeUtils;
@@ -31,7 +29,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.function.Predicate;
/** A {@link LookupTable} for primary key table which provides lookup by secondary key. */
public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
@@ -40,28 +37,22 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
private final KeyProjectedRow secKeyRow;
- public SecondaryIndexLookupTable(
- RocksDBStateFactory stateFactory,
- RowType rowType,
- List<String> primaryKey,
- List<String> secKey,
- Predicate<InternalRow> recordFilter,
- long lruCacheSize)
- throws IOException {
- super(stateFactory, rowType, primaryKey, recordFilter, lruCacheSize / 2);
- List<String> fieldNames = rowType.getFieldNames();
- int[] secKeyMapping = secKey.stream().mapToInt(fieldNames::indexOf).toArray();
+ public SecondaryIndexLookupTable(Context context, long lruCacheSize) throws IOException {
+ super(context, lruCacheSize / 2);
+ List<String> fieldNames = projectedType.getFieldNames();
+ int[] secKeyMapping = context.joinKey.stream().mapToInt(fieldNames::indexOf).toArray();
this.secKeyRow = new KeyProjectedRow(secKeyMapping);
this.indexState =
stateFactory.setState(
"sec-index",
- InternalSerializers.create(TypeUtils.project(rowType, secKeyMapping)),
- InternalSerializers.create(TypeUtils.project(rowType, primaryKeyMapping)),
+ InternalSerializers.create(TypeUtils.project(projectedType, secKeyMapping)),
+ InternalSerializers.create(
+ TypeUtils.project(projectedType, primaryKeyMapping)),
lruCacheSize / 2);
}
@Override
- public List<InternalRow> get(InternalRow key) throws IOException {
+ public List<InternalRow> innerGet(InternalRow key) throws IOException {
List<InternalRow> pks = indexState.get(key);
List<InternalRow> values = new ArrayList<>(pks.size());
for (InternalRow pk : pks) {
@@ -78,14 +69,14 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
throws IOException {
while (incremental.hasNext()) {
InternalRow row = incremental.next();
- primaryKey.replaceRow(row);
+ primaryKeyRow.replaceRow(row);
boolean previousFetched = false;
InternalRow previous = null;
if (orderByLastField) {
- previous = tableState.get(primaryKey);
+ previous = tableState.get(primaryKeyRow);
previousFetched = true;
- int orderIndex = rowType.getFieldCount() - 1;
+ int orderIndex = projectedType.getFieldCount() - 1;
if (previous != null && previous.getLong(orderIndex) > row.getLong(orderIndex)) {
continue;
}
@@ -93,21 +84,21 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
if (row.getRowKind() == RowKind.INSERT || row.getRowKind() == RowKind.UPDATE_AFTER) {
if (!previousFetched) {
- previous = tableState.get(primaryKey);
+ previous = tableState.get(primaryKeyRow);
}
if (previous != null) {
- indexState.retract(secKeyRow.replaceRow(previous), primaryKey);
+ indexState.retract(secKeyRow.replaceRow(previous), primaryKeyRow);
}
- if (recordFilter.test(row)) {
- tableState.put(primaryKey, row);
- indexState.add(secKeyRow.replaceRow(row), primaryKey);
+ if (recordFilter().test(row)) {
+ tableState.put(primaryKeyRow, row);
+ indexState.add(secKeyRow.replaceRow(row), primaryKeyRow);
} else {
- tableState.delete(primaryKey);
+ tableState.delete(primaryKeyRow);
}
} else {
- tableState.delete(primaryKey);
- indexState.retract(secKeyRow.replaceRow(row), primaryKey);
+ tableState.delete(primaryKeyRow);
+ indexState.retract(secKeyRow.replaceRow(row), primaryKeyRow);
}
}
}
@@ -115,6 +106,6 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
@Override
public void bulkLoadWritePlus(byte[] key, byte[] value) throws IOException {
InternalRow row = tableState.deserializeValue(value);
- indexState.add(secKeyRow.replaceRow(row), primaryKey.replaceRow(row));
+ indexState.add(secKeyRow.replaceRow(row), primaryKeyRow.replaceRow(row));
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java
index 81e13a857..208691477 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java
@@ -29,7 +29,6 @@ import org.apache.paimon.flink.log.LogSourceProvider;
import org.apache.paimon.flink.log.LogStoreTableFactory;
import org.apache.paimon.flink.lookup.FileStoreLookupFunction;
import org.apache.paimon.flink.lookup.LookupRuntimeProviderFactory;
-import org.apache.paimon.flink.lookup.LookupTable;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.table.AppendOnlyFileStoreTable;
@@ -55,10 +54,8 @@ import org.apache.flink.table.plan.stats.TableStats;
import javax.annotation.Nullable;
import java.time.Duration;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.apache.paimon.CoreOptions.CHANGELOG_PRODUCER;
@@ -305,11 +302,6 @@ public class DataTableSource extends FlinkTableSource {
Options options = new Options(table.options());
boolean enableAsync = options.get(LOOKUP_ASYNC);
int asyncThreadNumber = options.get(LOOKUP_ASYNC_THREAD_NUMBER);
- LookupTable.validatePartialCacheMode(
- table,
- Arrays.stream(joinKey)
- .mapToObj(i -> table.rowType().getFieldNames().get(projection[i]))
- .collect(Collectors.toList()));
return LookupRuntimeProviderFactory.create(
new FileStoreLookupFunction(table, projection, joinKey, predicate),
enableAsync,
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
index 711f58633..76b4fd1c5 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
@@ -53,15 +53,17 @@ public class LookupJoinITCase extends CatalogITCaseBase {
"CREATE TABLE PARTITIONED_DIM (i INT, j INT, k1 INT, k2 INT, PRIMARY KEY (i, j) NOT ENFORCED)"
+ "PARTITIONED BY (`i`) WITH ('continuous.discovery-interval'='1 ms' %s)";
- String lruOption =
- ", 'changelog-producer'='lookup', 'lookup.cache' = 'partial', 'bucket' = '2'";
+ String fullOption =
+ ", 'changelog-producer'='lookup', 'lookup.cache' = 'full', 'bucket' = '2'";
+
+ String lruOption = ", 'changelog-producer'='lookup', 'bucket' = '2'";
switch (cacheMode) {
case FULL:
- tEnv.executeSql(String.format(dim, ""));
- tEnv.executeSql(String.format(partitioned, ""));
+ tEnv.executeSql(String.format(dim, fullOption));
+ tEnv.executeSql(String.format(partitioned, fullOption));
break;
- case PARTIAL:
+ case AUTO:
tEnv.executeSql(String.format(dim, lruOption));
tEnv.executeSql(String.format(partitioned, lruOption));
break;
@@ -313,23 +315,14 @@ public class LookupJoinITCase extends CatalogITCaseBase {
iterator.close();
}
- @ParameterizedTest
- @EnumSource(LookupCacheMode.class)
- public void testNonPkLookup(LookupCacheMode cacheMode) throws Exception {
- initTable(cacheMode);
+ @Test
+ public void testNonPkLookup() throws Exception {
+ initTable(LookupCacheMode.AUTO);
sql("INSERT INTO DIM VALUES (1, 11, 111, 1111), (2, 22, 222, 2222), (3, 22, 333, 3333)");
String query =
"SELECT D.i, T.i, D.k1, D.k2 FROM T LEFT JOIN DIM for system_time as of T.proctime AS D ON T.i = D.j";
- BlockingIterator<Row, Row> iterator;
- if (cacheMode == LookupCacheMode.PARTIAL) {
- assertThatThrownBy(() -> sEnv.executeSql(query))
- .hasRootCauseMessage(
- "For partial cache mode, the join key: [j] must equal to the primary key: [i]");
- return;
- } else {
- iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
- }
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
sql("INSERT INTO T VALUES (11), (22), (33)");
List<Row> result = iterator.collect(4);
@@ -602,21 +595,12 @@ public class LookupJoinITCase extends CatalogITCaseBase {
iterator.close();
}
- @ParameterizedTest
- @EnumSource(LookupCacheMode.class)
- public void testLookupPartitionedTable(LookupCacheMode cacheMode) throws Exception {
- initTable(cacheMode);
+ @Test
+ public void testLookupPartitionedTable() throws Exception {
+ initTable(LookupCacheMode.AUTO);
String query =
"SELECT T.i, D.j, D.k1, D.k2 FROM T LEFT JOIN PARTITIONED_DIM for system_time as of T.proctime AS D ON T.i = D.i";
- BlockingIterator<Row, Row> iterator;
- if (cacheMode == LookupCacheMode.PARTIAL) {
- assertThatThrownBy(() -> sEnv.executeSql(query))
- .hasRootCauseMessage(
- "For partial cache mode, the join key: [i] must equal to the primary key: [i, j]");
- return;
- } else {
- iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
- }
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
sql("INSERT INTO T VALUES (1), (2), (3)");
@@ -639,26 +623,16 @@ public class LookupJoinITCase extends CatalogITCaseBase {
iterator.close();
}
- @ParameterizedTest
- @EnumSource(LookupCacheMode.class)
- public void testLookupNonPkAppendTable(LookupCacheMode cacheMode) throws Exception {
+ @Test
+ public void testLookupNonPkAppendTable() throws Exception {
sql(
"CREATE TABLE DIM_NO_PK (i INT, j INT, k1 INT, k2 INT) "
- + "PARTITIONED BY (`i`) WITH ('continuous.discovery-interval'='1 ms' %s)",
- cacheMode == LookupCacheMode.PARTIAL ? ", 'lookup.cache' = 'partial'" : "");
+ + "PARTITIONED BY (`i`) WITH ('continuous.discovery-interval'='1 ms')");
String query =
"SELECT T.i, D.j, D.k1, D.k2 FROM T LEFT JOIN DIM_NO_PK for system_time as of T.proctime AS D ON T.i "
+ "= D.i";
- BlockingIterator<Row, Row> iterator;
- if (cacheMode == LookupCacheMode.PARTIAL) {
- assertThatThrownBy(() -> sEnv.executeSql(query))
- .hasRootCauseMessage(
- "For partial cache mode, the join key: [i] must equal to the primary key: []");
- return;
- } else {
- iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
- }
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
sql("INSERT INTO T VALUES (1), (2), (3)");
@@ -771,27 +745,4 @@ public class LookupJoinITCase extends CatalogITCaseBase {
iterator.close();
}
-
- @Test
- public void testPartialCacheValidation() {
- sql(
- "CREATE TABLE DIM2 (i INT PRIMARY KEY NOT ENFORCED, j INT, k1 INT, k2 INT) WITH"
- + " ('continuous.discovery-interval'='1 ms', 'changelog-producer' = 'lookup', "
- + "'lookup.cache' = 'partial', 'bucket' = '-1')");
-
- String query =
- "SELECT T.i, D.k1 FROM T LEFT JOIN DIM2 for system_time as of T.proctime AS D ON T.i = D.i";
- assertThatThrownBy(() -> sEnv.executeSql(query))
- .hasRootCauseMessage("Only support fixed bucket mode in partial cache mode.");
- sql(
- "CREATE TABLE DIM3 (i INT PRIMARY KEY NOT ENFORCED, j INT, k1 INT, k2 INT) WITH"
- + " ('continuous.discovery-interval'='1 ms', "
- + "'lookup.cache' = 'partial', 'bucket' = '1', 'merge-engine' = 'aggregation')");
-
- String query2 =
- "SELECT T.i, D.k1 FROM T LEFT JOIN DIM3 for system_time as of T.proctime AS D ON T.i = D.i";
-
- assertThatThrownBy(() -> sEnv.executeSql(query2))
- .hasRootCauseMessage("Only support deduplicate merge engine, but is: aggregation");
- }
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java
index 5741cbb7c..e67e5eb76 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java
@@ -19,6 +19,7 @@
package org.apache.paimon.flink.lookup;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.JoinedRow;
@@ -27,8 +28,6 @@ import org.apache.paimon.disk.IOManagerImpl;
import org.apache.paimon.flink.lookup.FullCacheLookupTable.TableBulkLoader;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.lookup.BulkLoader;
-import org.apache.paimon.lookup.RocksDBStateFactory;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
@@ -61,8 +60,8 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
-import static org.apache.paimon.schema.SystemColumns.SEQUENCE_NUMBER;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -71,36 +70,50 @@ public class LookupTableTest extends TableTestBase {
@TempDir Path tempDir;
- private RocksDBStateFactory stateFactory;
-
private RowType rowType;
private IOManager ioManager;
+ private FullCacheLookupTable table;
+
@BeforeEach
public void before() throws IOException {
- this.stateFactory = new RocksDBStateFactory(tempDir.toString(), new Options(), null);
this.rowType = RowType.of(new IntType(), new IntType(), new IntType());
this.ioManager = new IOManagerImpl(tempDir.toString());
}
@AfterEach
public void after() throws IOException {
- if (stateFactory != null) {
- stateFactory.close();
+ if (table != null) {
+ table.close();
}
}
+ private FileStoreTable createTable(List<String> primaryKeys, Options options) throws Exception {
+ Identifier identifier = new Identifier("default", "t");
+ Schema schema =
+ new Schema(
+ rowType.getFields(),
+ Collections.emptyList(),
+ primaryKeys,
+ options.toMap(),
+ null);
+ catalog.createTable(identifier, schema, false);
+ return (FileStoreTable) catalog.getTable(identifier);
+ }
+
@Test
- public void testPkTable() throws IOException, BulkLoader.WriteException {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType,
- singletonList("f0"),
- singletonList("f0"),
+ public void testPkTable() throws Exception {
+ FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> true,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f0"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
// test bulk load error
{
@@ -147,14 +160,18 @@ public class LookupTableTest extends TableTestBase {
@Test
public void testPkTableWithSequenceField() throws Exception {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType.appendDataField(SEQUENCE_NUMBER, DataTypes.BIGINT()),
- singletonList("f0"),
- singletonList("f0"),
+ Options options = new Options();
+ options.set(CoreOptions.SEQUENCE_FIELD, "f1");
+ FileStoreTable storeTable = createTable(singletonList("f0"), options);
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> true,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f0"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
List<Pair<byte[], byte[]>> records = new ArrayList<>();
for (int i = 1; i <= 10; i++) {
@@ -172,31 +189,33 @@ public class LookupTableTest extends TableTestBase {
table.refresh(singletonList(sequence(row(1, 22, 222), 1L)).iterator(), true);
List<InternalRow> result = table.get(row(1));
assertThat(result).hasSize(1);
- assertRow(result.get(0), 1, 22, 222, 1);
+ assertRow(result.get(0), 1, 22, 222);
// refresh with old sequence
table.refresh(singletonList((sequence(row(1, 33, 333), 0L))).iterator(), true);
result = table.get(row(1));
assertThat(result).hasSize(1);
- assertRow(result.get(0), 1, 22, 222, 1);
+ assertRow(result.get(0), 1, 22, 222);
// test refresh delete data with old sequence
table.refresh(
singletonList(sequence(row(RowKind.DELETE, 1, 11, 111), -1L)).iterator(), true);
assertThat(table.get(row(1))).hasSize(1);
- assertRow(result.get(0), 1, 22, 222, 1);
+ assertRow(result.get(0), 1, 22, 222);
}
@Test
- public void testPkTablePkFilter() throws IOException {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType,
- singletonList("f0"),
- singletonList("f0"),
+ public void testPkTablePkFilter() throws Exception {
+ FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> r.getInt(0) < 3,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f0"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
table.refresh(singletonList(sequence(row(1, 11, 111), -1L)).iterator(), false);
List<InternalRow> result = table.get(row(1));
@@ -217,15 +236,17 @@ public class LookupTableTest extends TableTestBase {
}
@Test
- public void testPkTableNonPkFilter() throws IOException {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType,
- singletonList("f0"),
- singletonList("f0"),
+ public void testPkTableNonPkFilter() throws Exception {
+ FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> r.getInt(1) < 22,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f0"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
table.refresh(singletonList(sequence(row(1, 11, 111), -1L)).iterator(), false);
List<InternalRow> result = table.get(row(1));
@@ -238,15 +259,17 @@ public class LookupTableTest extends TableTestBase {
}
@Test
- public void testSecKeyTable() throws IOException, BulkLoader.WriteException {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType,
- singletonList("f0"),
- singletonList("f1"),
+ public void testSecKeyTable() throws Exception {
+ FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> true,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f1"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
// test bulk load 100_000 records
List<Pair<byte[], byte[]>> records = new ArrayList<>();
@@ -279,14 +302,18 @@ public class LookupTableTest extends TableTestBase {
@Test
public void testSecKeyTableWithSequenceField() throws Exception {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType.appendDataField(SEQUENCE_NUMBER, DataTypes.BIGINT()),
- singletonList("f0"),
- singletonList("f1"),
+ Options options = new Options();
+ options.set(CoreOptions.SEQUENCE_FIELD, "f1");
+ FileStoreTable storeTable = createTable(singletonList("f0"), options);
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> true,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f1"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
List<Pair<byte[], byte[]>> records = new ArrayList<>();
Random rnd = new Random();
@@ -318,7 +345,7 @@ public class LookupTableTest extends TableTestBase {
true);
List<InternalRow> result = table.get(row(22));
assertThat(result.stream().map(row -> row.getInt(0))).contains(1);
- assertThat(result.stream().map(InternalRow::getFieldCount)).allMatch(n -> n == 4);
+ assertThat(result.stream().map(InternalRow::getFieldCount)).allMatch(n -> n == 3);
// refresh with old value
table.refresh(
@@ -330,15 +357,17 @@ public class LookupTableTest extends TableTestBase {
}
@Test
- public void testSecKeyTablePkFilter() throws IOException {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType,
- singletonList("f0"),
- singletonList("f1"),
+ public void testSecKeyTablePkFilter() throws Exception {
+ FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> r.getInt(0) < 3,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f1"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
table.refresh(singletonList(sequence(row(1, 11, 111), -1L)).iterator(), false);
List<InternalRow> result = table.get(row(11));
@@ -368,15 +397,17 @@ public class LookupTableTest extends TableTestBase {
}
@Test
- public void testNoPrimaryKeyTable() throws IOException, BulkLoader.WriteException {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType,
- Collections.emptyList(),
- singletonList("f1"),
+ public void testNoPrimaryKeyTable() throws Exception {
+ FileStoreTable storeTable = createTable(emptyList(), new Options());
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> true,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f1"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
// test bulk load 100_000 records
List<Pair<byte[], byte[]>> records = new ArrayList<>();
@@ -408,15 +439,17 @@ public class LookupTableTest extends TableTestBase {
}
@Test
- public void testNoPrimaryKeyTableFilter() throws IOException {
- FullCacheLookupTable table =
- LookupTable.create(
- stateFactory,
- rowType,
- Collections.emptyList(),
- singletonList("f1"),
+ public void testNoPrimaryKeyTableFilter() throws Exception {
+ FileStoreTable storeTable = createTable(emptyList(), new Options());
+ FullCacheLookupTable.Context context =
+ new FullCacheLookupTable.Context(
+ storeTable,
+ new int[] {0, 1, 2},
+ null,
+ tempDir.toFile(),
r -> r.getInt(2) < 222,
- ThreadLocalRandom.current().nextInt(2) * 10);
+ singletonList("f1"));
+ table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
table.refresh(singletonList(row(1, 11, 333)).iterator(), false);
List<InternalRow> result = table.get(row(11));
@@ -437,13 +470,9 @@ public class LookupTableTest extends TableTestBase {
@Test
public void testPartialLookupTable() throws Exception {
FileStoreTable dimTable = createDimTable();
- PartialCacheLookupTable table =
- LookupTable.create(
- dimTable,
- new int[] {0, 1, 2},
- Collections.singletonList("pk"),
- tempDir.toFile());
- TableFileMonitor monitor = new TableFileMonitor(dimTable, null);
+ PrimaryKeyPartialLookupTable table =
+ new PrimaryKeyPartialLookupTable(
+ dimTable, null, new int[] {0, 1, 2}, tempDir.toFile());
List<InternalRow> result = table.get(row(1, -1));
assertThat(result).hasSize(0);
@@ -451,7 +480,7 @@ public class LookupTableTest extends TableTestBase {
result = table.get(row(1, -1));
assertThat(result).hasSize(0);
- table.refresh(monitor.readChanges());
+ table.refresh();
result = table.get(row(1, -1));
assertThat(result).hasSize(1);
assertRow(result.get(0), 1, -1, 11);
@@ -460,7 +489,7 @@ public class LookupTableTest extends TableTestBase {
assertRow(result.get(0), 2, -2, 22);
write(dimTable, ioManager, GenericRow.ofKind(RowKind.DELETE, 1, -1, 11));
- table.refresh(monitor.readChanges());
+ table.refresh();
result = table.get(row(1, -1));
assertThat(result).hasSize(0);
}
@@ -468,13 +497,9 @@ public class LookupTableTest extends TableTestBase {
@Test
public void testPartialLookupTableWithProjection() throws Exception {
FileStoreTable dimTable = createDimTable();
- PartialCacheLookupTable table =
- LookupTable.create(
- dimTable,
- new int[] {2, 1},
- Collections.singletonList("pk"),
- tempDir.toFile());
- TableFileMonitor monitor = new TableFileMonitor(dimTable, null);
+ PrimaryKeyPartialLookupTable table =
+ new PrimaryKeyPartialLookupTable(
+ dimTable, null, new int[] {2, 1}, tempDir.toFile());
List<InternalRow> result = table.get(row(1, -1));
assertThat(result).hasSize(0);
@@ -482,7 +507,7 @@ public class LookupTableTest extends TableTestBase {
result = table.get(row(1, -1));
assertThat(result).hasSize(0);
- table.refresh(monitor.readChanges());
+ table.refresh();
result = table.get(row(1, -1));
assertThat(result).hasSize(1);
assertRow(result.get(0), 11, -1);