You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/01/04 14:03:39 UTC
(incubator-paimon) branch master updated: [service] Introduce ServiceManager and KvQueryClient (#2631)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a7a69147a [service] Introduce ServiceManager and KvQueryClient (#2631)
a7a69147a is described below
commit a7a69147abf1bbe9b7155525d3f93fb8efe9ee32
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Thu Jan 4 22:03:35 2024 +0800
[service] Introduce ServiceManager and KvQueryClient (#2631)
---
.../java/org/apache/paimon/AbstractFileStore.java | 6 +
.../src/main/java/org/apache/paimon/FileStore.java | 3 +
.../QueryLookup.java => query/QueryLocation.java} | 15 +-
.../org/apache/paimon/query/QueryLocationImpl.java | 54 +++++
.../paimon/{lookup => query}/QueryServer.java | 2 +-
.../QueryLookup.java => query/TableQuery.java} | 13 +-
.../org/apache/paimon/service/ServiceManager.java | 76 +++++++
.../apache/paimon/table}/sink/ChannelComputer.java | 2 +-
.../paimon/catalog/PrimaryKeyTableTestBase.java | 2 +-
.../apache/paimon/service/ServiceManagerTest.java | 59 ++++++
.../flink/sink/cdc/CdcAssignerChannelComputer.java | 2 +-
.../flink/sink/cdc/CdcDynamicBucketSink.java | 2 +-
.../cdc/CdcMultiplexRecordChannelComputer.java | 2 +-
.../flink/sink/cdc/CdcRecordChannelComputer.java | 2 +-
.../sink/cdc/CdcWithBucketChannelComputer.java | 2 +-
.../cdc/MultiTableCommittableChannelComputer.java | 2 +-
.../flink/sink/BucketsRowChannelComputer.java | 1 +
.../paimon/flink/sink/DynamicBucketSink.java | 1 +
.../paimon/flink/sink/FlinkStreamPartitioner.java | 2 +
.../sink/MultiTablesStoreCompactOperator.java | 1 +
.../flink/sink/RowAssignerChannelComputer.java | 1 +
.../paimon/flink/sink/RowDataChannelComputer.java | 1 +
.../paimon/flink/sink/RowDynamicBucketSink.java | 1 +
.../flink/sink/RowWithBucketChannelComputer.java | 1 +
.../paimon/flink/sink/StoreCompactOperator.java | 1 +
.../paimon/flink/sink/TableWriteOperator.java | 1 +
.../sink/index/KeyPartRowChannelComputer.java | 2 +-
.../paimon/service/client/KvQueryClient.java | 119 +++++++++++
.../UnknownPartitionBucketException.java | 14 +-
.../paimon/service/network/NetworkClient.java | 2 +-
.../paimon/service/network/NetworkServer.java | 4 +-
paimon-service/paimon-service-runtime/pom.xml | 98 +++++++++
.../paimon/service/server/KvQueryServer.java | 29 ++-
.../paimon/service/server/KvServerHandler.java | 22 +-
.../apache/paimon/service/KvQueryTableTest.java | 227 +++++++++++++++++++++
35 files changed, 726 insertions(+), 46 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index 3615b488e..0bd878c96 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -34,6 +34,7 @@ import org.apache.paimon.operation.SnapshotDeletion;
import org.apache.paimon.operation.TagDeletion;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.service.ServiceManager;
import org.apache.paimon.table.CatalogEnvironment;
import org.apache.paimon.table.sink.CallbackUtils;
import org.apache.paimon.table.sink.TagCallback;
@@ -259,4 +260,9 @@ public abstract class AbstractFileStore<T> implements FileStore<T> {
}
return callbacks;
}
+
+ @Override
+ public ServiceManager newServiceManager() {
+ return new ServiceManager(fileIO, options.path());
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/FileStore.java b/paimon-core/src/main/java/org/apache/paimon/FileStore.java
index f044102dc..1b7f9f1c9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/FileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/FileStore.java
@@ -30,6 +30,7 @@ import org.apache.paimon.operation.FileStoreWrite;
import org.apache.paimon.operation.PartitionExpire;
import org.apache.paimon.operation.SnapshotDeletion;
import org.apache.paimon.operation.TagDeletion;
+import org.apache.paimon.service.ServiceManager;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.table.sink.TagCallback;
import org.apache.paimon.tag.TagAutoCreation;
@@ -90,6 +91,8 @@ public interface FileStore<T> extends Serializable {
@Nullable
TagAutoCreation newTagCreationManager();
+ ServiceManager newServiceManager();
+
boolean mergeSchema(RowType rowType, boolean allowExplicitCast);
List<TagCallback> createTagCallbacks();
diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java b/paimon-core/src/main/java/org/apache/paimon/query/QueryLocation.java
similarity index 70%
copy from paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java
copy to paimon-core/src/main/java/org/apache/paimon/query/QueryLocation.java
index 163e8dfda..ca783005a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java
+++ b/paimon-core/src/main/java/org/apache/paimon/query/QueryLocation.java
@@ -16,12 +16,19 @@
* limitations under the License.
*/
-package org.apache.paimon.lookup;
+package org.apache.paimon.query;
import org.apache.paimon.data.BinaryRow;
-/** An interface to provide lookup function. */
-public interface QueryLookup {
+import java.net.InetSocketAddress;
- BinaryRow[] lookup(BinaryRow partition, int bucket, BinaryRow[] keys);
+/** An interface to get query location. */
+public interface QueryLocation {
+
+ /**
+ * Get location from partition and bucket.
+ *
+ * @param forceUpdate whether to refresh location cache.
+ */
+ InetSocketAddress getLocation(BinaryRow partition, int bucket, boolean forceUpdate);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/query/QueryLocationImpl.java b/paimon-core/src/main/java/org/apache/paimon/query/QueryLocationImpl.java
new file mode 100644
index 000000000..d815caf3d
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/query/QueryLocationImpl.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.query;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.service.ServiceManager;
+
+import java.net.InetSocketAddress;
+import java.util.Optional;
+
+import static org.apache.paimon.service.ServiceManager.PRIMARY_KEY_LOOKUP;
+import static org.apache.paimon.table.sink.ChannelComputer.select;
+
+/** An implementation of {@link QueryLocation} to get location from {@link ServiceManager}. */
+public class QueryLocationImpl implements QueryLocation {
+
+ private final ServiceManager manager;
+
+ private InetSocketAddress[] addressesCache;
+
+ public QueryLocationImpl(ServiceManager manager) {
+ this.manager = manager;
+ }
+
+ @Override
+ public InetSocketAddress getLocation(BinaryRow partition, int bucket, boolean forceUpdate) {
+ if (addressesCache == null || forceUpdate) {
+ Optional<InetSocketAddress[]> addresses = manager.service(PRIMARY_KEY_LOOKUP);
+ if (!addresses.isPresent()) {
+ throw new RuntimeException(
+ "Cannot find address for table path: " + manager.tablePath());
+ }
+ addressesCache = addresses.get();
+ }
+
+ return addressesCache[select(partition, bucket, addressesCache.length)];
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryServer.java b/paimon-core/src/main/java/org/apache/paimon/query/QueryServer.java
similarity index 97%
rename from paimon-core/src/main/java/org/apache/paimon/lookup/QueryServer.java
rename to paimon-core/src/main/java/org/apache/paimon/query/QueryServer.java
index bc5386009..2e1df8bc3 100644
--- a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryServer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/query/QueryServer.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.paimon.lookup;
+package org.apache.paimon.query;
import java.net.InetSocketAddress;
diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java b/paimon-core/src/main/java/org/apache/paimon/query/TableQuery.java
similarity index 75%
copy from paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java
copy to paimon-core/src/main/java/org/apache/paimon/query/TableQuery.java
index 163e8dfda..0f23e22fe 100644
--- a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java
+++ b/paimon-core/src/main/java/org/apache/paimon/query/TableQuery.java
@@ -16,12 +16,17 @@
* limitations under the License.
*/
-package org.apache.paimon.lookup;
+package org.apache.paimon.query;
import org.apache.paimon.data.BinaryRow;
-/** An interface to provide lookup function. */
-public interface QueryLookup {
+import java.io.IOException;
- BinaryRow[] lookup(BinaryRow partition, int bucket, BinaryRow[] keys);
+/**
+ * Query for a table, provides lookup method to lookup value by key. It does not download files on
+ * its own, maintaining its role requires refreshing its file list.
+ */
+public interface TableQuery {
+
+ BinaryRow[] lookup(BinaryRow partition, int bucket, BinaryRow[] keys) throws IOException;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/service/ServiceManager.java b/paimon-core/src/main/java/org/apache/paimon/service/ServiceManager.java
new file mode 100644
index 000000000..4d29e671c
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/service/ServiceManager.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.service;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.utils.JsonSerdeUtil;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.UncheckedIOException;
+import java.net.InetSocketAddress;
+import java.util.Optional;
+
+/** A manager to manage services, for example, the service to lookup row from the primary key. */
+public class ServiceManager implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SERVICE_PREFIX = "service-";
+
+ public static final String PRIMARY_KEY_LOOKUP = "primary-key-lookup";
+
+ private final FileIO fileIO;
+ private final Path tablePath;
+
+ public ServiceManager(FileIO fileIO, Path tablePath) {
+ this.fileIO = fileIO;
+ this.tablePath = tablePath;
+ }
+
+ public Path tablePath() {
+ return tablePath;
+ }
+
+ public Optional<InetSocketAddress[]> service(String id) {
+ try {
+ return fileIO.readOverwrittenFileUtf8(servicePath(id))
+ .map(s -> JsonSerdeUtil.fromJson(s, InetSocketAddress[].class));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public void resetService(String id, InetSocketAddress[] addresses) {
+ try {
+ fileIO.overwriteFileUtf8(servicePath(id), JsonSerdeUtil.toJson(addresses));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public void deleteService(String id) {
+ fileIO.deleteQuietly(servicePath(id));
+ }
+
+ private Path servicePath(String id) {
+ return new Path(tablePath + "/service/" + SERVICE_PREFIX + id);
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/ChannelComputer.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/ChannelComputer.java
similarity index 98%
rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/ChannelComputer.java
rename to paimon-core/src/main/java/org/apache/paimon/table/sink/ChannelComputer.java
index 4a10d4386..52f1dda36 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/ChannelComputer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/ChannelComputer.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.paimon.flink.sink;
+package org.apache.paimon.table.sink;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.utils.SerializableFunction;
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
index 529f84b6e..cc765786f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
@@ -40,7 +40,7 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Base class to test catalog primary key table. */
public abstract class PrimaryKeyTableTestBase {
- @TempDir java.nio.file.Path tempPath;
+ @TempDir protected java.nio.file.Path tempPath;
protected AbstractFileStoreTable table;
protected String commitUser;
diff --git a/paimon-core/src/test/java/org/apache/paimon/service/ServiceManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/service/ServiceManagerTest.java
new file mode 100644
index 000000000..b750940f0
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/service/ServiceManagerTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.service;
+
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Optional;
+
+import static org.apache.paimon.service.ServiceManager.PRIMARY_KEY_LOOKUP;
+import static org.apache.paimon.service.ServiceManager.SERVICE_PREFIX;
+import static org.assertj.core.api.Assertions.assertThat;
+
+class ServiceManagerTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ public void test() throws IOException {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path path = new Path(tempDir.toUri());
+ ServiceManager manager = new ServiceManager(fileIO, path);
+ InetSocketAddress[] addresses =
+ new InetSocketAddress[] {
+ new InetSocketAddress("127.0.0.1", 7000),
+ new InetSocketAddress("127.0.0.1", 7500)
+ };
+ manager.resetService(PRIMARY_KEY_LOOKUP, addresses);
+
+ String fileString =
+ fileIO.readFileUtf8(
+ new Path(new Path(path, "service"), SERVICE_PREFIX + PRIMARY_KEY_LOOKUP));
+ assertThat(fileString).isEqualTo("[ \"127.0.0.1:7000\", \"127.0.0.1:7500\" ]");
+
+ Optional<InetSocketAddress[]> result = manager.service(PRIMARY_KEY_LOOKUP);
+ assertThat(result).hasValue(addresses);
+ }
+}
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcAssignerChannelComputer.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcAssignerChannelComputer.java
index 93cc31b47..c93bd52be 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcAssignerChannelComputer.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcAssignerChannelComputer.java
@@ -18,8 +18,8 @@
package org.apache.paimon.flink.sink.cdc;
-import org.apache.paimon.flink.sink.ChannelComputer;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.utils.MathUtils;
import static org.apache.paimon.index.BucketAssigner.computeAssigner;
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java
index bf4acf979..ec72405b0 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java
@@ -19,12 +19,12 @@
package org.apache.paimon.flink.sink.cdc;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.flink.sink.ChannelComputer;
import org.apache.paimon.flink.sink.Committable;
import org.apache.paimon.flink.sink.DynamicBucketSink;
import org.apache.paimon.flink.sink.StoreSinkWrite;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.PartitionKeyExtractor;
import org.apache.paimon.utils.SerializableFunction;
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputer.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputer.java
index e626dc1ed..86a6c398f 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputer.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputer.java
@@ -20,8 +20,8 @@ package org.apache.paimon.flink.sink.cdc;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.Identifier;
-import org.apache.paimon.flink.sink.ChannelComputer;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputer.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputer.java
index 43eb14a6a..f90cb4ad8 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputer.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputer.java
@@ -19,8 +19,8 @@
package org.apache.paimon.flink.sink.cdc;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.flink.sink.ChannelComputer;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.KeyAndBucketExtractor;
/** {@link ChannelComputer} for {@link CdcRecord}. */
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcWithBucketChannelComputer.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcWithBucketChannelComputer.java
index a24e6e3a0..198a3b0e0 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcWithBucketChannelComputer.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcWithBucketChannelComputer.java
@@ -18,8 +18,8 @@
package org.apache.paimon.flink.sink.cdc;
-import org.apache.paimon.flink.sink.ChannelComputer;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.flink.api.java.tuple.Tuple2;
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableCommittableChannelComputer.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableCommittableChannelComputer.java
index 72bca035a..c12259204 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableCommittableChannelComputer.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableCommittableChannelComputer.java
@@ -18,8 +18,8 @@
package org.apache.paimon.flink.sink.cdc;
-import org.apache.paimon.flink.sink.ChannelComputer;
import org.apache.paimon.flink.sink.MultiTableCommittable;
+import org.apache.paimon.table.sink.ChannelComputer;
import java.util.Objects;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java
index 665e0dc84..d93d38e00 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java
@@ -19,6 +19,7 @@
package org.apache.paimon.flink.sink;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.system.BucketsTable;
import org.apache.flink.table.data.RowData;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
index 5ef2ebe7b..a4880877d 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
@@ -20,6 +20,7 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.PartitionKeyExtractor;
import org.apache.paimon.utils.SerializableFunction;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java
index 5c79842d4..78e532053 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java
@@ -18,6 +18,8 @@
package org.apache.paimon.flink.sink;
+import org.apache.paimon.table.sink.ChannelComputer;
+
import org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.datastream.DataStream;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java
index 6adbbd128..46a7931b9 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java
@@ -26,6 +26,7 @@ import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFileMetaSerializer;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.utils.Preconditions;
import org.apache.flink.runtime.state.StateInitializationContext;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowAssignerChannelComputer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowAssignerChannelComputer.java
index da004af24..aaa759eeb 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowAssignerChannelComputer.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowAssignerChannelComputer.java
@@ -20,6 +20,7 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.RowPartitionKeyExtractor;
import org.apache.paimon.utils.MathUtils;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataChannelComputer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataChannelComputer.java
index bf2687fdb..77020f1ad 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataChannelComputer.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataChannelComputer.java
@@ -21,6 +21,7 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.FixedBucketRowKeyExtractor;
import org.apache.paimon.table.sink.KeyAndBucketExtractor;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java
index d4917fb90..bf6c70f0a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java
@@ -21,6 +21,7 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.PartitionKeyExtractor;
import org.apache.paimon.table.sink.RowPartitionKeyExtractor;
import org.apache.paimon.utils.SerializableFunction;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowWithBucketChannelComputer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowWithBucketChannelComputer.java
index a00c85c31..4b9ff3516 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowWithBucketChannelComputer.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowWithBucketChannelComputer.java
@@ -20,6 +20,7 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.RowPartitionKeyExtractor;
import org.apache.flink.api.java.tuple.Tuple2;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java
index 86f72fe36..184288490 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java
@@ -25,6 +25,7 @@ import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFileMetaSerializer;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.Preconditions;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java
index 004fad293..f38e0ad6b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java
@@ -22,6 +22,7 @@ import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.flink.sink.StoreSinkWriteState.StateValueFilter;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.state.StateInitializationContext;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/KeyPartRowChannelComputer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/KeyPartRowChannelComputer.java
index c581c02ea..dedb07c95 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/KeyPartRowChannelComputer.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/KeyPartRowChannelComputer.java
@@ -22,7 +22,7 @@ import org.apache.paimon.codegen.CodeGenUtils;
import org.apache.paimon.codegen.Projection;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.flink.sink.ChannelComputer;
+import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.types.RowType;
import org.apache.flink.api.java.tuple.Tuple2;
diff --git a/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/client/KvQueryClient.java b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/client/KvQueryClient.java
new file mode 100644
index 000000000..60c1ac0f8
--- /dev/null
+++ b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/client/KvQueryClient.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.service.client;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.query.QueryLocation;
+import org.apache.paimon.service.exceptions.UnknownPartitionBucketException;
+import org.apache.paimon.service.messages.KvRequest;
+import org.apache.paimon.service.messages.KvResponse;
+import org.apache.paimon.service.network.NetworkClient;
+import org.apache.paimon.service.network.messages.MessageSerializer;
+import org.apache.paimon.service.network.stats.DisabledServiceRequestStats;
+import org.apache.paimon.utils.FutureUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+/** A class for the Client to get values from Servers. */
+public class KvQueryClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KvQueryClient.class);
+
+ private final NetworkClient<KvRequest, KvResponse> networkClient;
+ private final QueryLocation queryLocation;
+
+ public KvQueryClient(QueryLocation queryLocation, int numEventLoopThreads) {
+ this.queryLocation = queryLocation;
+ final MessageSerializer<KvRequest, KvResponse> messageSerializer =
+ new MessageSerializer<>(
+ new KvRequest.KvRequestDeserializer(),
+ new KvResponse.KvResponseDeserializer());
+
+ this.networkClient =
+ new NetworkClient<>(
+ "Kv Query Client",
+ numEventLoopThreads,
+ messageSerializer,
+ new DisabledServiceRequestStats());
+ }
+
+ public CompletableFuture<BinaryRow[]> getValues(
+ BinaryRow partition, int bucket, BinaryRow[] keys) {
+ CompletableFuture<BinaryRow[]> response = new CompletableFuture<>();
+ executeActionAsync(response, new KvRequest(partition, bucket, keys), false);
+ return response;
+ }
+
+ private void executeActionAsync(
+ final CompletableFuture<BinaryRow[]> result,
+ final KvRequest request,
+ final boolean update) {
+ if (!result.isDone()) {
+ final CompletableFuture<KvResponse> operationFuture = getResponse(request, update);
+ operationFuture.whenCompleteAsync(
+ (t, throwable) -> {
+ if (throwable != null) {
+ if (throwable instanceof UnknownPartitionBucketException
+ || throwable.getCause() instanceof ConnectException) {
+
+ // These failures are likely to be caused by out-of-sync location.
+ // Therefore, we retry this query and force lookup the location.
+
+ LOG.debug(
+ "Retrying after failing to retrieve state due to: {}.",
+ throwable.getMessage());
+ executeActionAsync(result, request, true);
+ } else {
+ result.completeExceptionally(throwable);
+ }
+ } else {
+ result.complete(t.values());
+ }
+ });
+
+ result.whenComplete((t, throwable) -> operationFuture.cancel(false));
+ }
+ }
+
+ private CompletableFuture<KvResponse> getResponse(
+ final KvRequest request, final boolean forceUpdate) {
+ InetSocketAddress serverAddress =
+ queryLocation.getLocation(request.partition(), request.bucket(), forceUpdate);
+ if (serverAddress == null) {
+ return FutureUtils.completedExceptionally(
+ new RuntimeException("Cannot find address for bucket: " + request.bucket()));
+ }
+ return networkClient.sendRequest(serverAddress, request);
+ }
+
+ public void shutdown() {
+ try {
+ networkClient.shutdown().get(10L, TimeUnit.SECONDS);
+ LOG.info("{} was shutdown successfully.", networkClient.getClientName());
+ } catch (Exception e) {
+ LOG.warn(String.format("%s shutdown failed.", networkClient.getClientName()), e);
+ }
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/exceptions/UnknownPartitionBucketException.java
similarity index 63%
rename from paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java
rename to paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/exceptions/UnknownPartitionBucketException.java
index 163e8dfda..850a2c161 100644
--- a/paimon-core/src/main/java/org/apache/paimon/lookup/QueryLookup.java
+++ b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/exceptions/UnknownPartitionBucketException.java
@@ -16,12 +16,16 @@
* limitations under the License.
*/
-package org.apache.paimon.lookup;
+package org.apache.paimon.service.exceptions;
-import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.service.network.BadRequestException;
-/** An interface to provide lookup function. */
-public interface QueryLookup {
+/** Thrown if the server does not hold the given partition and bucket. */
+public class UnknownPartitionBucketException extends BadRequestException {
- BinaryRow[] lookup(BinaryRow partition, int bucket, BinaryRow[] keys);
+ private static final long serialVersionUID = 1L;
+
+ public UnknownPartitionBucketException(String serverName) {
+ super(serverName, "The server does not hold the given partition and bucket.");
+ }
}
diff --git a/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java
index 3a69d8eca..39bcd8c4b 100644
--- a/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java
+++ b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java
@@ -108,7 +108,7 @@ public class NetworkClient<REQ extends MessageBody, RESP extends MessageBody> {
final ThreadFactory threadFactory =
new ThreadFactoryBuilder()
.setDaemon(true)
- .setNameFormat("Flink " + clientName + " Event Loop Thread %d")
+ .setNameFormat("Paimon " + clientName + " Event Loop Thread %d")
.build();
final EventLoopGroup nioGroup = new NioEventLoopGroup(numEventLoopThreads, threadFactory);
diff --git a/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkServer.java b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkServer.java
index 029d72c0d..38df44025 100644
--- a/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkServer.java
+++ b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkServer.java
@@ -149,7 +149,7 @@ public abstract class NetworkServer<REQ extends MessageBody, RESP extends Messag
ThreadFactory threadFactory =
new ThreadFactoryBuilder()
.setDaemon(true)
- .setNameFormat("Flink " + getServerName() + " Thread %d")
+ .setNameFormat("Paimon " + getServerName() + " Thread %d")
.build();
return Executors.newFixedThreadPool(numQueryThreads, threadFactory);
}
@@ -233,7 +233,7 @@ public abstract class NetworkServer<REQ extends MessageBody, RESP extends Messag
final ThreadFactory threadFactory =
new ThreadFactoryBuilder()
.setDaemon(true)
- .setNameFormat("Flink " + serverName + " EventLoop Thread %d")
+ .setNameFormat("Paimon " + serverName + " EventLoop Thread %d")
.build();
final NioEventLoopGroup nioGroup =
diff --git a/paimon-service/paimon-service-runtime/pom.xml b/paimon-service/paimon-service-runtime/pom.xml
index 39e1d4e1f..8f13ca3aa 100644
--- a/paimon-service/paimon-service-runtime/pom.xml
+++ b/paimon-service/paimon-service-runtime/pom.xml
@@ -55,6 +55,13 @@ under the License.
<!-- test dependencies -->
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-format</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-test-utils</artifactId>
@@ -69,5 +76,96 @@ under the License.
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</project>
diff --git a/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvQueryServer.java b/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvQueryServer.java
index 86fcf9124..90ef67446 100644
--- a/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvQueryServer.java
+++ b/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvQueryServer.java
@@ -18,8 +18,8 @@
package org.apache.paimon.service.server;
-import org.apache.paimon.lookup.QueryLookup;
-import org.apache.paimon.lookup.QueryServer;
+import org.apache.paimon.query.QueryServer;
+import org.apache.paimon.query.TableQuery;
import org.apache.paimon.service.messages.KvRequest;
import org.apache.paimon.service.messages.KvResponse;
import org.apache.paimon.service.network.AbstractServerHandler;
@@ -40,44 +40,39 @@ public class KvQueryServer extends NetworkServer<KvRequest, KvResponse> implemen
private static final Logger LOG = LoggerFactory.getLogger(KvQueryServer.class);
- /** The {@link QueryLookup} to query. */
- private final QueryLookup lookup;
-
+ private final int serverId;
+ private final int numServers;
+ private final TableQuery lookup;
private final ServiceRequestStats stats;
- private MessageSerializer<KvRequest, KvResponse> serializer;
-
public KvQueryServer(
+ final int serverId,
+ final int numServers,
final String bindAddress,
final Iterator<Integer> bindPortIterator,
final Integer numEventLoopThreads,
final Integer numQueryThreads,
- final QueryLookup lookup,
+ final TableQuery lookup,
final ServiceRequestStats stats) {
-
super(
"Kv Query Server",
bindAddress,
bindPortIterator,
numEventLoopThreads,
numQueryThreads);
+ this.serverId = serverId;
+ this.numServers = numServers;
this.stats = Preconditions.checkNotNull(stats);
this.lookup = Preconditions.checkNotNull(lookup);
}
@Override
public AbstractServerHandler<KvRequest, KvResponse> initializeHandler() {
- this.serializer =
+ MessageSerializer<KvRequest, KvResponse> serializer =
new MessageSerializer<>(
new KvRequest.KvRequestDeserializer(),
new KvResponse.KvResponseDeserializer());
- return new KvServerHandler(this, lookup, serializer, stats);
- }
-
- public MessageSerializer<KvRequest, KvResponse> getSerializer() {
- Preconditions.checkState(
- serializer != null, "Server " + getServerName() + " has not been started.");
- return serializer;
+ return new KvServerHandler(this, serverId, numServers, lookup, serializer, stats);
}
@Override
diff --git a/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvServerHandler.java b/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvServerHandler.java
index 4f40c1326..b83c3e80d 100644
--- a/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvServerHandler.java
+++ b/paimon-service/paimon-service-runtime/src/main/java/org/apache/paimon/service/server/KvServerHandler.java
@@ -19,7 +19,8 @@
package org.apache.paimon.service.server;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.lookup.QueryLookup;
+import org.apache.paimon.query.TableQuery;
+import org.apache.paimon.service.exceptions.UnknownPartitionBucketException;
import org.apache.paimon.service.messages.KvRequest;
import org.apache.paimon.service.messages.KvResponse;
import org.apache.paimon.service.network.AbstractServerHandler;
@@ -32,6 +33,8 @@ import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandler;
import java.util.concurrent.CompletableFuture;
+import static org.apache.paimon.table.sink.ChannelComputer.select;
+
/**
* This handler dispatches asynchronous tasks, which query values and write the result to the
* channel.
@@ -43,7 +46,9 @@ import java.util.concurrent.CompletableFuture;
@ChannelHandler.Sharable
public class KvServerHandler extends AbstractServerHandler<KvRequest, KvResponse> {
- private final QueryLookup lookup;
+ private final int serverId;
+ private final int numServers;
+ private final TableQuery lookup;
/**
* Create the handler used by the {@link KvQueryServer}.
@@ -56,10 +61,14 @@ public class KvServerHandler extends AbstractServerHandler<KvRequest, KvResponse
*/
public KvServerHandler(
final KvQueryServer server,
- final QueryLookup lookup,
+ final int serverId,
+ final int numServers,
+ final TableQuery lookup,
final MessageSerializer<KvRequest, KvResponse> serializer,
final ServiceRequestStats stats) {
super(server, serializer, stats);
+ this.serverId = serverId;
+ this.numServers = numServers;
this.lookup = Preconditions.checkNotNull(lookup);
}
@@ -68,6 +77,13 @@ public class KvServerHandler extends AbstractServerHandler<KvRequest, KvResponse
final long requestId, final KvRequest request) {
final CompletableFuture<KvResponse> responseFuture = new CompletableFuture<>();
+ int selectServerId = select(request.partition(), request.bucket(), numServers);
+ if (selectServerId != serverId) {
+ responseFuture.completeExceptionally(
+ new UnknownPartitionBucketException(getServerName()));
+ return responseFuture;
+ }
+
try {
BinaryRow[] values =
lookup.lookup(request.partition(), request.bucket(), request.keys());
diff --git a/paimon-service/paimon-service-runtime/src/test/java/org/apache/paimon/service/KvQueryTableTest.java b/paimon-service/paimon-service-runtime/src/test/java/org/apache/paimon/service/KvQueryTableTest.java
new file mode 100644
index 000000000..1a18567d1
--- /dev/null
+++ b/paimon-service/paimon-service-runtime/src/test/java/org/apache/paimon/service/KvQueryTableTest.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.service;
+
+import org.apache.paimon.catalog.PrimaryKeyTableTestBase;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.query.QueryLocationImpl;
+import org.apache.paimon.query.TableQuery;
+import org.apache.paimon.service.client.KvQueryClient;
+import org.apache.paimon.service.network.stats.DisabledServiceRequestStats;
+import org.apache.paimon.service.server.KvQueryServer;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.utils.Pair;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.paimon.io.DataFileTestUtils.row;
+import static org.apache.paimon.service.ServiceManager.PRIMARY_KEY_LOOKUP;
+import static org.apache.paimon.table.sink.ChannelComputer.select;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for remote lookup. */
+public class KvQueryTableTest extends PrimaryKeyTableTestBase {
+
+ private TestTableQuery query0;
+ private TestTableQuery query1;
+
+ private KvQueryServer server0;
+ private KvQueryServer server1;
+
+ private KvQueryClient client;
+
+ @BeforeEach
+ public void beforeEach() {
+ this.query0 = new TestTableQuery();
+ this.query1 = new TestTableQuery();
+
+ this.server0 = createServer(0, query0, 7777);
+ this.server1 = createServer(1, query1, 7900);
+ registryServers();
+
+ this.client =
+ new KvQueryClient(new QueryLocationImpl(table.store().newServiceManager()), 1);
+ }
+
+ private void registryServers() {
+ InetSocketAddress[] addresses =
+ new InetSocketAddress[] {server0.getServerAddress(), server1.getServerAddress()};
+ ServiceManager serviceManager = table.store().newServiceManager();
+ serviceManager.resetService(PRIMARY_KEY_LOOKUP, addresses);
+ }
+
+ private KvQueryServer createServer(int serverId, TableQuery query, int port) {
+ try {
+ KvQueryServer server =
+ new KvQueryServer(
+ serverId,
+ 2,
+ InetAddress.getLocalHost().getHostName(),
+ Collections.singletonList(port).iterator(),
+ 1,
+ 1,
+ query,
+ new DisabledServiceRequestStats());
+ server.start();
+ return server;
+ } catch (Throwable e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @AfterEach
+ public void afterEach() {
+ shutdownServers();
+ if (client != null) {
+ client.shutdown();
+ }
+ }
+
+ private void shutdownServers() {
+ if (server0 != null) {
+ server0.shutdown();
+ }
+ if (server1 != null) {
+ server1.shutdown();
+ }
+ }
+
+ @Test
+ public void testRemoteGet() throws Exception {
+ // test not exists
+ BinaryRow[] result = client.getValues(row(1), 0, new BinaryRow[] {row(1)}).get();
+ assertThat(result).containsOnly((BinaryRow) null);
+
+ // test 1 row
+ write(1, 1, 1);
+ result = client.getValues(row(1), 0, new BinaryRow[] {row(1)}).get();
+ assertThat(result).containsOnly(row(1, 1, 1));
+
+ // test many partitions
+ for (int i = 2; i < 10; i++) {
+ write(i, 1, 2);
+ result = client.getValues(row(i), 0, new BinaryRow[] {row(1)}).get();
+ assertThat(result).containsOnly(row(i, 1, 2));
+ }
+
+ // test 2 rows
+ write(1, 2, 1);
+ result = client.getValues(row(1), 0, new BinaryRow[] {row(1), row(2)}).get();
+ assertThat(result).containsOnly(row(1, 1, 1), row(1, 2, 1));
+ }
+
+ @Test
+ public void testServerRestartSamePorts() throws Throwable {
+ innerTestServerRestart(
+ () -> {
+ shutdownServers();
+ KvQueryTableTest.this.server0 = createServer(0, query0, 7777);
+ KvQueryTableTest.this.server1 = createServer(1, query1, 7900);
+ registryServers();
+ });
+ }
+
+ @Test
+ public void testServerRestartSwitchPorts() throws Throwable {
+ innerTestServerRestart(
+ () -> {
+ shutdownServers();
+ KvQueryTableTest.this.server0 = createServer(0, query0, 7900);
+ KvQueryTableTest.this.server1 = createServer(1, query1, 7777);
+ registryServers();
+ });
+ }
+
+ @Test
+ public void testServerRestartChangePorts() throws Throwable {
+ innerTestServerRestart(
+ () -> {
+ shutdownServers();
+ KvQueryTableTest.this.server0 = createServer(0, query0, 7778);
+ KvQueryTableTest.this.server1 = createServer(1, query1, 7901);
+ registryServers();
+ });
+ }
+
+ private void innerTestServerRestart(Runnable restart) throws Throwable {
+ // insert many records
+ BinaryRow[] result;
+ for (int i = 1; i < 10; i++) {
+ write(i, 1, 2);
+ result = client.getValues(row(i), 0, new BinaryRow[] {row(1)}).get();
+ assertThat(result).containsOnly(row(i, 1, 2));
+ }
+
+ restart.run();
+
+ // query again
+ for (int i = 1; i < 10; i++) {
+ result = client.getValues(row(i), 0, new BinaryRow[] {row(1)}).get();
+ assertThat(result).containsOnly(row(i, 1, 2));
+ }
+ }
+
+ private void write(int partition, int key, int value) throws Exception {
+ int bucket = computeBucket(partition, key, value);
+ TestTableQuery query = select(row(partition), bucket, 2) == 0 ? query0 : query1;
+ query.put(row(partition), bucket, row(key), row(partition, key, value));
+ }
+
+ private int computeBucket(int partition, int key, int value) throws Exception {
+ try (BatchTableWrite write = table.newBatchWriteBuilder().newWrite()) {
+ write.write(GenericRow.of(partition, key, value));
+ List<CommitMessage> commitMessages = write.prepareCommit();
+ return commitMessages.get(0).bucket();
+ }
+ }
+
+ private static class TestTableQuery implements TableQuery {
+
+ private final Map<Pair<BinaryRow, Integer>, Map<BinaryRow, BinaryRow>> values =
+ new HashMap<>();
+
+ public void put(BinaryRow partition, int bucket, BinaryRow key, BinaryRow value) {
+ values.computeIfAbsent(Pair.of(partition, bucket), k -> new HashMap<>())
+ .put(key, value);
+ }
+
+ @Override
+ public BinaryRow[] lookup(BinaryRow partition, int bucket, BinaryRow[] keys) {
+ Map<BinaryRow, BinaryRow> map = values.get(Pair.of(partition, bucket));
+ BinaryRow[] values = new BinaryRow[keys.length];
+ if (map != null) {
+ for (int i = 0; i < keys.length; i++) {
+ values[i] = map.get(keys[i]);
+ }
+ }
+ return values;
+ }
+ }
+}