You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@paimon.apache.org by "leaves12138 (via GitHub)" <gi...@apache.org> on 2024/03/21 03:44:48 UTC
[PR] [core] File index interface and serder utils. [incubator-paimon]
leaves12138 opened a new pull request, #3068:
URL: https://github.com/apache/incubator-paimon/pull/3068
Related to PIP-17 (https://cwiki.apache.org/confluence/display/PAIMON/PIP-17%3A+Introduce+secondary+column+index)
class:
FileIndex:
File index interface.
InternalRowToBytesVisitor:
convert a row with specified position to bytes
ObjectToBytesVisitor:
convert object to bytes
FileIndexPreicateUtil, FileIndexPredicateTester:
Predicate utils. Add tests after Bloomfilter code added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540421577
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
Review Comment:
Document your file format.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540428449
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java:
##########
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.fs.ByteArraySeekableStream;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.predicate.CompoundPredicate;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.Or;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
+import org.apache.paimon.types.RowType;
+
+import javax.annotation.Nullable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Utils to check secondary index (e.g. bloom filter) predicate. */
+public class FileIndexPredicate implements Closeable {
+
+ private final FileIndexFile.Reader reader;
+ private final Map<String, FileIndexPredicateWorker> cacheTester = new HashMap<>();
+
+ public FileIndexPredicate(Path path, FileIO fileIO, RowType fileRowType) throws IOException {
+ this(fileIO.newInputStream(path), fileRowType);
+ }
+
+ public FileIndexPredicate(byte[] serializedBytes, RowType fileRowType) {
+ this(new ByteArraySeekableStream(serializedBytes), fileRowType);
+ }
+
+ public FileIndexPredicate(SeekableInputStream inputStream, RowType fileRowType) {
+ this.reader = FileIndexFile.createReader(inputStream, fileRowType);
+ }
+
+ public boolean testPredicate(@Nullable Predicate filePredicate) {
+ if (filePredicate == null) {
+ return true;
+ }
+
+ Set<String> requredFieldNames = getRequiredNames(filePredicate);
+
+ List<FileIndexPredicateWorker> testWorkers =
+ requredFieldNames.stream()
+ .map(
+ cname ->
+ cacheTester.computeIfAbsent(
+ cname,
+ k ->
+ new FileIndexPredicateWorker(
+ cname,
+ reader.readColumnIndex(cname))))
+ .collect(Collectors.toList());
+
+ for (FileIndexPredicateWorker testWorker : testWorkers) {
+ if (!testWorker.test(filePredicate)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private Set<String> getRequiredNames(Predicate filePredicate) {
+ return filePredicate.visit(
+ new PredicateVisitor<Set<String>>() {
+ final Set<String> names = new HashSet<>();
+
+ @Override
+ public Set<String> visit(LeafPredicate predicate) {
+ names.add(predicate.fieldName());
+ return names;
+ }
+
+ @Override
+ public Set<String> visit(CompoundPredicate predicate) {
+ for (Predicate child : predicate.children()) {
+ child.visit(this);
+ }
+ return names;
+ }
+ });
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.reader.close();
+ }
+
+ /** Predicate test worker. */
+ private static class FileIndexPredicateWorker implements PredicateVisitor<Boolean> {
Review Comment:
`FileIndexFieldPredicate`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "leaves12138 (via GitHub)" <gi...@apache.org>.
leaves12138 commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540541756
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - startPosition));
+ }
+
+ return baos.toByteArray();
+ }
+
+ private void writeHead(String indexType, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+
+ int headLength = calculateHeadLength(indexType, bodyInfo);
+
+ // writeMagic
+ dataOutputStream.writeLong(MAGIC);
+ // writeVersion
+ dataOutputStream.writeInt(Version.V_1.version());
+ // writeHeadLength
+ dataOutputStream.writeInt(headLength);
+ // writeIndexType
+ dataOutputStream.writeUTF(indexType);
+ // writeColumnSize
+ dataOutputStream.writeInt(bodyInfo.size());
+ // writeColumnInfo, offset = headLength
+ for (Map.Entry<String, Pair<Integer, Integer>> entry : bodyInfo.entrySet()) {
Review Comment:
We `must` read all head columns and put them into a hashmap not a array list, so we don't need to sort.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "leaves12138 (via GitHub)" <gi...@apache.org>.
leaves12138 commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540401295
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - 1));
Review Comment:
Good question, but I think `startPosition` and `endPosition` are more convenient to wrap for `BytesInputWrapper`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540424641
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
Review Comment:
just inline `constructBody`.
It is better to avoid pass a variable field to the method.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540428746
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java:
##########
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.FunctionVisitor;
+
+import java.util.List;
+
+/**
+ * Read file index from serialized bytes. Return true, means we need to search this file, else means
+ * needn't.
+ */
+public interface FileIndexReader extends FunctionVisitor<Boolean> {
+
+ FileIndexReader recoverFrom(SeekableInputStream inputStream);
Review Comment:
directly using byte[]?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and serder utils. [incubator-paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/incubator-paimon/pull/3068#discussion_r1533450695
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFunctionVisitor.java:
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.FunctionVisitor;
+
+import java.util.List;
+
+/**
+ * Read file index from serialized bytes. Return true, means we need to search this file, else means
+ * needn't.
+ */
+public interface FileIndexFunctionVisitor extends FunctionVisitor<Boolean> {
Review Comment:
FileIndexReader
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicateTester.java:
##########
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.predicate.CompoundPredicate;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.Or;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
+
+/** Predicate test. */
+public class FileIndexPredicateTester implements PredicateVisitor<Boolean> {
Review Comment:
Wrap this class into `FileIndexReader`.
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicateUtil.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.predicate.CompoundPredicate;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Pair;
+
+import javax.annotation.Nullable;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Utils to check secondary index (e.g. bloom filter) predicate. */
+public class FileIndexPredicateUtil {
Review Comment:
`FileIndexPredicate`
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicateUtil.java:
##########
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.predicate.CompoundPredicate;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Pair;
+
+import javax.annotation.Nullable;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Utils to check secondary index (e.g. bloom filter) predicate. */
+public class FileIndexPredicateUtil {
+
+ public static boolean checkPredicate(
+ Path path, FileIO fileIO, RowType fileRowType, @Nullable Predicate filePredicate)
+ throws IOException {
+ try (DataInputStream dataInput = new DataInputStream(fileIO.newInputStream(path))) {
+ return checkPredicate(dataInput, fileRowType, filePredicate);
+ }
+ }
+
+ public static boolean checkPredicate(
+ byte[] serializedBytes, RowType fileRowType, @Nullable Predicate filePredicate) {
+ DataInput dataInput = new DataInputStream(new ByteArrayInputStream(serializedBytes));
+ return checkPredicate(dataInput, fileRowType, filePredicate);
+ }
+
+ public static boolean checkPredicate(
+ DataInput dataInput, RowType fileRowType, @Nullable Predicate filePredicate) {
+ if (filePredicate == null) {
+ return true;
+ }
+
+ Set<String> requiredFields =
+ filePredicate.visit(
+ new PredicateVisitor<Set<String>>() {
+ final Set<String> names = new HashSet<>();
+
+ @Override
+ public Set<String> visit(LeafPredicate predicate) {
+ names.add(predicate.fieldName());
+ return names;
+ }
+
+ @Override
+ public Set<String> visit(CompoundPredicate predicate) {
+ for (Predicate child : predicate.children()) {
+ child.visit(this);
+ }
+ return names;
+ }
+ });
+
+ Pair<String, Map<String, byte[]>> pair = deserializeIndexString(dataInput, requiredFields);
+ Map<String, DataType> fileTypes = new HashMap<>();
+ fileRowType.getFields().forEach(field -> fileTypes.put(field.name(), field.type()));
+
+ String type = pair.getLeft();
+ Map<String, byte[]> checker = pair.getRight();
+
+ List<FileIndexPredicateTester> testers =
+ checker.entrySet().stream()
+ .map(
+ entry ->
+ new FileIndexPredicateTester(
+ entry.getKey(),
+ FileIndexer.create(
+ type, fileTypes.get(entry.getKey()))
+ .createVisitor()
+ .recoverFrom(entry.getValue())))
+ .collect(Collectors.toList());
+
+ for (FileIndexPredicateTester tester : testers) {
+ if (!tester.test(filePredicate)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static byte[] serializeIndexMap(String type, Map<String, byte[]> indexMap) {
+ try {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
Review Comment:
You need to define a file structure to separate offsets and index bytes.
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/ObjectToBytesVisitor.java:
##########
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.DataGetters;
+import org.apache.paimon.data.Decimal;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalMap;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.BigIntType;
+import org.apache.paimon.types.BinaryType;
+import org.apache.paimon.types.BooleanType;
+import org.apache.paimon.types.CharType;
+import org.apache.paimon.types.DataTypeVisitor;
+import org.apache.paimon.types.DateType;
+import org.apache.paimon.types.DecimalType;
+import org.apache.paimon.types.DoubleType;
+import org.apache.paimon.types.FloatType;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.LocalZonedTimestampType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.MultisetType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.SmallIntType;
+import org.apache.paimon.types.TimeType;
+import org.apache.paimon.types.TimestampType;
+import org.apache.paimon.types.TinyIntType;
+import org.apache.paimon.types.VarBinaryType;
+import org.apache.paimon.types.VarCharType;
+
+import java.util.Arrays;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+/** Convert different type object to bytes. */
+public class ObjectToBytesVisitor implements DataTypeVisitor<Function<Object, byte[]>> {
Review Comment:
Maybe we don't need this class now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi merged PR #3068:
URL: https://github.com/apache/paimon/pull/3068
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "zyl891229 (via GitHub)" <gi...@apache.org>.
zyl891229 commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540395797
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - 1));
Review Comment:
Would it be easier to read with offset and length?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and serder utils. [incubator-paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/incubator-paimon/pull/3068#discussion_r1533282350
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndex.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.types.DataType;
+
+/**
+ * Secondary index filter interface. Return true, means we need to search this file, else means
+ * needn't.
+ */
+public interface FileIndex {
Review Comment:
Maybe it can be a `FunctionVisitor<Boolean>`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540426475
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - startPosition));
+ }
+
+ return baos.toByteArray();
+ }
+
+ private void writeHead(String indexType, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+
+ int headLength = calculateHeadLength(indexType, bodyInfo);
+
+ // writeMagic
+ dataOutputStream.writeLong(MAGIC);
+ // writeVersion
+ dataOutputStream.writeInt(Version.V_1.version());
+ // writeHeadLength
+ dataOutputStream.writeInt(headLength);
+ // writeIndexType
+ dataOutputStream.writeUTF(indexType);
+ // writeColumnSize
+ dataOutputStream.writeInt(bodyInfo.size());
+ // writeColumnInfo, offset = headLength
+ for (Map.Entry<String, Pair<Integer, Integer>> entry : bodyInfo.entrySet()) {
+ dataOutputStream.writeUTF(entry.getKey());
+ dataOutputStream.writeInt(entry.getValue().getLeft() + headLength);
+ dataOutputStream.writeInt(entry.getValue().getRight());
+ }
+ // writeRedundantLength
+ dataOutputStream.writeInt(REDUNDANT_LENGTH);
+ }
+
+ int calculateHeadLength(String indexType, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ // magic 8 bytes, version 4 bytes, head length 4 bytes,
+ // column size 4 bytes, body info start&end 8 bytes per
+ // item, redundant length 4 bytes;
+ int baseLength = 8 + 4 + 4 + 4 + bodyInfo.size() * 8 + 4;
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(baos);
+ dataOutput.writeUTF(indexType);
+ for (String s : bodyInfo.keySet()) {
+ dataOutput.writeUTF(s);
+ }
+
+ return baseLength + baos.size();
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.closeQuietly(dataOutputStream);
+ }
+ }
+
+ /** Reader for file index file. */
+ public static class Reader implements Closeable {
+
+ private final SeekableInputStream seekableInputStream;
+ // get header and cache it.
+ private final Map<String, Pair<Integer, Integer>> header = new HashMap<>();
+ private final Map<String, DataField> fields = new HashMap<>();
+ private final String type;
+
+ public Map<String, Pair<Integer, Integer>> getHeader() {
Review Comment:
this method should be placed after constructor.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "leaves12138 (via GitHub)" <gi...@apache.org>.
leaves12138 commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540541756
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - startPosition));
+ }
+
+ return baos.toByteArray();
+ }
+
+ private void writeHead(String indexType, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+
+ int headLength = calculateHeadLength(indexType, bodyInfo);
+
+ // writeMagic
+ dataOutputStream.writeLong(MAGIC);
+ // writeVersion
+ dataOutputStream.writeInt(Version.V_1.version());
+ // writeHeadLength
+ dataOutputStream.writeInt(headLength);
+ // writeIndexType
+ dataOutputStream.writeUTF(indexType);
+ // writeColumnSize
+ dataOutputStream.writeInt(bodyInfo.size());
+ // writeColumnInfo, offset = headLength
+ for (Map.Entry<String, Pair<Integer, Integer>> entry : bodyInfo.entrySet()) {
Review Comment:
We `must` read all head columns and put them into a hashmap not an array list, so we don't need to sort.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540427919
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - startPosition));
+ }
+
+ return baos.toByteArray();
+ }
+
+ private void writeHead(String indexType, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+
+ int headLength = calculateHeadLength(indexType, bodyInfo);
+
+ // writeMagic
+ dataOutputStream.writeLong(MAGIC);
+ // writeVersion
+ dataOutputStream.writeInt(Version.V_1.version());
+ // writeHeadLength
+ dataOutputStream.writeInt(headLength);
+ // writeIndexType
+ dataOutputStream.writeUTF(indexType);
+ // writeColumnSize
+ dataOutputStream.writeInt(bodyInfo.size());
+ // writeColumnInfo, offset = headLength
+ for (Map.Entry<String, Pair<Integer, Integer>> entry : bodyInfo.entrySet()) {
+ dataOutputStream.writeUTF(entry.getKey());
+ dataOutputStream.writeInt(entry.getValue().getLeft() + headLength);
+ dataOutputStream.writeInt(entry.getValue().getRight());
+ }
+ // writeRedundantLength
+ dataOutputStream.writeInt(REDUNDANT_LENGTH);
+ }
+
+ int calculateHeadLength(String indexType, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ // magic 8 bytes, version 4 bytes, head length 4 bytes,
+ // column size 4 bytes, body info start&end 8 bytes per
+ // item, redundant length 4 bytes;
+ int baseLength = 8 + 4 + 4 + 4 + bodyInfo.size() * 8 + 4;
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(baos);
+ dataOutput.writeUTF(indexType);
+ for (String s : bodyInfo.keySet()) {
+ dataOutput.writeUTF(s);
+ }
+
+ return baseLength + baos.size();
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.closeQuietly(dataOutputStream);
+ }
+ }
+
+ /** Reader for file index file. */
+ public static class Reader implements Closeable {
+
+ private final SeekableInputStream seekableInputStream;
+ // get header and cache it.
+ private final Map<String, Pair<Integer, Integer>> header = new HashMap<>();
+ private final Map<String, DataField> fields = new HashMap<>();
+ private final String type;
+
+ public Map<String, Pair<Integer, Integer>> getHeader() {
+ return header;
+ }
+
+ public Reader(SeekableInputStream seekableInputStream, RowType fileRowType) {
+ this.seekableInputStream = seekableInputStream;
+ DataInputStream dataInputStream = new DataInputStream(seekableInputStream);
+ fileRowType.getFields().forEach(field -> this.fields.put(field.name(), field));
+ try {
+ long magic = dataInputStream.readLong();
+ if (magic != MAGIC) {
+ throw new RuntimeException("This file is not file index file.");
+ }
+
+ int version = dataInputStream.readInt();
+ if (version != Version.V_1.version()) {
+ throw new RuntimeException(
+ "This index file is version of "
+ + version
+ + ", not in supported version list ["
+ + Version.V_1.version()
+ + "]");
+ }
+
+ int headLength = dataInputStream.readInt();
+ byte[] head = new byte[headLength - 8 - 4 - 4];
+ dataInputStream.readFully(head);
+
+ try (DataInputStream dataInput =
+ new DataInputStream(new ByteArrayInputStream(head))) {
+ this.type = dataInput.readUTF();
+ int columnSize = dataInput.readInt();
+ for (int i = 0; i < columnSize; i++) {
+ this.header.put(
+ dataInput.readUTF(),
+ Pair.of(dataInput.readInt(), dataInput.readInt()));
+ }
+ }
+
+ } catch (IOException e) {
+ IOUtils.closeQuietly(seekableInputStream);
+ throw new RuntimeException(
+ "Exception happens while construct file index reader.", e);
+ }
+ }
+
+ public FileIndexReader readColumnIndex(String columnName) {
+
+ return readColumnInputStream(columnName)
+ .map(
+ inputStream ->
+ FileIndexer.create(type, fields.get(columnName).type())
+ .createReader()
+ .recoverFrom(inputStream))
+ .orElse(null);
+ }
+
+ @VisibleForTesting
+ Optional<SeekableInputStream> readColumnInputStream(String columnName) {
+ return Optional.ofNullable(header.getOrDefault(columnName, null))
+ .map(
+ startEnd ->
+ new BytesInputWrapper(
+ seekableInputStream,
+ startEnd.getLeft(),
+ startEnd.getRight()))
+ .map(
+ stream -> {
+ try {
+ return stream.resetHead();
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Error happens while read column from index file.", e);
+ }
+ });
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.closeQuietly(seekableInputStream);
+ }
+ }
+
+ /** Wrap the origin stream with offset start and end. */
+ static class BytesInputWrapper extends SeekableInputStream {
Review Comment:
I cannot get why we need this, why not just read bytes directly?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540428552
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java:
##########
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.fs.ByteArraySeekableStream;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.predicate.CompoundPredicate;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.Or;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
+import org.apache.paimon.types.RowType;
+
+import javax.annotation.Nullable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Utils to check secondary index (e.g. bloom filter) predicate. */
+public class FileIndexPredicate implements Closeable {
+
+ private final FileIndexFile.Reader reader;
+ private final Map<String, FileIndexPredicateWorker> cacheTester = new HashMap<>();
Review Comment:
cacheTester -> fieldPredicates?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540421577
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
Review Comment:
Document your file format. Convert your picture to words.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540424641
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
Review Comment:
why can you just write to `dataOutputStream`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "zyl891229 (via GitHub)" <gi...@apache.org>.
zyl891229 commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540537276
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - startPosition));
+ }
+
+ return baos.toByteArray();
+ }
+
+ private void writeHead(String indexType, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+
+ int headLength = calculateHeadLength(indexType, bodyInfo);
+
+ // writeMagic
+ dataOutputStream.writeLong(MAGIC);
+ // writeVersion
+ dataOutputStream.writeInt(Version.V_1.version());
+ // writeHeadLength
+ dataOutputStream.writeInt(headLength);
+ // writeIndexType
+ dataOutputStream.writeUTF(indexType);
+ // writeColumnSize
+ dataOutputStream.writeInt(bodyInfo.size());
+ // writeColumnInfo, offset = headLength
+ for (Map.Entry<String, Pair<Integer, Integer>> entry : bodyInfo.entrySet()) {
Review Comment:
can we sort the key when write and use an algorithm (e.g. binary search) to speed up the search when read
in case of the large number of keys
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "zyl891229 (via GitHub)" <gi...@apache.org>.
zyl891229 commented on PR #3068:
URL: https://github.com/apache/paimon/pull/3068#issuecomment-2022228514
+1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and serder utils. [incubator-paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/incubator-paimon/pull/3068#discussion_r1533282350
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndex.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.types.DataType;
+
+/**
+ * Secondary index filter interface. Return true, means we need to search this file, else means
+ * needn't.
+ */
+public interface FileIndex {
Review Comment:
Maybe it can be a `FunctionVisitor<Boolean>`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540424195
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
Review Comment:
`LocalIndexFile`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540424195
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
Review Comment:
`FileIndexFormat`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [core] File index interface and file format. [paimon]
Posted by "leaves12138 (via GitHub)" <gi...@apache.org>.
leaves12138 commented on code in PR #3068:
URL: https://github.com/apache/paimon/pull/3068#discussion_r1540401295
##########
paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFile.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.Pair;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** File index file format. Put all column and offset in the header. */
+public final class FileIndexFile {
+
+ private static final long MAGIC = 1493475289347502L;
+
+ enum Version {
+ V_1(1);
+
+ private final int version;
+
+ Version(int version) {
+ this.version = version;
+ }
+
+ public int version() {
+ return version;
+ }
+ }
+
+ public static Writer createWriter(OutputStream outputStream) {
+ return new Writer(outputStream);
+ }
+
+ public static Reader createReader(SeekableInputStream inputStream, RowType fileRowType) {
+ return new Reader(inputStream, fileRowType);
+ }
+
+ /** Writer for file index file. */
+ public static class Writer implements Closeable {
+
+ private final DataOutputStream dataOutputStream;
+
+ // for version compatible
+ private static final int REDUNDANT_LENGTH = 0;
+
+ public Writer(OutputStream outputStream) {
+ this.dataOutputStream = new DataOutputStream(outputStream);
+ }
+
+ public void writeColumnIndex(String indexType, Map<String, byte[]> bytesMap)
+ throws IOException {
+
+ Map<String, Pair<Integer, Integer>> bodyInfo = new HashMap<>();
+ byte[] body = constructBody(bytesMap, bodyInfo);
+
+ writeHead(indexType, bodyInfo);
+
+ // writeBody
+ dataOutputStream.write(body);
+ }
+
+ private byte[] constructBody(
+ Map<String, byte[]> bytesMap, Map<String, Pair<Integer, Integer>> bodyInfo)
+ throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (Map.Entry<String, byte[]> entry : bytesMap.entrySet()) {
+ Integer startPosition = baos.size();
+ baos.write(entry.getValue());
+ bodyInfo.put(entry.getKey(), Pair.of(startPosition, baos.size() - 1));
Review Comment:
Good question, but I think `startPosition` and `endPosition` are more convenient to wrap for `BytesInputWrapper`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org