You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ni...@apache.org on 2022/04/15 12:18:40 UTC
[ignite] branch IGNITE-14449 updated: IGNITE-14449 Code review fixes
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch IGNITE-14449
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/IGNITE-14449 by this push:
new 45d3c3242b8 IGNITE-14449 Code review fixes
45d3c3242b8 is described below
commit 45d3c3242b80aa10d5485887c63294cd17a99e96
Author: Nikolay Izhikov <ni...@apache.org>
AuthorDate: Fri Apr 15 15:18:27 2022 +0300
IGNITE-14449 Code review fixes
---
.../ignite/internal/cdc/CdcConsumerState.java | 255 ++++++---------------
.../org/apache/ignite/internal/cdc/CdcMain.java | 38 ++-
2 files changed, 89 insertions(+), 204 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java
index e351d3636ab..7e83993ed90 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcConsumerState.java
@@ -18,6 +18,8 @@
package org.apache.ignite.internal.cdc;
import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
@@ -65,12 +67,6 @@ public class CdcConsumerState {
/** */
public static final String MAPPINGS_STATE_FILE_NAME = "cdc-mappings-state" + FILE_SUFFIX;
- /** Long size in bytes. */
- private static final int LONG_SZ = 8;
-
- /** Integer size in bytes. */
- private static final int INT_SZ = 4;
-
/** Log. */
private final IgniteLogger log;
@@ -111,16 +107,57 @@ public class CdcConsumerState {
* @param state WAL pointer and index of {@link DataEntry} inside {@link DataRecord}.
*/
public void saveWal(T2<WALPointer, Integer> state) throws IOException {
- save(() -> {
+ ByteBuffer buf = ByteBuffer.allocate(POINTER_SIZE);
+
+ buf.putLong(state.get1().index());
+ buf.putInt(state.get1().fileOffset());
+ buf.putInt(state.get2());
+ buf.flip();
+
+ try (FileChannel ch = FileChannel.open(tmpWalPtr, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
+ ch.write(buf);
+
+ ch.force(true);
+ }
+
+ Files.move(tmpWalPtr, walPtr, ATOMIC_MOVE, REPLACE_EXISTING);
+ }
+
+ /**
+ * Loads CDC consumer state from file.
+ *
+ * @return Saved state.
+ */
+ public T2<WALPointer, Integer> loadWalState() {
+ if (!Files.exists(walPtr))
+ return null;
+
+ T2<WALPointer, Integer> state;
+
+ try (FileChannel ch = FileChannel.open(walPtr, StandardOpenOption.READ)) {
ByteBuffer buf = ByteBuffer.allocate(POINTER_SIZE);
- buf.putLong(state.get1().index());
- buf.putInt(state.get1().fileOffset());
- buf.putInt(state.get2());
+ int read = ch.read(buf);
+
+ if (read != POINTER_SIZE)
+ return null;
+
buf.flip();
- return buf;
- }, tmpWalPtr, walPtr);
+ long idx = buf.getLong();
+ int offset = buf.getInt();
+ int entryIdx = buf.getInt();
+
+ state = new T2<>(new WALPointer(idx, offset, 0), entryIdx);
+ }
+ catch (IOException e) {
+ throw new IgniteException("Failed to read state [file=" + walPtr + ']', e);
+ }
+
+ if (state != null && log.isInfoEnabled())
+ log.info("Initial WAL state loaded [ptr=" + state.get1() + ", idx=" + state.get2() + ']');
+
+ return state;
}
/**
@@ -138,20 +175,16 @@ public class CdcConsumerState {
* @param mappingsState Mappings state.
*/
public void saveMappings(Set<T2<Integer, Byte>> mappingsState) throws IOException {
- save(() -> {
- ByteBuffer buf = ByteBuffer.allocate(INT_SZ + (INT_SZ + 1 /* byte */) * mappingsState.size());
-
- buf.putInt(mappingsState.size());
-
- for (T2<Integer, Byte> entry : mappingsState) {
- buf.putInt(entry.get1());
- buf.put(entry.get2());
- }
+ save(mappingsState, tmpMappings, mappings);
+ }
- buf.flip();
+ /** Save object to file. */
+ private <T> void save(T state, Path tmp, Path file) throws IOException {
+ try (ObjectOutputStream oos = new ObjectOutputStream(Files.newOutputStream(tmp))) {
+ oos.writeObject(state);
+ }
- return buf;
- }, tmpMappings, mappings);
+ Files.move(tmp, file, ATOMIC_MOVE, REPLACE_EXISTING);
}
/**
@@ -159,49 +192,17 @@ public class CdcConsumerState {
*
* @return Saved state.
*/
- public Set<T2<Integer, Byte>> loadMappings() {
- Set<T2<Integer, Byte>> state = load(mappings, ch -> {
- ByteBuffer buf = ByteBuffer.allocate(INT_SZ);
-
- int read = ch.read(buf);
-
- if (read != INT_SZ)
- return null;
-
- buf.flip();
-
- int sz = buf.getInt();
- int dataAmount = sz * (INT_SZ + 1 /* byte */);
-
- buf = ByteBuffer.allocate(dataAmount);
-
- read = ch.read(buf);
-
- if (read != dataAmount)
- return null;
-
- buf.flip();
-
- Set<T2<Integer, Byte>> data = new HashSet<>();
-
- for (int i = 0; i < sz; i++) {
- int typeId = buf.getInt();
- byte platform = buf.get();
-
- data.add(new T2<>(typeId, platform));
- }
-
- return data;
- });
+ public Set<T2<Integer, Byte>> loadMappingsState() {
+ Set<T2<Integer, Byte>> state = load(mappings, HashSet::new);
log.info("Initial mappings state loaded [mappingsCnt=" + (state != null ? state.size() : 0) + ']');
- if (state != null && log.isDebugEnabled()) {
+ if (log.isDebugEnabled()) {
for (T2<Integer, Byte> m : state)
log.debug("Mapping [typeId=" + m.get1() + ", platform=" + m.get2() + ']');
}
- return state != null ? state : new HashSet<>();
+ return state;
}
/**
@@ -209,8 +210,8 @@ public class CdcConsumerState {
*
* @return Saved state.
*/
- public Map<Integer, Long> loadTypes() {
- Map<Integer, Long> state = load(types);
+ public Map<Integer, Long> loadTypesState() {
+ Map<Integer, Long> state = load(types, HashMap::new);
log.info("Initial types state loaded [typesCnt=" + state.size() + ']');
@@ -222,130 +223,16 @@ public class CdcConsumerState {
return state;
}
- /**
- * Loads CDC consumer state from file.
- *
- * @return Saved state.
- */
- public T2<WALPointer, Integer> loadWal() {
- T2<WALPointer, Integer> state = load(walPtr, ch -> {
- ByteBuffer buf = ByteBuffer.allocate(POINTER_SIZE);
-
- int read = ch.read(buf);
-
- if (read != POINTER_SIZE)
- return null;
-
- buf.flip();
-
- long idx = buf.getLong();
- int offset = buf.getInt();
- int entryIdx = buf.getInt();
-
- return new T2<>(new WALPointer(idx, offset, 0), entryIdx);
- });
-
- if (state != null && log.isInfoEnabled())
- log.info("Initial WAL state loaded [ptr=" + state.get1() + ", idx=" + state.get2() + ']');
-
- return state;
- }
-
- /** Saves state to file. */
- private void save(Map<Integer, Long> state, Path tmpFile, Path file) throws IOException {
- save(() -> {
- ByteBuffer buf = ByteBuffer.allocate(INT_SZ + (LONG_SZ + INT_SZ) * state.size());
-
- buf.putInt(state.size());
-
- for (Map.Entry<Integer, Long> entry : state.entrySet()) {
- buf.putInt(entry.getKey());
- buf.putLong(entry.getValue());
- }
-
- buf.flip();
-
- return buf;
- }, tmpFile, file);
- }
-
- /** Loads state from file. */
- private Map<Integer, Long> load(Path file) {
- Map<Integer, Long> state = load(file, ch -> {
- ByteBuffer buf = ByteBuffer.allocate(INT_SZ);
-
- int read = ch.read(buf);
-
- if (read != INT_SZ)
- return null;
-
- buf.flip();
-
- int sz = buf.getInt();
-
- int dataAmount = sz * (LONG_SZ + INT_SZ);
-
- buf = ByteBuffer.allocate(dataAmount);
-
- read = ch.read(buf);
-
- if (read != dataAmount)
- return null;
-
- buf.flip();
-
- Map<Integer, Long> data = new HashMap<>();
-
- for (int i = 0; i < sz; i++) {
- int typeId = buf.getInt();
- long timestamp = buf.getLong();
-
- data.put(typeId, timestamp);
- }
-
- return data;
- });
-
- return state != null ? state : new HashMap<>();
- }
-
- /**
- * @param file File to load from.
- * @param ldr Object loader.
- * @param <T> Result type.
- * @return Result loaded from file.
- */
- private static <T> T load(Path file, IOFunction<FileChannel, T> ldr) {
- if (!Files.exists(file))
- return null;
+ /** Loads data from path. */
+ private <D> D load(Path state, Supplier<D> dflt) {
+ if (!Files.exists(state))
+ return dflt.get();
- try (FileChannel ch = FileChannel.open(file, StandardOpenOption.READ)) {
- return ldr.apply(ch);
+ try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(state))) {
+ return (D)ois.readObject();
}
- catch (IOException e) {
- throw new IgniteException("Failed to read state [file=" + file + ']', e);
- }
- }
-
- /**
- * @param bytes Bytes to save supplier.
- * @param tmp Temp file.
- * @param file Destination file.
- * @throws IOException In case of error.
- */
- private static void save(Supplier<ByteBuffer> bytes, Path tmp, Path file) throws IOException {
- try (FileChannel ch = FileChannel.open(tmp, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
- ch.write(bytes.get());
- ch.force(true);
+ catch (IOException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
}
-
- Files.move(tmp, file, ATOMIC_MOVE, REPLACE_EXISTING);
- }
-
- /** */
- @FunctionalInterface
- private interface IOFunction<T, R> {
- /** */
- R apply(T var) throws IOException;
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
index 578b5eef335..63394c05f4d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java
@@ -31,7 +31,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Function;
import java.util.stream.Stream;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -64,7 +63,6 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.platform.PlatformType;
import org.apache.ignite.startup.cmdline.CdcCommandLineStartup;
@@ -291,9 +289,9 @@ public class CdcMain implements Runnable {
state = createState(cdcDir.resolve(STATE_DIR));
- walState = state.loadWal();
- typesState = state.loadTypes();
- mappingsState = state.loadMappings();
+ walState = state.loadWalState();
+ typesState = state.loadTypesState();
+ mappingsState = state.loadMappingsState();
if (walState != null) {
committedSegmentIdx.value(walState.get1().index());
@@ -603,25 +601,25 @@ public class CdcMain implements Runnable {
if (files == null)
return;
- Iterator<TypeMapping> changedMappings = Arrays.stream(files)
- .map(f -> {
- String fileName = f.getName();
+ Iterator<TypeMapping> changedMappings = Arrays.stream(files).map(f -> {
+ String fileName = f.getName();
- int typeId = BinaryUtils.mappedTypeId(fileName);
- byte platformId = BinaryUtils.mappedFilePlatformId(fileName);
+ int typeId = BinaryUtils.mappedTypeId(fileName);
+ byte platformId = BinaryUtils.mappedFilePlatformId(fileName);
- T2<Integer, Byte> data = new T2<>(typeId, platformId);
+ T2<Integer, Byte> state = new T2<>(typeId, platformId);
- // Filter out files already in `mappingsState`.
- return mappingsState.contains(data) ? null : new Object[] {data, f};
- })
+ if (mappingsState.contains(state))
+ return null;
+
+ mappingsState.add(state);
+
+ return (TypeMapping)new TypeMappingImpl(
+ typeId,
+ BinaryUtils.readMapping(f),
+ platformId == 0 ? PlatformType.JAVA : PlatformType.DOTNET);
+ })
.filter(Objects::nonNull)
- .peek(d -> mappingsState.add((T2<Integer, Byte>)d[0])) // Adding peeked up mappings to state.
- .map((Function<Object[], TypeMapping>)(t -> new TypeMappingImpl(
- ((IgniteBiTuple<Integer, Byte>)t[0]).get1(),
- BinaryUtils.readMapping((File)t[1]),
- ((IgniteBiTuple<Integer, Byte>)t[0]).get2() == 0 ? PlatformType.JAVA : PlatformType.DOTNET)
- ))
.iterator();
if (!changedMappings.hasNext())