You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/04/02 18:49:28 UTC
Change in asterixdb[cheshire-cat]: [NO ISSUE][REP] Log details of files to be replicated
From Murtadha Hubail <mh...@apache.org>:
Murtadha Hubail has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843 )
Change subject: [NO ISSUE][REP] Log details of files to be replicated
......................................................................
[NO ISSUE][REP] Log details of files to be replicated
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Log details of the file to be replicated on master being sending it
to replicas.
- Log details of the file to be replicated before receiving it on a replica.
- Log details on file transfer failures.
Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
---
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
3 files changed, 24 insertions(+), 3 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/43/10843/1
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index d572ba2..97dd049 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -34,9 +34,13 @@
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NetworkingUtil {
+ private static final Logger LOGGER = LogManager.getLogger();
+
private NetworkingUtil() {
throw new AssertionError("This util class should not be initialized.");
}
@@ -72,9 +76,15 @@
long fileSize = fileChannel.size();
long count = fileSize;
long numTransferred = 0;
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ try {
+ while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
+ pos += numTransferred;
+ count -= numTransferred;
+ }
+ } catch (Exception e) {
+ LOGGER.info("failed to download file; file size {}, pos {}, count {}, numTransferred {}", fileSize, pos,
+ count, numTransferred);
+ throw e;
}
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 35dbd49..5f7bee6 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -63,6 +63,7 @@
@Override
public void perform(INcApplicationContext appCtx, IReplicationWorker worker) {
try {
+ LOGGER.info("attempting to replicate {}", this);
final IIOManager ioManager = appCtx.getIoManager();
// resolve path
final FileReference localPath = ioManager.resolve(file);
@@ -127,4 +128,10 @@
final boolean isMetadata = input.readBoolean();
return new ReplicateFileTask(s, i, isMetadata);
}
+
+ @Override
+ public String toString() {
+ return "ReplicateFileTask{" + "file='" + file + '\'' + ", size=" + size + ", indexMetadata=" + indexMetadata
+ + '}';
+ }
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
index f0a4229..7bb2858 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
@@ -32,9 +32,12 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class FileSynchronizer {
+ private static final Logger LOGGER = LogManager.getLogger();
private final INcApplicationContext appCtx;
private final PartitionReplica replica;
@@ -53,6 +56,7 @@
final ISocketChannel channel = replica.getChannel();
final FileReference filePath = ioManager.resolve(file);
ReplicateFileTask task = new ReplicateFileTask(file, filePath.getFile().length(), metadata);
+ LOGGER.info("attempting to replicate {} to replica {}", task, replica);
ReplicationProtocol.sendTo(replica, task);
// send the file itself
try (RandomAccessFile fromFile = new RandomAccessFile(filePath.getFile(), "r");
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
Gerrit-Change-Number: 10843
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-MessageType: newchange
Change in asterixdb[cheshire-cat]: [NO ISSUE][REP] Log details of files to be replicated
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
Anon. E. Moose #1000171 has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843 )
Change subject: [NO ISSUE][REP] Log details of files to be replicated
......................................................................
Patch Set 1:
Analytics Compatibility Compilation Successful
https://cbjenkins.page.link/JHSpW5Th2sf38qKDA : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
Gerrit-Change-Number: 10843
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-CC: Anon. E. Moose #1000171
Gerrit-CC: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Comment-Date: Fri, 02 Apr 2021 18:54:47 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: No
Gerrit-MessageType: comment
Change in asterixdb[cheshire-cat]: [NO ISSUE][REP] Log details of files to be replicated
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Murtadha Hubail <mh...@apache.org>:
Murtadha Hubail has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843 )
Change subject: [NO ISSUE][REP] Log details of files to be replicated
......................................................................
[NO ISSUE][REP] Log details of files to be replicated
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Log details of the file to be replicated on master being sending it
to replicas.
- Log details of the file to be replicated before receiving it on a replica.
- Log details on file transfer failures.
Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
---
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
3 files changed, 24 insertions(+), 3 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/43/10843/1
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index d572ba2..97dd049 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -34,9 +34,13 @@
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NetworkingUtil {
+ private static final Logger LOGGER = LogManager.getLogger();
+
private NetworkingUtil() {
throw new AssertionError("This util class should not be initialized.");
}
@@ -72,9 +76,15 @@
long fileSize = fileChannel.size();
long count = fileSize;
long numTransferred = 0;
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ try {
+ while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
+ pos += numTransferred;
+ count -= numTransferred;
+ }
+ } catch (Exception e) {
+ LOGGER.info("failed to download file; file size {}, pos {}, count {}, numTransferred {}", fileSize, pos,
+ count, numTransferred);
+ throw e;
}
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 35dbd49..5f7bee6 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -63,6 +63,7 @@
@Override
public void perform(INcApplicationContext appCtx, IReplicationWorker worker) {
try {
+ LOGGER.info("attempting to replicate {}", this);
final IIOManager ioManager = appCtx.getIoManager();
// resolve path
final FileReference localPath = ioManager.resolve(file);
@@ -127,4 +128,10 @@
final boolean isMetadata = input.readBoolean();
return new ReplicateFileTask(s, i, isMetadata);
}
+
+ @Override
+ public String toString() {
+ return "ReplicateFileTask{" + "file='" + file + '\'' + ", size=" + size + ", indexMetadata=" + indexMetadata
+ + '}';
+ }
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
index f0a4229..7bb2858 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
@@ -32,9 +32,12 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class FileSynchronizer {
+ private static final Logger LOGGER = LogManager.getLogger();
private final INcApplicationContext appCtx;
private final PartitionReplica replica;
@@ -53,6 +56,7 @@
final ISocketChannel channel = replica.getChannel();
final FileReference filePath = ioManager.resolve(file);
ReplicateFileTask task = new ReplicateFileTask(file, filePath.getFile().length(), metadata);
+ LOGGER.info("attempting to replicate {} to replica {}", task, replica);
ReplicationProtocol.sendTo(replica, task);
// send the file itself
try (RandomAccessFile fromFile = new RandomAccessFile(filePath.getFile(), "r");
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
Gerrit-Change-Number: 10843
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-MessageType: newchange
Change in asterixdb[cheshire-cat]: [NO ISSUE][REP] Log details of files to be replicated
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Murtadha Hubail <mh...@apache.org>:
Murtadha Hubail has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843 )
Change subject: [NO ISSUE][REP] Log details of files to be replicated
......................................................................
[NO ISSUE][REP] Log details of files to be replicated
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Log details of the file to be replicated on master being sending it
to replicas.
- Log details of the file to be replicated before receiving it on a replica.
- Log details on file transfer failures.
Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
M asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
3 files changed, 24 insertions(+), 3 deletions(-)
Approvals:
Murtadha Hubail: Looks good to me, but someone else must approve
Hussain Towaileb: Looks good to me, approved
Jenkins: Verified; Verified
Objections:
Anon. E. Moose #1000171: Violations found
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index d572ba2..97dd049 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -34,9 +34,13 @@
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NetworkingUtil {
+ private static final Logger LOGGER = LogManager.getLogger();
+
private NetworkingUtil() {
throw new AssertionError("This util class should not be initialized.");
}
@@ -72,9 +76,15 @@
long fileSize = fileChannel.size();
long count = fileSize;
long numTransferred = 0;
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ try {
+ while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
+ pos += numTransferred;
+ count -= numTransferred;
+ }
+ } catch (Exception e) {
+ LOGGER.info("failed to download file; file size {}, pos {}, count {}, numTransferred {}", fileSize, pos,
+ count, numTransferred);
+ throw e;
}
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 35dbd49..5f7bee6 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -63,6 +63,7 @@
@Override
public void perform(INcApplicationContext appCtx, IReplicationWorker worker) {
try {
+ LOGGER.info("attempting to replicate {}", this);
final IIOManager ioManager = appCtx.getIoManager();
// resolve path
final FileReference localPath = ioManager.resolve(file);
@@ -127,4 +128,10 @@
final boolean isMetadata = input.readBoolean();
return new ReplicateFileTask(s, i, isMetadata);
}
+
+ @Override
+ public String toString() {
+ return "ReplicateFileTask{" + "file='" + file + '\'' + ", size=" + size + ", indexMetadata=" + indexMetadata
+ + '}';
+ }
}
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
index f0a4229..7bb2858 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/FileSynchronizer.java
@@ -32,9 +32,12 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class FileSynchronizer {
+ private static final Logger LOGGER = LogManager.getLogger();
private final INcApplicationContext appCtx;
private final PartitionReplica replica;
@@ -53,6 +56,7 @@
final ISocketChannel channel = replica.getChannel();
final FileReference filePath = ioManager.resolve(file);
ReplicateFileTask task = new ReplicateFileTask(file, filePath.getFile().length(), metadata);
+ LOGGER.info("attempting to replicate {} to replica {}", task, replica);
ReplicationProtocol.sendTo(replica, task);
// send the file itself
try (RandomAccessFile fromFile = new RandomAccessFile(filePath.getFile(), "r");
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
Gerrit-Change-Number: 10843
Gerrit-PatchSet: 2
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Hussain Towaileb <hu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-MessageType: merged
Change in asterixdb[cheshire-cat]: [NO ISSUE][REP] Log details of files to be replicated
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Murtadha Hubail <mh...@apache.org>:
Murtadha Hubail has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843 )
Change subject: [NO ISSUE][REP] Log details of files to be replicated
......................................................................
Patch Set 1: Code-Review+1
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10843
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I645cc8b466fd9e5eaad791a5dcddc3525db22e3f
Gerrit-Change-Number: 10843
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <mh...@apache.org>
Gerrit-CC: Anon. E. Moose #1000171
Gerrit-Comment-Date: Fri, 02 Apr 2021 20:44:41 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment