You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudberry.apache.org by ma...@apache.org on 2025/01/14 10:58:57 UTC
(cloudberry) 11/16: Fix flacky case 002_archiving.pl
This is an automated email from the ASF dual-hosted git repository.
maxyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit 0d6ac3a13e97fafe46c97a0be3c06e3027194959
Author: Wenru Yan <94...@users.noreply.github.com>
AuthorDate: Wed Jun 21 18:09:07 2023 +0800
Fix flacky case 002_archiving.pl
Commit 700cad0 add 'has_streaming => 1' when make a standby node.
This will start a walreceiver process to receive the wal log from
primary node, rather than just store its contents on the archives
of the primary. In this way, the inserted 2000 rows can all have
been present on standby node.
Revert the enable streaming change when create standby, append conn
info to the postgresql.conf after pass the first check and restart
standby node.
---
src/test/recovery/t/002_archiving.pl | 148 ++++++++++++++++++-----------------
1 file changed, 76 insertions(+), 72 deletions(-)
diff --git a/src/test/recovery/t/002_archiving.pl b/src/test/recovery/t/002_archiving.pl
index 9ad37a4aa9..7d0dc5083f 100644
--- a/src/test/recovery/t/002_archiving.pl
+++ b/src/test/recovery/t/002_archiving.pl
@@ -9,40 +9,41 @@ use TestLib;
use Test::More tests => 14;
use File::Copy;
-# Initialize master node, doing archives
-my $node_master = get_new_node('master');
-$node_master->init(
+# Initialize primary node, doing archives
+my $node_primary = get_new_node('primary');
+$node_primary->init(
has_archiving => 1,
allows_streaming => 1);
my $backup_name = 'my_backup';
+my $primary_connstr = $node_primary->connstr;
# Start it
-$node_master->start;
+$node_primary->start;
# Take backup for standby
-$node_master->backup($backup_name);
+$node_primary->backup($backup_name);
# Initialize standby node from backup, fetching WAL from archives
my $node_standby = get_new_node('standby');
# Note that this makes the standby store its contents on the archives
# of the primary.
-$node_standby->init_from_backup($node_master, $backup_name,
- has_streaming => 1, has_restoring => 1);
+$node_standby->init_from_backup($node_primary, $backup_name,
+ has_restoring => 1);
$node_standby->append_conf('postgresql.conf',
"wal_retrieve_retry_interval = '100ms'");
$node_standby->start;
-# Create some content on master
-$node_master->safe_psql('postgres',
+# Create some content on primary
+$node_primary->safe_psql('postgres',
"CREATE TABLE tab_int AS SELECT generate_series(1,1000) AS a");
my $current_lsn =
- $node_master->safe_psql('postgres', "SELECT pg_current_wal_lsn();");
+ $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn();");
-# Force archiving of WAL file to make it present on master
-$node_master->safe_psql('postgres', "SELECT pg_switch_wal()");
+# Force archiving of WAL file to make it present on primary
+$node_primary->safe_psql('postgres', "SELECT pg_switch_wal()");
# Add some more content, it should not be present on standby
-$node_master->safe_psql('postgres',
+$node_primary->safe_psql('postgres',
"INSERT INTO tab_int VALUES (generate_series(1001,2000))");
# Wait until necessary replay has been done on standby
@@ -55,25 +56,28 @@ my $result =
$node_standby->safe_psql('postgres', "SELECT count(*) FROM tab_int");
is($result, qq(1000), 'check content from archives');
+$node_standby->append_conf('postgresql.conf', qq(primary_conninfo='$primary_connstr'));
+$node_standby->restart;
+
###################### partial wal file tests ############################
# Test the following scenario:
-# master is alive but the standby is promoted. In this case, the last wal file
+# primary is alive but the standby is promoted. In this case, the last wal file
# on the old timeline in the mirror's pg_wal dir is renamed with the suffix ".partial"
# This partial file also gets archived. The original wal file only gets archived once
# the user runs pg_rewind.
# Consider the following example: Let's assume that 0000100004 is the current
-# wal file on the master
+# wal file on the primary
-# start with a master and standby pair
-# add data to master
-# contents of pg_wal on master
+# start with a primary and standby pair
+# add data to primary
+# contents of pg_wal on primary
# 0000100001
# .....
# 0000100003
-# 0000100004 - current wal file on master
+# 0000100004 - current wal file on primary
#
-# master is alive but standby gets promoted
+# primary is alive but standby gets promoted
# contents of pg_wal on standby
# 0000100001
# ....
@@ -85,36 +89,36 @@ is($result, qq(1000), 'check content from archives');
# 0000100003
# 0000100004
#
-# stop master with pg_ctl stop -m fast
-# contents of pg_wal on master
-# 0000100004 on the master gets flushed and gets archived
-# 0000100004.done gets created on master
+# stop primary with pg_ctl stop -m fast
+# contents of pg_wal on primary
+# 0000100004 on the primary gets flushed and gets archived
+# 0000100004.done gets created on primary
# Contents of the archive location
# 0000100003
# 0000100004.partial
# 0000100004
# pg_rewind
-# copies from standby to master
-# removes 0000100004 and 0000100004.done from master's pg_wal dir
+# copies from standby to primary
+# removes 0000100004 and 0000100004.done from primary's pg_wal dir
-$node_master->safe_psql('postgres',
+$node_primary->safe_psql('postgres',
"CREATE TABLE test_partial_wal as SELECT generate_series(1,1000)");
-my $latest_wal_filename_old_timeline = $node_master->safe_psql('postgres', "SELECT pg_walfile_name(pg_current_wal_lsn());");
+my $latest_wal_filename_old_timeline = $node_primary->safe_psql('postgres', "SELECT pg_walfile_name(pg_current_wal_lsn());");
my $latest_done_old_timeline = '/pg_wal/archive_status/' . $latest_wal_filename_old_timeline . '.done';
-my $latest_wal_filepath_old_timeline = $node_master->data_dir . '/pg_wal/' . $latest_wal_filename_old_timeline;
-my $latest_archived_wal_old_timeline = $node_master->archive_dir . '/' . $latest_wal_filename_old_timeline;
+my $latest_wal_filepath_old_timeline = $node_primary->data_dir . '/pg_wal/' . $latest_wal_filename_old_timeline;
+my $latest_archived_wal_old_timeline = $node_primary->archive_dir . '/' . $latest_wal_filename_old_timeline;
my $partial_wal_file_path = '/pg_wal/' . $latest_wal_filename_old_timeline . '.partial';
my $partial_done_file_path = '/pg_wal/archive_status/' . $latest_wal_filename_old_timeline . '.partial.done';
-my $archived_partial_wal_file = $node_master->archive_dir . '/' . $latest_wal_filename_old_timeline . '.partial';
+my $archived_partial_wal_file = $node_primary->archive_dir . '/' . $latest_wal_filename_old_timeline . '.partial';
-#assert that 0000100004 exists on master but it's not archived
-ok(-f "$latest_wal_filepath_old_timeline", 'latest wal file from the old timeline exists on master');
+#assert that 0000100004 exists on primary but it's not archived
+ok(-f "$latest_wal_filepath_old_timeline", 'latest wal file from the old timeline exists on primary');
ok(!-f "$latest_archived_wal_old_timeline", 'latest wal file from the old timeline is not archived yet');
-#Only promote standby once the latest wal file from the master's current timeline has been streamed to the standby
-my $master_current_wal_loc = $node_master->safe_psql('postgres', "SELECT pg_current_wal_lsn();");
-my $query = "SELECT pg_last_wal_receive_lsn() >= '$master_current_wal_loc'::pg_lsn;";
+#Only promote standby once the latest wal file from the primary's current timeline has been streamed to the standby
+my $primary_current_wal_loc = $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn();");
+my $query = "SELECT pg_last_wal_receive_lsn() >= '$primary_current_wal_loc'::pg_lsn;";
$node_standby->poll_query_until('postgres', $query)
or die "Timed out while waiting for standby to receive the latest wal file";
@@ -133,43 +137,43 @@ $node_standby->safe_psql('postgres',
# that the latest file from the old timeline is archived properly
post_standby_promotion_tests();
-$node_master->stop;
+$node_primary->stop;
$node_standby->safe_psql('postgres',
"INSERT INTO test_partial_wal SELECT generate_series(1,1000)");
-post_master_stop_tests();
+post_primary_stop_tests();
my $tmp_check = TestLib::tempdir;
-my $master_datadir = $node_master->data_dir;
-# Keep a temporary postgresql.conf for master node or it would be
+my $primary_datadir = $node_primary->data_dir;
+# Keep a temporary postgresql.conf for primary node or it would be
# overwritten during the rewind.
-copy("$master_datadir/postgresql.conf",
- "$tmp_check/master-postgresql.conf.tmp");
+copy("$primary_datadir/postgresql.conf",
+ "$tmp_check/primary-postgresql.conf.tmp");
local $ENV{PGOPTIONS} = '-c gp_role=utility';
command_ok(['pg_rewind',
"--debug",
"--source-server",
'port='. $node_standby->port . ' dbname=postgres',
- '--target-pgdata=' . $node_master->data_dir],
+ '--target-pgdata=' . $node_primary->data_dir],
'pg_rewind');
post_pg_rewind_tests();
# Now move back postgresql.conf with old settings
-move("$tmp_check/master-postgresql.conf.tmp",
- "$master_datadir/postgresql.conf");
+move("$tmp_check/primary-postgresql.conf.tmp",
+ "$primary_datadir/postgresql.conf");
-# Start the master
-$node_master->start;
-$node_master->safe_psql('postgres',
+# Start the primary
+$node_primary->start;
+$node_primary->safe_psql('postgres',
"INSERT INTO test_partial_wal SELECT generate_series(1,1000)");
sub wait_until_file_exists
{
my ($filepath, $filedesc) = @_;
my $query = "SELECT size IS NOT NULL FROM pg_stat_file('$filepath')";
- # we aren't querying master because we stop the master node for some of the
+ # we aren't querying primary because we stop the primary node for some of the
# scenarios
$node_standby->poll_query_until('postgres', $query)
or die "Timed out while waiting for $filedesc $filepath";
@@ -177,8 +181,8 @@ sub wait_until_file_exists
sub post_standby_promotion_tests
{
- #assert that 0000100004 exists on master
- wait_until_file_exists($latest_wal_filepath_old_timeline, "latest wal file from the old timeline to exist on master");
+ #assert that 0000100004 exists on primary
+ wait_until_file_exists($latest_wal_filepath_old_timeline, "latest wal file from the old timeline to exist on primary");
#assert that 0000100004.partial exists on standby
wait_until_file_exists($node_standby->data_dir . $partial_wal_file_path, "partial wal file from the old timeline to exist on standby");
#assert that 0000100004.partial.done exists on standby
@@ -186,12 +190,12 @@ sub post_standby_promotion_tests
#assert that 0000100004.partial got archived
wait_until_file_exists($archived_partial_wal_file, "latest partial wal file from the old timeline to be archived");
- #assert that 0000100004.partial doesn't exist on master
- ok(!-f $node_master->data_dir . $partial_wal_file_path, 'partial wal file from the old timeline should not exist on master');
- #assert that 0000100004.partial.done doesn't exist on master
- ok(!-f $node_master->data_dir . $partial_done_file_path, 'partial done file from the old timeline should not exist on master');
- #assert that 0000100004.done doesn't exist on master
- ok(!-f $node_master->data_dir . $latest_done_old_timeline, 'done file from the old timeline should not exist on master');
+ #assert that 0000100004.partial doesn't exist on primary
+ ok(!-f $node_primary->data_dir . $partial_wal_file_path, 'partial wal file from the old timeline should not exist on primary');
+ #assert that 0000100004.partial.done doesn't exist on primary
+ ok(!-f $node_primary->data_dir . $partial_done_file_path, 'partial done file from the old timeline should not exist on primary');
+ #assert that 0000100004.done doesn't exist on primary
+ ok(!-f $node_primary->data_dir . $latest_done_old_timeline, 'done file from the old timeline should not exist on primary');
#assert that 0000100004 hasn't been archived
ok(!-f $latest_archived_wal_old_timeline, 'wal file from the old timeline should not be archived');
#assert that 0000100004 doesn't exist on standby
@@ -200,32 +204,32 @@ sub post_standby_promotion_tests
check_history_files();
}
-sub post_master_stop_tests
+sub post_primary_stop_tests
{
- #assert that 0000100004 still exists on master
- wait_until_file_exists($latest_wal_filepath_old_timeline, "latest wal file from the old timeline to exist on master");
- #assert that 0000100004.done exists on master
- wait_until_file_exists($node_master->data_dir . $latest_done_old_timeline, "done file from the old timeline to exist on master");
+ #assert that 0000100004 still exists on primary
+ wait_until_file_exists($latest_wal_filepath_old_timeline, "latest wal file from the old timeline to exist on primary");
+ #assert that 0000100004.done exists on primary
+ wait_until_file_exists($node_primary->data_dir . $latest_done_old_timeline, "done file from the old timeline to exist on primary");
#assert that 0000100004 is archived
wait_until_file_exists($latest_archived_wal_old_timeline, "latest wal file from the old timeline to be archived");
}
sub post_pg_rewind_tests
{
- #assert that 0000100004.partial exists on master
- wait_until_file_exists($node_master->data_dir . $partial_wal_file_path, "latest partial wal file from the old timeline to exist on master");
- #assert that 0000100004.partial.done exists on master
- wait_until_file_exists($node_master->data_dir . $partial_done_file_path, "latest partial done file from the old timeline to exist on master");
+ #assert that 0000100004.partial exists on primary
+ wait_until_file_exists($node_primary->data_dir . $partial_wal_file_path, "latest partial wal file from the old timeline to exist on primary");
+ #assert that 0000100004.partial.done exists on primary
+ wait_until_file_exists($node_primary->data_dir . $partial_done_file_path, "latest partial done file from the old timeline to exist on primary");
#assert that 0000100004 is still archived
wait_until_file_exists($latest_archived_wal_old_timeline, "latest wal file from the old timeline to be archived");
#partial wal file is still archived
wait_until_file_exists($archived_partial_wal_file, "latest partial wal file from the old timeline to be archived");
- #assert that 0000100004 does not exist on master
+ #assert that 0000100004 does not exist on primary
ok(!-f "$latest_wal_filepath_old_timeline", 'latest wal file from the old timeline should not exist on standby');
- #assert that 0000100004.done does not exist on master
- ok(!-f $node_master->data_dir . $latest_done_old_timeline, 'latest done file from the old timeline should not exist on master');
+ #assert that 0000100004.done does not exist on primary
+ ok(!-f $node_primary->data_dir . $latest_done_old_timeline, 'latest done file from the old timeline should not exist on primary');
}
@@ -243,11 +247,11 @@ sub check_history_files
# primary once the promotion of the standby completes. This ensures that
# the second standby created below will be able to restore this file,
# creating a RECOVERYHISTORY.
- my $primary_archive = $node_master->archive_dir;
+ my $primary_archive = $node_primary->archive_dir;
wait_until_file_exists("$primary_archive/00000002.history", "history file to be archived");
my $node_standby2 = get_new_node('standby2');
- $node_standby2->init_from_backup($node_master, $backup_name,
+ $node_standby2->init_from_backup($node_primary, $backup_name,
has_streaming => 1, has_restoring => 1);
$node_standby2->start;
@@ -269,4 +273,4 @@ sub check_history_files
"RECOVERYHISTORY removed after promotion");
ok( !-f "$node_standby2_data/pg_wal/RECOVERYXLOG",
"RECOVERYXLOG removed after promotion");
-}
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cloudberry.apache.org
For additional commands, e-mail: commits-help@cloudberry.apache.org