You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vcl.apache.org by ar...@apache.org on 2014/03/05 19:03:11 UTC
svn commit: r1574592 - in /vcl/trunk/managementnode: bin/vcld
lib/VCL/DataStructure.pm lib/VCL/Module/Provisioning.pm
lib/VCL/Module/State.pm lib/VCL/inuse.pm
Author: arkurth
Date: Wed Mar 5 18:03:10 2014
New Revision: 1574592
URL: http://svn.apache.org/r1574592
Log:
VCL-734
Updated vcld to not check for cluster requests and to not update the request state to pending. The data returned from get_management_node_requests only includes reservations assigned to the running management node. Cluster reservations assigned to other MN's isn't included. Moved logic to State.pm.
Updated State.pm::initialize:
-Call reservation_failed if any of the supporting objects can't be created
-Added additional checks before a request is set to pending to detect if it was deleted or if any cluster child reservations have already failed.
Added State.pm::does_loadstate_exist_any_reservation. It is used to detect if any other cluster reservations have failed.
Moved steps which were in State.pm::DESTROY to State.pm::state_exit and added a call to state_exit in DESTROY. This ensures the steps always occur. Added a flag to state_exit so it is only run once. Added failed cluster reservation check. Added check before request state is updated to determine if it was updated by another process.
Added temporary condition to inuse.pm to skip all connection checks for cluster requests unless the previous state was reserved.
Added get_parent_reservation_id to DataStructure.pm
Modified:
vcl/trunk/managementnode/bin/vcld
vcl/trunk/managementnode/lib/VCL/DataStructure.pm
vcl/trunk/managementnode/lib/VCL/Module/Provisioning.pm
vcl/trunk/managementnode/lib/VCL/Module/State.pm
vcl/trunk/managementnode/lib/VCL/inuse.pm
Modified: vcl/trunk/managementnode/bin/vcld
URL: http://svn.apache.org/viewvc/vcl/trunk/managementnode/bin/vcld?rev=1574592&r1=1574591&r2=1574592&view=diff
==============================================================================
--- vcl/trunk/managementnode/bin/vcld (original)
+++ vcl/trunk/managementnode/bin/vcld Wed Mar 5 18:03:10 2014
@@ -196,8 +196,6 @@ sub main () {
my $request_preload = $info{request}{$request_id}{preload};
my @reservation_ids = sort { $a <=> $b } keys %{$info{request}{$request_id}{reservation}};
- my $reservation_count = scalar @reservation_ids;
- my $parent_reservation_id = min(@reservation_ids);
$ENV{request_id} = $request_id;
$ENV{reservation_id} = 0;
@@ -272,7 +270,7 @@ sub main () {
next RESERVATION;
}
else {
- notify($ERRORS{'WARNING'}, $LOGFILE, "reservation $reservation_id is already being processed");
+ notify($ERRORS{'DEBUG'}, $LOGFILE, "reservation $reservation_id is already being processed");
next RESERVATION;
}
}
@@ -280,37 +278,6 @@ sub main () {
notify($ERRORS{'DEBUG'}, $LOGFILE, "reservation $reservation_id is NOT already being processed");
}
- # Check if this is a child reservation of a cluster request
- # Child reservations must wait for the parent to start before starting
- if ($reservation_count > 1) {
- # Get the computerloadlog entries for the parent reservation
- my $request_loadstate_names = get_request_loadstate_names($request_id);
- for my $check_reservation_id (keys %$request_loadstate_names) {
- if (grep { $_ eq 'failed' } @{$request_loadstate_names->{$reservation_id}}) {
- notify($ERRORS{'DEBUG'}, $LOGFILE, "reservation will not be processed, another reservation in the cluster failed: $check_reservation_id");
- next REQUEST;
- }
- }
-
- if ($reservation_id ne $parent_reservation_id) {
- my @parent_computerloadstate_entries = @{$request_loadstate_names->{$parent_reservation_id}};
- if (@parent_computerloadstate_entries) {
- notify($ERRORS{'DEBUG'}, $LOGFILE, "retrieved computerloadlog entries for parent reservation: $parent_reservation_id:\n" . join("\n", @parent_computerloadstate_entries));
- if (grep {$_ eq 'begin'} @parent_computerloadstate_entries) {
- notify($ERRORS{'DEBUG'}, $LOGFILE, "processing may proceed, parent reservation $parent_reservation_id is waiting for child reservation processes to start");
- }
- else {
- notify($ERRORS{'WARNING'}, $LOGFILE, "processing delayed, computerloadlog 'begin' entry does not exist for parent reservation $parent_reservation_id:\n" . join("\n", @parent_computerloadstate_entries));
- next REQUEST;
- }
- }
- else {
- notify($ERRORS{'DEBUG'}, $LOGFILE, "processing delayed, parent reservation $parent_reservation_id process has not started yet");
- next REQUEST;
- }
- }
- }
-
# Get the full set of database data for this request
if (!%request_info) {
if (%request_info = get_request_info($request_id)) {
@@ -356,16 +323,11 @@ sub main () {
delete_computerloadlog_reservation($reservation_id, 'exited');
# Insert a computerloadlog 'begin' entry to indicate processing has begun for this reservation
+ # This should prevent multiple processes from being forked for a given reservation
+ # Do this before forking the new process because a cluster child will wait for the parent in State.pm::initialize
my $computer_id = $data_structure->get_computer_id();
insertloadlog($reservation_id, $computer_id, "begin", "beginning to process, state is $request_state_name");
-
- # Update state to pending for non-cluster reservations to limit another attempt at processing
- if ($reservation_count == 1) {
- # Update the request state to pending
- if (!update_request_state($request_id, "pending", $request_state_name)) {
- notify($ERRORS{'WARNING'}, 0, "failed to update request state to pending");
- }
- }
+
# Make a new child process, passing it the request/reservation info
make_new_child({request_info => \%request_info, data_structure => $data_structure});
} ## end foreach my $reservation_id (keys %{$info{request...
Modified: vcl/trunk/managementnode/lib/VCL/DataStructure.pm
URL: http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/DataStructure.pm?rev=1574592&r1=1574591&r2=1574592&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/DataStructure.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/DataStructure.pm Wed Mar 5 18:03:10 2014
@@ -1013,6 +1013,24 @@ sub get_reservation_ids {
#/////////////////////////////////////////////////////////////////////////////
+=head2 get_parent_reservation_id
+
+ Parameters : none
+ Returns : integer
+ Description : Returns the reservation ID for the parent reservation of a
+ cluster request.
+
+=cut
+
+sub get_parent_reservation_id {
+ my $self = shift;
+
+ # The parent reservation has the lowest ID
+ return min $self->get_reservation_ids();
+}
+
+#/////////////////////////////////////////////////////////////////////////////
+
=head2 is_parent_reservation
Parameters : None
@@ -1028,17 +1046,14 @@ sub is_parent_reservation {
my $self = shift;
my $reservation_id = $self->get_reservation_id();
- my @reservation_ids = $self->get_reservation_ids();
-
- # The parent reservation has the lowest ID
- my $parent_reservation_id = min @reservation_ids;
+ my $parent_reservation_id = $self->get_parent_reservation_id();
if ($reservation_id == $parent_reservation_id) {
- notify($ERRORS{'DEBUG'}, 0, "returning true: parent reservation ID for this request: $parent_reservation_id");
+ notify($ERRORS{'DEBUG'}, 0, "this is the parent reservation");
return 1;
}
else {
- notify($ERRORS{'DEBUG'}, 0, "returning false: parent reservation ID for this request: $parent_reservation_id");
+ notify($ERRORS{'DEBUG'}, 0, "this is a child reservation, parent reservation ID for this request: $parent_reservation_id");
return 0;
}
} ## end sub is_parent_reservation
Modified: vcl/trunk/managementnode/lib/VCL/Module/Provisioning.pm
URL: http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/Module/Provisioning.pm?rev=1574592&r1=1574591&r2=1574592&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/Module/Provisioning.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/Module/Provisioning.pm Wed Mar 5 18:03:10 2014
@@ -331,11 +331,11 @@ sub retrieve_image {
# Make sure the parent image repository path exists
my ($image_repository_directory_name, $image_repository_parent_directory_path) = fileparse($image_repository_path_local, qr/\.[^\\\/]*/);
if (!$image_repository_parent_directory_path) {
- notify($ERRORS{'WARNING'}, 0, "unable to retrieve image, unable to determine parent directory of local image repository path: $image_repository_path_local");
+ notify($ERRORS{'WARNING'}, 0, "unable to retrieve image from another management node because the path specified as the VM host profile repository could not be parsed: $image_repository_path_local");
return;
}
elsif (!$self->mn_os->file_exists($image_repository_parent_directory_path)) {
- notify($ERRORS{'WARNING'}, 0, "unable to retrieve image, local image repository parent path does not exist: $image_repository_parent_directory_path");
+ notify($ERRORS{'WARNING'}, 0, "unable to retrieve image from another management node because the path on this management node where the image files are to be copied does not exist, the path specified as the VM host profile repository path is: $image_repository_parent_directory_path, either a directory, mount point, or symbolic link must exist on this management node in this location");
return;
}
Modified: vcl/trunk/managementnode/lib/VCL/Module/State.pm
URL: http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/Module/State.pm?rev=1574592&r1=1574591&r2=1574592&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/Module/State.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/Module/State.pm Wed Mar 5 18:03:10 2014
@@ -102,7 +102,7 @@ sub initialize {
notify($ERRORS{'DEBUG'}, 0, "obtained a database handle for this state process, stored as \$ENV{dbh}");
}
else {
- notify($ERRORS{'WARNING'}, 0, "unable to obtain a database handle for this state process");
+ notify($ERRORS{'CRITICAL'}, 0, "unable to obtain a database handle for this state process");
return;
}
@@ -128,8 +128,7 @@ sub initialize {
$self->set_os($os);
}
else {
- notify($ERRORS{'WARNING'}, 0, "failed to create OS object");
- return;
+ $self->reservation_failed("failed to create OS object");
}
# Create a VM host OS object if vmhostid is set for the computer
@@ -137,8 +136,7 @@ sub initialize {
if ($is_vm) {
$vmhost_os = $self->create_vmhost_os_object();
if (!$vmhost_os) {
- notify($ERRORS{'WARNING'}, 0, "failed to create VM host OS object");
- return;
+ $self->reservation_failed("failed to create VM host OS object");
}
$self->set_vmhost_os($vmhost_os);
}
@@ -155,8 +153,7 @@ sub initialize {
$self->os->set_provisioner($self->provisioner());
}
else {
- notify($ERRORS{'WARNING'}, 0, "failed to create provisioning object");
- return;
+ $self->reservation_failed("failed to create provisioning object");
}
# Create a VM host OS object if vmhostid is set for the computer
@@ -169,24 +166,36 @@ sub initialize {
}
}
+ # If this is a cluster request, wait for all reservations to begin before proceeding
+ if ($reservation_count > 1) {
+ if (!$self->wait_for_all_reservations_to_begin('begin', 90, 5)) {
+ $self->reservation_failed("failed to detect start of processing for all reservation processes");
+ }
+ }
+
# Parent reservation needs to update the request state to pending
if ($is_parent_reservation) {
- # Check if this is a cluster request - don't update the request state until all child reservation processes have started
- # Otherwise, child reservations assigned to other management nodes won't launch
+ if (is_request_deleted($request_id)) {
+ notify($ERRORS{'OK'}, 0, "request has been deleted, exiting");
+ $self->state_exit();
+ }
+
if ($reservation_count > 1) {
- # Wait for all child processes to begin
- if (!$self->wait_for_child_reservations_to_begin('begin', 60, 3)) {
- $self->reservation_failed("child reservation processes failed to begin");
+ # Check if any reservations have failed
+ if (my @failed_reservation_ids = $self->does_loadstate_exist_any_reservation('failed')) {
+ notify($ERRORS{'WARNING'}, 0, "reservations failed: " . join(', ', @failed_reservation_ids));
+ $self->state_exit('failed');
}
+ }
-
- # Update the request state to pending for this cluster reservation.
- # Single reservations are set to pending in vcld
- if (!update_request_state($request_id, "pending", $request_state_name)) {
- notify($ERRORS{'WARNING'}, 0, "failed to update request state to pending");
- }
+ # Update the request state to pending for this reservation
+ if (!update_request_state($request_id, "pending", $request_state_name)) {
+ $self->reservation_failed("failed to update request state to pending");
}
}
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "child reservation, not updating request state to 'pending'");
+ }
return 1;
} ## end sub initialize
@@ -234,7 +243,7 @@ sub reservation_failed {
my $request_state_name = $self->data->get_request_state_name();
my $request_laststate_name = $self->data->get_request_laststate_name();
my $computer_state_name = $self->data->get_computer_state_name();
-
+
# Check if the request has been deleted
if (is_request_deleted($request_id)) {
notify($ERRORS{'OK'}, 0, "request has been deleted, setting computer state to available and exiting");
@@ -251,7 +260,7 @@ sub reservation_failed {
else {
notify($ERRORS{'WARNING'}, 0, "computer $computer_short_name ($computer_id) state NOT set to available because the current state is $computer_state_name");
}
-
+
notify($ERRORS{'OK'}, 0, "exiting 0");
exit 0;
} ## end if (is_request_deleted($request_id))
@@ -263,15 +272,27 @@ sub reservation_failed {
# Display the message
notify($ERRORS{'CRITICAL'}, 0, "reservation failed on $computer_short_name: $message");
-
- # Insert a row into the computerloadlog table
- if (insertloadlog($reservation_id, $computer_id, "failed", $message)) {
- notify($ERRORS{'OK'}, 0, "inserted computerloadlog entry");
+
+ my $new_request_state_name;
+ my $new_computer_state_name;
+ if ($request_state_name eq 'image') {
+ $new_request_state_name = 'maintenance';
+ $new_computer_state_name = 'maintenance';
}
else {
- notify($ERRORS{'WARNING'}, 0, "failed to insert computerloadlog entry");
+ $new_request_state_name = 'failed';
+ $new_computer_state_name = 'failed';
}
+ # Update the request state to failed
+ # Don't check if parent reservation - allow child reservation to change state
+ # Otherwise, multiple failed attempts may be made
+ if (update_request_state($request_id, $new_request_state_name, $request_state_name)) {
+ notify($ERRORS{'OK'}, 0, "set request state to $new_request_state_name/$request_state_name");
+ }
+ else {
+ notify($ERRORS{'WARNING'}, 0, "unable to set request to $new_request_state_name/$request_state_name");
+ }
if ($request_state_name =~ /^(new|reserved|inuse|image)/){
# Update log table ending column to failed for this request
@@ -282,26 +303,26 @@ sub reservation_failed {
notify($ERRORS{'WARNING'}, 0, "failed to update log ending value to 'failed', logid=$request_logid");
}
}
-
+
+ # Insert a row into the computerloadlog table
+ if (insertloadlog($reservation_id, $computer_id, "failed", $message)) {
+ notify($ERRORS{'OK'}, 0, "inserted computerloadlog 'failed' entry for reservation $reservation_id");
+ }
+ else {
+ notify($ERRORS{'WARNING'}, 0, "failed to insert computerloadlog entry");
+ }
+
# Update the computer state to failed as long as it's not currently maintenance
if ($computer_state_name !~ /^(maintenance)/){
- if (update_computer_state($computer_id, "failed")) {
- notify($ERRORS{'OK'}, 0, "computer $computer_short_name ($computer_id) state set to failed");
+ if (update_computer_state($computer_id, $new_computer_state_name)) {
+ notify($ERRORS{'OK'}, 0, "computer $computer_short_name ($computer_id) state set to $new_computer_state_name");
}
else {
- notify($ERRORS{'WARNING'}, 0, "unable to set computer $computer_short_name ($computer_id) state to failed");
+ notify($ERRORS{'WARNING'}, 0, "unable to set computer $computer_short_name ($computer_id) state to $new_computer_state_name");
}
}
else {
- notify($ERRORS{'WARNING'}, 0, "computer $computer_short_name ($computer_id) state NOT set to failed because the current state is $computer_state_name");
- }
-
- # Update the request state to failed
- if (update_request_state($request_id, "failed", $request_laststate_name)) {
- notify($ERRORS{'OK'}, 0, "set request state to 'failed'/'$request_laststate_name'");
- }
- else {
- notify($ERRORS{'WARNING'}, 0, "unable to set request to 'failed'/'$request_laststate_name'");
+ notify($ERRORS{'WARNING'}, 0, "computer $computer_short_name ($computer_id) state NOT set to $new_computer_state_name because the current state is $computer_state_name");
}
# Check if computer is part of a blockrequest, if so pull out of blockcomputers table
@@ -324,7 +345,7 @@ sub reservation_failed {
#/////////////////////////////////////////////////////////////////////////////
-=head2 does_loadstate_entry_exist
+=head2 does_loadstate_exist_all_reservations
Parameters : $loadstate_name, $ignore_current_reservation (optional)
Returns : boolean
@@ -338,7 +359,7 @@ sub reservation_failed {
=cut
-sub does_loadstate_entry_exist {
+sub does_loadstate_exist_all_reservations {
my $self = shift;
if (ref($self) !~ /VCL/) {
notify($ERRORS{'CRITICAL'}, 0, "subroutine can only be called as a class method of a VCL object");
@@ -366,7 +387,6 @@ sub does_loadstate_entry_exist {
my @exists;
my @does_not_exist;
- my @failed;
for my $check_reservation_id (keys %$request_loadstate_names) {
# Ignore the current reservation
if ($ignore_current_reservation && $check_reservation_id eq $reservation_id) {
@@ -380,16 +400,6 @@ sub does_loadstate_entry_exist {
else {
push @does_not_exist, $check_reservation_id;
}
-
- if (grep { $_ eq 'failed' } @loadstate_names) {
- push @failed, $check_reservation_id;
- }
- }
-
- # Check if any child reservations failed
- if (@failed) {
- $self->reservation_failed("child reservation process failed: " . join(', ', @failed));
- return;
}
if (@does_not_exist) {
@@ -407,7 +417,77 @@ sub does_loadstate_entry_exist {
#/////////////////////////////////////////////////////////////////////////////
-=head2 wait_for_child_reservations_to_begin
+=head2 does_loadstate_exist_any_reservation
+
+ Parameters : $loadstate_name, $ignore_current_reservation (optional)
+ Returns : array or integer
+ Description : Checks the computerloadlog entries for all reservations belonging
+ to the request. An array is returned containing reservation IDs
+ of any reservations for which have a corresponding
+ computerloadlog $loadstate_name entry. The
+ $ignore_current_reservation argument may be used to check all
+ reservations other than the one currently being processed.
+
+=cut
+
+sub does_loadstate_exist_any_reservation {
+ my $self = shift;
+ if (ref($self) !~ /VCL/) {
+ notify($ERRORS{'CRITICAL'}, 0, "subroutine can only be called as a class method of a VCL object");
+ return;
+ }
+
+ my $loadstate_name = shift;
+ if (!defined($loadstate_name)) {
+ notify($ERRORS{'WARNING'}, 0, "computerloadlog loadstate name argument was not supplied");
+ return;
+ }
+
+ my $ignore_current_reservation = shift;
+
+ my $request_id = $self->data->get_request_id();
+ my $request_state = $self->data->get_request_state_name();
+ my $reservation_id = $self->data->get_reservation_id();
+
+ # Retrieve computerloadlog entries for all reservations
+ my $request_loadstate_names = get_request_loadstate_names($request_id);
+ if (!$request_loadstate_names) {
+ notify($ERRORS{'WARNING'}, 0, "failed to retrieve request loadstate names");
+ return;
+ }
+
+ my @exists;
+ my @does_not_exist;
+ for my $check_reservation_id (keys %$request_loadstate_names) {
+ # Ignore the current reservation
+ if ($ignore_current_reservation && $check_reservation_id eq $reservation_id) {
+ next;
+ }
+
+ my @loadstate_names = @{$request_loadstate_names->{$check_reservation_id}};
+ if (grep { $_ eq $loadstate_name } @loadstate_names) {
+ push @exists, $check_reservation_id;
+ }
+ else {
+ push @does_not_exist, $check_reservation_id;
+ }
+ }
+
+ if (@exists) {
+ notify($ERRORS{'DEBUG'}, 0, "computerloadlog '$loadstate_name' entry exists for reservation:\n" .
+ "exists for reservation IDs: " . join(', ', @exists) . "\n" .
+ "does not exist for reservation IDs: " . join(', ', @does_not_exist)
+ );
+ }
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "computerloadlog '$loadstate_name' entry does NOT exist for any reservation");
+ }
+ return (wantarray) ? @exists : scalar(@exists);
+}
+
+#/////////////////////////////////////////////////////////////////////////////
+
+=head2 wait_for_all_reservations_to_begin
Parameters : $loadstate_name (optional), $total_wait_seconds (optional), $attempt_delay_seconds (optional)
Returns : boolean
@@ -420,7 +500,7 @@ sub does_loadstate_entry_exist {
=cut
-sub wait_for_child_reservations_to_begin {
+sub wait_for_all_reservations_to_begin {
my $self = shift;
if (ref($self) !~ /VCL/) {
notify($ERRORS{'CRITICAL'}, 0, "subroutine can only be called as a class method of a VCL object");
@@ -446,10 +526,10 @@ sub wait_for_child_reservations_to_begin
exit;
}
- return $self->does_loadstate_entry_exist($loadstate_name, 1);
+ return $self->does_loadstate_exist_all_reservations($loadstate_name, 1);
},
[],
- "waiting for child reservation processes to begin", $total_wait_seconds, $attempt_delay_seconds
+ "waiting for all reservation processes to begin", $total_wait_seconds, $attempt_delay_seconds
);
}
@@ -480,7 +560,7 @@ sub wait_for_child_reservations_to_exit
my $request_state_name = $self->data->get_request_state_name();
return $self->code_loop_timeout(
- \&does_loadstate_entry_exist,
+ \&does_loadstate_exist_all_reservations,
[$self, 'exited', 1],
"waiting for child reservation processes to exit", $total_wait_seconds, $attempt_delay_seconds
);
@@ -504,8 +584,15 @@ sub state_exit {
return;
}
+ # Set flag to avoid this subroutine from being called more than once
+ $ENV{state_exit} = 1;
+
my ($request_state_name_new, $computer_state_name_new, $request_log_ending) = @_;
+ notify($ERRORS{'DEBUG'}, 0, "beginning state module exit tasks");
+
+ my $calling_sub = get_calling_subroutine();
+
my $request_id = $self->data->get_request_id();
my $request_logid = $self->data->get_request_log_id();
my $reservation_id = $self->data->get_reservation_id();
@@ -515,56 +602,75 @@ sub state_exit {
my $request_state_name_old = $self->data->get_request_state_name();
my $request_laststate_name_old = $self->data->get_request_laststate_name();
my $computer_id = $self->data->get_computer_id();
- my $computer_state_name_old = $self->data->get_computer_state_name();
my $computer_shortname = $self->data->get_computer_short_name();
-
- # If new request state name argument was not supplied, set it back to the previous state
- if (!$request_state_name_new) {
- $request_state_name_new = $request_state_name_old;
- }
-
+
if ($is_parent_reservation) {
# If parent of a cluster request, wait for child processes to exit before switching the state
if ($reservation_count > 1) {
$self->wait_for_child_reservations_to_exit();
}
- # Never set request state to failed if previous state is image
- if ($request_state_name_old eq 'image' && $request_state_name_new !~ /(complete|maintenance)/) {
- notify($ERRORS{'CRITICAL'}, 0, "previous request state is $request_state_name_old, not setting request state to $request_state_name_new, setting request and computer state to maintenance");
- $request_state_name_new = 'maintenance';
- $computer_state_name_new = 'maintenance';
- }
- elsif ($request_state_name_old eq 'inuse' && $request_state_name_new !~ /(inuse|timeout|maintenance)/) {
- notify($ERRORS{'CRITICAL'}, 0, "previous request state is $request_state_name_old, not setting request state to $request_state_name_new, setting request and computer state to inuse");
- $request_state_name_new = 'inuse';
- $computer_state_name_new = 'inuse';
- }
-
- # Update the reservation.lastcheck time to now if the next request state is inuse
- # Do this to ensure that reservations are not processed again quickly after this process exits
- # For cluster requests, the parent may have had to wait a while for child processes to exit
- # Resetting reservation.lastcheck causes reservations to wait the full interval between inuse checks
- if ($request_state_name_new =~ /(inuse)/) {
- update_reservation_lastcheck(@reservation_ids);
- }
-
- # Update the request state
- if (!is_request_deleted($request_id)) {
- notify($ERRORS{'OK'}, 0, "request has NOT been deleted, updating new state: $request_state_name_new old state: $request_state_name_old");
- if (!update_request_state($request_id, $request_state_name_new, $request_state_name_old)) {
- notify($ERRORS{'WARNING'}, 0, "failed to change request state: $request_state_name_old/$request_laststate_name_old --> $request_state_name_new/$request_state_name_old");
+ # Check if any reservations failed
+ if (!$request_state_name_new || $request_state_name_new ne 'failed') {
+ if ($self->does_loadstate_exist_any_reservation('failed')) {
+ notify($ERRORS{'OK'}, 0, "another reservation failed, request state will be updated to 'failed'");
+ $request_state_name_new = 'failed';
}
}
-
+
+ if ($request_state_name_new) {
+ # Never set request state to failed if previous state is image
+ if ($request_state_name_old eq 'image' && $request_state_name_new !~ /(complete|maintenance)/) {
+ notify($ERRORS{'CRITICAL'}, 0, "previous request state is $request_state_name_old, not setting request state to $request_state_name_new, setting request and computer state to maintenance");
+ $request_state_name_new = 'maintenance';
+ $computer_state_name_new = 'maintenance';
+ }
+ elsif ($request_state_name_old eq 'inuse' && $request_state_name_new !~ /(inuse|timeout|maintenance)/) {
+ notify($ERRORS{'CRITICAL'}, 0, "previous request state is $request_state_name_old, not setting request state to $request_state_name_new, setting request and computer state to inuse");
+ $request_state_name_new = 'inuse';
+ $computer_state_name_new = 'inuse';
+ }
+
+ # Update the reservation.lastcheck time to now if the next request state is inuse
+ # Do this to ensure that reservations are not processed again quickly after this process exits
+ # For cluster requests, the parent may have had to wait a while for child processes to exit
+ # Resetting reservation.lastcheck causes reservations to wait the full interval between inuse checks
+ if ($request_state_name_new =~ /(inuse)/) {
+ update_reservation_lastcheck(@reservation_ids);
+ }
+
+ # Update the request state
+ if (!is_request_deleted($request_id)) {
+ # Check if the request state has already been updated
+ # This can occur if another reservation in a cluster failed
+ my ($request_state_name_current, $request_laststate_name_current) = get_request_current_state_name($request_id);
+ if ($request_state_name_current eq $request_state_name_new && $request_laststate_name_current eq $request_state_name_old) {
+ notify($ERRORS{'OK'}, 0, "request has NOT been deleted, current state already set to: $request_state_name_current/$request_laststate_name_current");
+ }
+ else {
+ notify($ERRORS{'OK'}, 0, "request has NOT been deleted, updating request state: $request_state_name_old/$request_laststate_name_old --> $request_state_name_new/$request_state_name_old");
+ if (!update_request_state($request_id, $request_state_name_new, $request_state_name_old)) {
+ notify($ERRORS{'WARNING'}, 0, "failed to change request state: $request_state_name_old/$request_laststate_name_old --> $request_state_name_new/$request_state_name_old");
+ }
+ }
+ }
+ }
+
+ # Delete all computerloadlog rows with loadstatename = 'begin' for all reservations in this request
+ delete_computerloadlog_reservation(\@reservation_ids, 'begin');
+
# Update log.ending if this is the parent reservation and argument was supplied
- if ($request_log_ending && !update_log_ending($request_logid, $request_log_ending)) {
- notify($ERRORS{'CRITICAL'}, 0, "failed to set log ending to $request_log_ending, log ID: $request_logid");
+ if ($request_log_ending) {
+ if (!update_log_ending($request_logid, $request_log_ending)) {
+ notify($ERRORS{'CRITICAL'}, 0, "failed to set log ending to $request_log_ending, log ID: $request_logid");
+ }
}
}
# Update the computer state if argument was supplied
if ($computer_state_name_new) {
+ my $computer_state_name_old = $self->data->get_computer_state_name();
+
if ($computer_state_name_new eq $computer_state_name_old) {
notify($ERRORS{'DEBUG'}, 0, "state of computer $computer_shortname not updated, already set to $computer_state_name_old");
}
@@ -576,6 +682,20 @@ sub state_exit {
# Insert a computerloadlog 'exited' entry
# This is used by the parent cluster reservation
insertloadlog($reservation_id, $computer_id, "exited", "vcld process exiting");
+
+ # Don't call exit if this was called from DESTROY or else DESTROY gets called again
+ if ($calling_sub) {
+ if ($calling_sub =~ /DESTROY/) {
+ notify($ERRORS{'DEBUG'}, 0, "calling subroutine: $calling_sub, skipping call to exit");
+ return;
+ }
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "calling subroutine: $calling_sub, calling exit");
+ }
+ }
+ else {
+ notify($ERRORS{'DEBUG'}, 0, "calling subroutine not defined, calling exit");
+ }
exit;
}
@@ -599,67 +719,48 @@ sub DESTROY {
my $address = sprintf('%x', $self);
notify($ERRORS{'DEBUG'}, 0, ref($self) . " destructor called, address: $address");
- # If not a blockrequest, delete computerloadlog entry
- if ($self && $self->data && !$self->data->is_blockrequest()) {
- my $request_id = $self->data->get_request_id();
- my @reservation_ids = $self->data->get_reservation_ids();
- my $is_parent_reservation = $self->data->is_parent_reservation();
-
- if (@reservation_ids && $request_id) {
- if ($is_parent_reservation) {
- # Delete all computerloadlog rows with loadstatename = 'begin' for all reservations in this request
- delete_computerloadlog_reservation(\@reservation_ids, 'begin');
- get_request_loadstate_names($request_id);
+ my $calling_sub = get_calling_subroutine();
+
+ # Check if normal module object data is available
+ if ($calling_sub && $self && $self->data(0) && !$self->data->is_blockrequest()) {
+ if (!$ENV{state_exit}) {
+ my $request_id = $self->data->get_request_id();
+ my @reservation_ids = $self->data->get_reservation_ids();
+ if (@reservation_ids && $request_id) {
+ $self->state_exit();
}
- else {
- notify($ERRORS{'DEBUG'}, 0, "child reservation, computerloadlog 'begin' entries not removed");
+ elsif (!$SETUP_MODE) {
+ notify($ERRORS{'WARNING'}, 0, "failed to retrieve the reservation ID, computerloadlog 'begin' rows not removed");
}
}
- elsif (!$SETUP_MODE) {
- notify($ERRORS{'WARNING'}, 0, "failed to retrieve the reservation ID, computerloadlog 'begin' rows not removed");
- }
}
+ # Uncomment to enable database metrics
# Print the number of database handles this process created for testing/development
- if (defined $ENV{dbh_count}) {
- #notify($ERRORS{'DEBUG'}, 0, "number of database handles state process created: $ENV{dbh_count}");
- }
- else {
- notify($ERRORS{'DEBUG'}, 0, "state process created unknown number of database handles, \$ENV{dbh_count} is undefined");
- }
-
- if (defined $ENV{database_select_count}) {
- #notify($ERRORS{'DEBUG'}, 0, "database select queries: $ENV{database_select_count}");
- }
-
- if (defined $ENV{database_select_calls}) {
- my $database_select_calls_string;
- my %hash = %{$ENV{database_select_calls}};
- my @sorted_keys = sort { $hash{$b} <=> $hash{$a} } keys(%hash);
- for my $key (@sorted_keys) {
- $database_select_calls_string .= "$ENV{database_select_calls}{$key}: $key\n";
- }
-
- #notify($ERRORS{'DEBUG'}, 0, "database select called from:\n$database_select_calls_string");
- }
-
- if (defined $ENV{database_execute_count}) {
- #notify($ERRORS{'DEBUG'}, 0, "database execute queries: $ENV{database_execute_count}");
- }
+ #if (defined $ENV{dbh_count}) {
+ # notify($ERRORS{'DEBUG'}, 0, "number of database handles state process created: $ENV{dbh_count}");
+ #}
+ #if (defined $ENV{database_select_count}) {
+ # notify($ERRORS{'DEBUG'}, 0, "database select queries: $ENV{database_select_count}");
+ #}
+ #if (defined $ENV{database_select_calls}) {
+ # my $database_select_calls_string;
+ # my %hash = %{$ENV{database_select_calls}};
+ # my @sorted_keys = sort { $hash{$b} <=> $hash{$a} } keys(%hash);
+ # for my $key (@sorted_keys) {
+ # $database_select_calls_string .= "$ENV{database_select_calls}{$key}: $key\n";
+ # }
+ # notify($ERRORS{'DEBUG'}, 0, "database select called from:\n$database_select_calls_string");
+ #}
+ #if (defined $ENV{database_execute_count}) {
+ # notify($ERRORS{'DEBUG'}, 0, "database execute queries: $ENV{database_execute_count}");
+ #}
# Close the database handle
if (defined $ENV{dbh}) {
- #notify($ERRORS{'DEBUG'}, 0, "process has a database handle stored in \$ENV{dbh}, attempting disconnect");
-
- if ($ENV{dbh}->disconnect) {
- #notify($ERRORS{'DEBUG'}, 0, "\$ENV{dbh}: database disconnect successful");
- }
- else {
+ if (!$ENV{dbh}->disconnect) {
notify($ERRORS{'WARNING'}, 0, "\$ENV{dbh}: database disconnect failed, " . DBI::errstr());
}
- } ## end if (defined $ENV{dbh})
- else {
- #notify($ERRORS{'DEBUG'}, 0, "process does not have a database handle stored in \$ENV{dbh}");
}
# Check for an overridden destructor
Modified: vcl/trunk/managementnode/lib/VCL/inuse.pm
URL: http://svn.apache.org/viewvc/vcl/trunk/managementnode/lib/VCL/inuse.pm?rev=1574592&r1=1574591&r2=1574592&view=diff
==============================================================================
--- vcl/trunk/managementnode/lib/VCL/inuse.pm (original)
+++ vcl/trunk/managementnode/lib/VCL/inuse.pm Wed Mar 5 18:03:10 2014
@@ -296,6 +296,12 @@ sub process {
$self->os->firewall_compare_update();
}
+ # TODO: fix user connection checking for cluster requests
+ if ($reservation_count > 1 && $request_laststate_name ne 'reserved') {
+ notify($ERRORS{'OK'}, 0, "skipping user connection check for cluster request");
+ $self->state_exit('inuse', 'inuse');
+ }
+
# Check to see if user is connected. user_connected will true(1) for servers and requests > 24 hours
if (!$self->code_loop_timeout(sub{$self->user_connected()}, [], "waiting for user to connect to $computer_short_name", ($connect_timeout_minutes*60), 15)) {
if (!$imagemeta_checkuser || !$request_checkuser) {
@@ -381,14 +387,14 @@ sub user_connected {
# Check if this is a server request, causes process to exit if server request
if ($server_request_id) {
- notify($ERRORS{'DEBUG'}, 0, "Server reservation detected, set as user is connected");
- insertloadlog($reservation_id, $computer_id, "connected", "user connected to $computer_short_name");
- return 1;
+ notify($ERRORS{'DEBUG'}, 0, "server reservation detected, set as user is connected");
+ insertloadlog($reservation_id, $computer_id, "connected", "user connected to $computer_short_name");
+ return 1;
}
# If duration is >= 24 hrs set as connected and return
if($request_duration_hrs >= 24 ) {
- notify($ERRORS{'OK'}, 0, "Duration is $request_duration_hrs hrs . Is >= to 24 hrs, skipping inuse checks");
+ notify($ERRORS{'OK'}, 0, "duration is $request_duration_hrs hrs . Is >= to 24 hrs, skipping inuse checks");
insertloadlog($reservation_id, $computer_id, "connected", "user connected to $computer_short_name");
return 1;
}