Commit 490483ed authored by Francesc Guasch's avatar Francesc Guasch
Browse files

wip(backend): deal with closed SSH connection

issue #1306
parent f939f781
......@@ -1798,7 +1798,7 @@ sub _remove_domain_cascade($self,$user, $cascade = 1) {
my $vm;
eval { $vm = Ravada::VM->open($instance->{id_vm}) };
die $@ if $@ && $@ !~ /I can't find VM/i;
next if !$vm->is_active;
next if !$vm || !$vm->is_active;
my $domain;
$@ = '';
eval { $domain = $vm->search_domain($domain_name) } if $vm;
......@@ -2490,7 +2490,7 @@ sub _around_is_active($orig, $self) {
};
if ( $@ ) {
return 0 if ref($@) && $@->code == 38; # broken pipe
return 0 if $@ =~ /can't connect/;
return 0 if $@ =~ /can't connect|error connecting/i;
die $@;
}
}
......@@ -3951,7 +3951,7 @@ sub _rsync_volumes_back($self, $request=undef) {
sub _pre_migrate($self, $node, $request = undef) {
die "Error: node not active" if !$node->is_active(1);
confess "Error: node not active" if !$node->is_active(1);
$self->_check_equal_storage_pools($node) if $self->_vm->is_active;
......
......@@ -71,9 +71,13 @@ sub _set_spice_ip($self, $password=undef, $listen_ip=$self->_vm->listen_ip) {
sub is_active {
my $self = shift;
my $ret = 0;
eval { $ret = $self->_value('is_active') };
eval {
$ret = $self->_value('is_active') ;
$ret = 0 if !defined $ret;
};
return $ret if !$@;
return 0 if $@ =~ /Error connecting/;
return 0 if $@ =~ /Error connecting|can't connect/;
warn $@;
die $@;
}
......
......@@ -137,6 +137,7 @@ around 'import_domain' => \&_around_import_domain;
around 'ping' => \&_around_ping;
around 'connect' => \&_around_connect;
after 'disconnect' => \&_post_disconnect;
#############################################################
#
......@@ -293,6 +294,16 @@ sub _around_connect($orig, $self) {
return $result;
}
sub _post_disconnect($self) {
if (!$self->is_local) {
if ($self->netssh) {
$self->netssh->disconnect();
}
$self->clear_netssh();
delete $SSH{$self->host};
}
}
sub _pre_create_domain {
_check_create_domain(@_);
_connect(@_);
......@@ -328,7 +339,7 @@ sub _connect_ssh($self) {
,batch_mode => 1
,forward_X11 => 0
,forward_agent => 0
,kill_ssh_on_timeout => 0
,kill_ssh_on_timeout => 1
);
last if !$ssh->error;
warn "RETRYING ssh ".$self->host." ".join(" ",$ssh->error);
......@@ -1081,6 +1092,10 @@ sub ping($self, $option=undef, $cache=1) {
return $ping;
}
sub _ping_nocache($self,$option=undef) {
return $self->ping($option,0);
}
sub _delete_cache($self, $key) {
$key = "_cache_$key";
delete $self->{$key};
......@@ -1176,6 +1191,9 @@ sub _do_is_active($self, $force=undef) {
}
$self->_cached_active($ret);
$self->_cached_active_time(time);
my $cache_key = "ping_".$self->host;
$self->_delete_cache($cache_key);
return $ret;
}
......@@ -1236,7 +1254,7 @@ sub run_command($self, @command) {
return $self->_run_command_local(@command) if $self->is_local();
my $ssh = $self->ssh or confess "Error: I can't connect to ".$self->host;
my $ssh = $self->ssh or confess "Error: Error connecting to ".$self->host;
my ($out, $err) = $ssh->capture2({timeout => 10},join " ",@command);
chomp $err if $err;
......
......@@ -1067,20 +1067,6 @@ sub _write_in_volumes($clone) {
}
}
sub _domain_node($node) {
my $vm = rvd_back->search_vm('KVM','localhost');
my $domain = $vm->search_domain($node->name);
$domain = rvd_back->import_domain(name => $node->name
,user => user_admin->name
,vm => 'KVM'
,spinoff_disks => 0
) if !$domain || !$domain->is_known;
ok($domain->id,"Expecting an ID for domain ".Dumper($domain)) or exit;
$domain->_set_vm($vm, 'force');
return $domain;
}
sub test_status($node) {
diag("[".$node->type."] testing domain status in front");
my ($base, $clone)= _create_clone($node);
......
......@@ -1230,10 +1230,15 @@ sub hibernate_node($node) {
$domain->shutdown_now(user_admin);
}
}
$node->disconnect;
my $domain_node = _domain_node($node);
$domain_node->hibernate( user_admin );
for (;;) {
$node->disconnect;
my $domain_node = _domain_node($node);
eval { $domain_node->hibernate( user_admin ) };
my $error = $@;
warn $error if $error;
last if !$error || $error =~ /is not active/;
}
my $max_wait = 30;
my $ping;
......@@ -1248,16 +1253,18 @@ sub hibernate_node($node) {
sub shutdown_node($node) {
if ($node->is_active) {
$node->run_command("service lightdm stop");
if ($node->_do_is_active(1)) {
eval {
$node->run_command("service lightdm stop");
$node->run_command("service gdm stop");
};
confess $@ if $@ && $@ !~ /ssh error|error connecting|control command failed/i;
for my $domain ($node->list_domains()) {
diag("Shutting down ".$domain->name." on node ".$node->name);
$domain->shutdown_now(user_admin);
}
}
$node->disconnect;
my $domain_node = _domain_node($node);
eval {
$domain_node->shutdown(user => user_admin);# if !$domain_node->is_active;
......@@ -1267,9 +1274,14 @@ sub shutdown_node($node) {
my $max_wait = 180;
for ( reverse 1 .. $max_wait ) {
last if !$node->ping(undef, 0);
diag("Waiting for node ".$node->name." to be inactive ... $_") if !($_ % 10);
if ( !($_ % 10) ) {
eval { $domain_node->shutdown(user => user_admin) };
warn $@ if $@;
diag("Waiting for node ".$node->name." to be inactive ... $_");
}
sleep 1;
}
$domain_node->shutdown_now(user_admin) if $domain_node->is_active;
is($node->ping(undef,0),0);
}
......@@ -1301,18 +1313,29 @@ sub start_node($node) {
is($node->ping('debug',0),1,"[".$node->type."] Expecting ping node ".$node->name) or exit;
for ( 1 .. 60 ) {
for my $try ( 1 .. 3) {
my $is_active;
eval {
$node->disconnect;
$node->clear_netssh();
$node->connect();
$is_active = $node->is_active(1)
};
warn $@ if $@;
for ( 1 .. 60 ) {
eval {
$node->disconnect;
$node->clear_netssh();
$node->connect();
$is_active = $node->is_active(1)
};
warn $@ if $@;
last if $is_active;
sleep 1;
diag("Waiting for active node ".$node->name." $_") if !($_ % 10);
}
last if $is_active;
sleep 1;
diag("Waiting for active node ".$node->name." $_") if !($_ % 10);
if ($try == 1 ) {
$domain->shutdown(user => user_admin);
sleep 2;
} elsif ( $try == 2 ) {
$domain->shutdown_now(user_admin);
sleep 2;
}
$domain->start(user => user_admin, remote_ip => '127.0.0.1');
}
is($node->_do_is_active,1,"Expecting active node ".$node->name) or exit;
......
......@@ -51,15 +51,16 @@ for my $vm_name ( 'KVM') {
is( $node->is_active, 1 );
$node->shutdown();
for ( 1 .. 60 ) {
last if !$node->ping;
sleep 1;
last if !$node->_ping_nocache();
}
is( $node->ping, 0 );
is( $node->is_active, 0 );
my $node2 = Ravada::VM->open(id => $node->id);
is( $node2->_ping_nocache(), 0 );
is( $node2->is_active(1), 0 );
# it actually dows nothing on virtual machines, just check it won't fail
eval { $node->_wake_on_lan() };
is($@,'');
is($@,'',"Expecting no error on wake on lan");
# KVM testing machines can't Wake On LAN
# sleep 1;
# $node->start();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment