wip(backend): deal with closed SSH connection

issue #1306

lib/Ravada/Domain.pm

@@ -1798,7 +1798,7 @@ sub _remove_domain_cascade($self,$user, $cascade = 1) {
         my $vm;
         eval { $vm = Ravada::VM->open($instance->{id_vm}) };
         die $@ if $@ && $@ !~ /I can't find VM/i;
-        next if !$vm->is_active;
+        next if !$vm || !$vm->is_active;
         my $domain;
         $@ = '';
         eval { $domain = $vm->search_domain($domain_name) } if $vm;
@@ -2490,7 +2490,7 @@ sub _around_is_active($orig, $self) {
         };
         if ( $@ ) {
             return 0 if ref($@) && $@->code == 38; # broken pipe
-            return 0 if $@ =~ /can't connect/;
+            return 0 if $@ =~ /can't connect|error connecting/i;
             die $@;
         }
     }
@@ -3951,7 +3951,7 @@ sub _rsync_volumes_back($self, $request=undef) {
 
 sub _pre_migrate($self, $node, $request = undef) {
 
-    die "Error: node not active" if !$node->is_active(1);
+    confess "Error: node not active" if !$node->is_active(1);
 
     $self->_check_equal_storage_pools($node) if $self->_vm->is_active;
 

lib/Ravada/Domain/Void.pm

@@ -71,9 +71,13 @@ sub _set_spice_ip($self, $password=undef, $listen_ip=$self->_vm->listen_ip) {
 sub is_active {
     my $self = shift;
     my $ret = 0;
-    eval { $ret = $self->_value('is_active') };
+    eval {
+        $ret = $self->_value('is_active') ;
+        $ret = 0 if !defined $ret;
+    };
     return $ret if !$@;
-    return 0 if $@ =~ /Error connecting/;
+    return 0 if $@ =~ /Error connecting|can't connect/;
+    warn $@;
     die $@;
 }
 

lib/Ravada/VM.pm

@@ -137,6 +137,7 @@ around 'import_domain' => \&_around_import_domain;
 
 around 'ping' => \&_around_ping;
 around 'connect' => \&_around_connect;
+after 'disconnect' => \&_post_disconnect;
 
 #############################################################
 #
@@ -293,6 +294,16 @@ sub _around_connect($orig, $self) {
     return $result;
 }
 
+sub _post_disconnect($self) {
+    if (!$self->is_local) {
+        if ($self->netssh) {
+            $self->netssh->disconnect();
+	    }
+        $self->clear_netssh();
+        delete $SSH{$self->host};
+    }
+}
+
 sub _pre_create_domain {
     _check_create_domain(@_);
     _connect(@_);
@@ -328,7 +339,7 @@ sub _connect_ssh($self) {
                  ,batch_mode => 1
                 ,forward_X11 => 0
               ,forward_agent => 0
-        ,kill_ssh_on_timeout => 0
+        ,kill_ssh_on_timeout => 1
             );
             last if !$ssh->error;
             warn "RETRYING ssh ".$self->host." ".join(" ",$ssh->error);
@@ -1081,6 +1092,10 @@ sub ping($self, $option=undef, $cache=1) {
     return $ping;
 }
 
+sub _ping_nocache($self,$option=undef) {
+    return $self->ping($option,0);
+}
+
 sub _delete_cache($self, $key) {
     $key = "_cache_$key";
     delete $self->{$key};
@@ -1176,6 +1191,9 @@ sub _do_is_active($self, $force=undef) {
     }
     $self->_cached_active($ret);
     $self->_cached_active_time(time);
+
+    my $cache_key = "ping_".$self->host;
+    $self->_delete_cache($cache_key);
     return $ret;
 }
 
@@ -1236,7 +1254,7 @@ sub run_command($self, @command) {
 
     return $self->_run_command_local(@command) if $self->is_local();
 
-    my $ssh = $self->ssh or confess "Error: I can't connect to ".$self->host;
+    my $ssh = $self->ssh or confess "Error: Error connecting to ".$self->host;
 
     my ($out, $err) = $ssh->capture2({timeout => 10},join " ",@command);
     chomp $err if $err;

t/kvm/n10_nodes.t

@@ -1067,20 +1067,6 @@ sub _write_in_volumes($clone) {
     }
 }
 
-sub _domain_node($node) {
-    my $vm = rvd_back->search_vm('KVM','localhost');
-    my $domain = $vm->search_domain($node->name);
-    $domain = rvd_back->import_domain(name => $node->name
-            ,user => user_admin->name
-            ,vm => 'KVM'
-            ,spinoff_disks => 0
-    )   if !$domain || !$domain->is_known;
-
-    ok($domain->id,"Expecting an ID for domain ".Dumper($domain)) or exit;
-    $domain->_set_vm($vm, 'force');
-    return $domain;
-}
-
 sub test_status($node) {
     diag("[".$node->type."] testing domain status in front");
     my ($base, $clone)= _create_clone($node);

t/lib/Test/Ravada.pm

@@ -1230,10 +1230,15 @@ sub hibernate_node($node) {
             $domain->shutdown_now(user_admin);
         }
     }
-    $node->disconnect;
 
-    my $domain_node = _domain_node($node);
-    $domain_node->hibernate( user_admin );
+    for (;;) {
+        $node->disconnect;
+        my $domain_node = _domain_node($node);
+        eval { $domain_node->hibernate( user_admin ) };
+        my $error = $@;
+        warn $error if $error;
+        last if !$error || $error =~ /is not active/;
+    }
 
     my $max_wait = 30;
     my $ping;
@@ -1248,16 +1253,18 @@ sub hibernate_node($node) {
 
 sub shutdown_node($node) {
 
-    if ($node->is_active) {
-        $node->run_command("service lightdm stop");
+    if ($node->_do_is_active(1)) {
+        eval {
+		$node->run_command("service lightdm stop");
         $node->run_command("service gdm stop");
+	};
+	confess $@ if $@ && $@ !~ /ssh error|error connecting|control command failed/i;
         for my $domain ($node->list_domains()) {
             diag("Shutting down ".$domain->name." on node ".$node->name);
             $domain->shutdown_now(user_admin);
         }
     }
     $node->disconnect;
-
     my $domain_node = _domain_node($node);
     eval {
         $domain_node->shutdown(user => user_admin);# if !$domain_node->is_active;
@@ -1267,9 +1274,14 @@ sub shutdown_node($node) {
     my $max_wait = 180;
     for ( reverse 1 .. $max_wait ) {
         last if !$node->ping(undef, 0);
-        diag("Waiting for node ".$node->name." to be inactive ... $_")  if !($_ % 10);
+        if ( !($_ % 10) ) {
+            eval { $domain_node->shutdown(user => user_admin) };
+            warn $@ if $@;
+            diag("Waiting for node ".$node->name." to be inactive ... $_");
+        }
         sleep 1;
     }
+    $domain_node->shutdown_now(user_admin) if $domain_node->is_active;
     is($node->ping(undef,0),0);
 }
 
@@ -1301,18 +1313,29 @@ sub start_node($node) {
 
     is($node->ping('debug',0),1,"[".$node->type."] Expecting ping node ".$node->name) or exit;
 
-    for ( 1 .. 60 ) {
+    for my $try ( 1 .. 3) {
         my $is_active;
-        eval {
-            $node->disconnect;
-            $node->clear_netssh();
-            $node->connect();
-            $is_active = $node->is_active(1)
-        };
-        warn $@ if $@;
+        for ( 1 .. 60 ) {
+            eval {
+                $node->disconnect;
+                $node->clear_netssh();
+                $node->connect();
+                $is_active = $node->is_active(1)
+            };
+            warn $@ if $@;
+            last if $is_active;
+            sleep 1;
+            diag("Waiting for active node ".$node->name." $_") if !($_ % 10);
+        }
         last if $is_active;
-        sleep 1;
-        diag("Waiting for active node ".$node->name." $_") if !($_ % 10);
+        if ($try == 1 ) {
+            $domain->shutdown(user => user_admin);
+            sleep 2;
+        } elsif ( $try == 2 ) {
+            $domain->shutdown_now(user_admin);
+            sleep 2;
+        }
+        $domain->start(user => user_admin, remote_ip => '127.0.0.1');
     }
     is($node->_do_is_active,1,"Expecting active node ".$node->name) or exit;
 

t/nodes/30_start_stop.t

@@ -51,15 +51,16 @@ for my $vm_name ( 'KVM') {
         is( $node->is_active, 1 );
         $node->shutdown();
         for ( 1 .. 60 ) {
-            last if !$node->ping;
             sleep 1;
+            last if !$node->_ping_nocache();
         }
-        is( $node->ping, 0 );
-        is( $node->is_active, 0 );
+        my $node2 = Ravada::VM->open(id => $node->id);
+        is( $node2->_ping_nocache(), 0 );
+        is( $node2->is_active(1), 0 );
 
         # it actually dows nothing on virtual machines, just check it won't fail
         eval { $node->_wake_on_lan() };
-        is($@,'');
+        is($@,'',"Expecting no error on wake on lan");
 #        KVM testing machines can't Wake On LAN
 #        sleep 1;
 #        $node->start();