feature(backend): check storage (#1388)

Added this as a request and runs before auto start machines

lib/Ravada.pm

@@ -1473,6 +1473,7 @@ sub _upgrade_tables {
     $self->_upgrade_table('requests','start_time','int(11) DEFAULT NULL');
     $self->_upgrade_table('requests','output','text DEFAULT NULL');
     $self->_upgrade_table('requests','after_request','int(11) DEFAULT NULL');
+    $self->_upgrade_table('requests','after_request_ok','int(11) DEFAULT NULL');
 
     $self->_upgrade_table('requests','at_time','int(11) DEFAULT NULL');
     $self->_upgrade_table('requests','run_time','float DEFAULT NULL');
@@ -3524,6 +3525,56 @@ sub _cmd_refresh_storage($self, $request=undef) {
     $vm->refresh_storage();
 }
 
+sub _list_mnt($vm, $type) {
+    my ($out, $err) = $vm->run_command("findmnt","-$type");
+    my %tab;
+    for my $line ( split /\n/,$out ) {
+        my ($target) = $line =~ /^(.*?) /;
+        next if $target eq 'TARGET';
+        $tab{$target} = $line;
+    }
+    return %tab;
+}
+
+sub _search_partition($path, $tab) {
+    confess if ref($path);
+    my $curr_path = "";
+    my $found='';
+    for my $dir (split /\//,$path ) {
+        $dir = "" if !defined $dir;
+        $curr_path .= "/$dir";
+        $curr_path =~ s{\/\/+}{/}g;
+        next if !exists $tab->{$curr_path};
+        my $curr_found = $tab->{$curr_path};
+        $found = $curr_path if $curr_found && length($curr_found) > length($found);
+    }
+    return $found;
+}
+
+sub _check_mounted($path, $fstab, $mtab) {
+    my $partition = _search_partition($path, $fstab);
+    return 1 if exists $mtab->{$partition} && $mtab->{$partition};
+
+    die "Error: partition $partition not mounted. Retry.\n";
+}
+
+sub _cmd_check_storage($self, $request) {
+    my $contents = "a" x 160;
+    for my $vm ( $self->list_vms ) {
+        next if !$vm->is_local;
+        my %fstab = _list_mnt($vm,"s");
+        my %mtab = _list_mnt($vm,"m");
+
+        for my $storage ( $vm->list_storage_pools ) {
+            next if $storage !~ /tst/;
+            my $path = ''.$vm->_storage_path($storage);
+            _check_mounted($path,\%fstab,\%mtab);
+            my ($ok,$err) = $vm->write_file("$path/check_storage",$contents);
+            die "Error on starage pool $storage : $err. Retry.\n" if $err;
+        }
+    }
+}
+
 sub _cmd_refresh_machine($self, $request) {
 
     my $id_domain = $request->args('id_domain');
@@ -3939,6 +3990,7 @@ sub _req_method {
         ,rebase => \&_cmd_rebase
 
 ,refresh_storage => \&_cmd_refresh_storage
+,check_storage => \&_cmd_check_storage
 ,refresh_machine => \&_cmd_refresh_machine
 ,domain_autostart=> \&_cmd_domain_autostart
 ,change_owner => \&_cmd_change_owner

lib/Ravada/Request.pm

@@ -78,6 +78,7 @@ our %VALID_ARG = (
     ,hybernate=> {uid => 1, id_domain => 1}
     ,download => {uid => 2, id_iso => 1, id_vm => 2, verbose => 2, delay => 2, test => 2}
     ,refresh_storage => { id_vm => 2 }
+    ,check_storage => { uid => 1 }
     ,set_base_vm=> {uid => 1, id_vm=> 1, id_domain => 1, value => 2 }
     ,cleanup => { }
     ,clone => { uid => 1, id_domain => 1, name => 2, memory => 2, number => 2
@@ -425,7 +426,7 @@ sub _check_args {
     my $args = { @_ };
 
     my $valid_args = $VALID_ARG{$sub};
-    for (qw(at after_request retry _no_duplicate)) {
+    for (qw(at after_request after_request_ok retry _no_duplicate)) {
         $valid_args->{$_}=2 if !exists $valid_args->{$_};
     }
 
@@ -576,12 +577,12 @@ sub _new_request {
             confess "ERROR: Different id_domain: ".Dumper(\%args)
                 if $args{id_domain} && $args{id_domain} ne $id_domain_args;
             $args{id_domain} = $id_domain_args;
-            $args{after_request} = delete $args{args}->{after_request}
-                if exists $args{args}->{after_request};
-            $args{retry} = delete $args{args}->{retry}
-                if exists $args{args}->{retry};
-
         }
+        for (qw(after_request after_request_ok retry)) {
+            $args{$_} = delete $args{args}->{$_}
+            if exists $args{args}->{$_};
+        }
+
         $args{args} = encode_json($args{args});
     }
     _init_connector()   if !$CONNECTOR || !$$CONNECTOR;
@@ -1284,11 +1285,25 @@ sub priority($self) {
 
 sub requirements_done($self) {
     my $after_request = $self->after_request();
-    return 1 if !defined $after_request;
-
-    my $req = Ravada::Request->open($self->after_request);
-    return 1 if $req->status eq 'done';
-    return 0;
+    my $after_request_ok = $self->after_request_ok();
+    return 1 if !defined $after_request && !defined $after_request_ok;
+
+    my $ok = 0;
+    if ($after_request) {
+        $ok = 0;
+        my $req = Ravada::Request->open($self->after_request);
+        $ok = 1 if $req->status eq 'done';
+    }
+    if ($after_request_ok) {
+        $ok = 0;
+        my $req = Ravada::Request->open($self->after_request_ok);
+        if ($req->status eq 'done' && $req->error ) {
+            $self->status('done');
+            $self->error($req->error);
+        }
+        $ok = 1 if $req->status eq 'done' && $req->error eq '';
+    }
+    return $ok;
 }
 
 sub AUTOLOAD {

lib/Ravada/VM/Void.pm

@@ -190,6 +190,11 @@ sub dir_img {
 sub dir_base  { return dir_img }
 sub dir_clone { return dir_img }
 
+sub _storage_path($self, $storage) {
+    confess "Error: unknown storage '$storage'" if $storage ne 'default';
+    return dir_img;
+}
+
 sub _list_domains_local($self, %args) {
     my $active = delete $args{active};
 

script/rvd_back

@@ -208,6 +208,10 @@ sub clean_old_requests {
 
 sub autostart_machines {
     my $ravada = shift;
+    my $req = Ravada::Request->check_storage(
+        uid => Ravada::Utils::user_daemon->id
+        ,retry => 10
+    );
     for my $domain ( $ravada->list_domains_data ) {
         next unless $domain->{autostart} && ! $domain->{is_base}
               && $domain->{status} !~ /active/i;
@@ -217,6 +221,7 @@ sub autostart_machines {
         Ravada::Request->start_domain(
             id_domain => $domain->{id}
             ,uid => $domain->{id_owner}
+            ,after_request_ok => $req->id
         );
     }
 }

t/lib/Test/Ravada.pm

@@ -828,6 +828,7 @@ sub wait_request {
             if ( $req->status ne 'done' ) {
                 diag("Waiting for request ".$req->id." ".$req->command." ".$req->status
                     ." ".($req->error or '')) if $debug && (time%5 == 0);
+                sleep 1;
                 $done_all = 0;
             } elsif (!$done{$req->id}) {
                 $t0 = time;
@@ -965,6 +966,10 @@ sub _delete_qemu_pool($pool) {
     my $xml = XML::LibXML->load_xml(string => $pool->get_xml_description());
     my ($path) = $xml->findnodes('/pool/target/path');
     my $dir = $path->textContent();
+    for my $file ( qw(check_storage) ) {
+        my $path ="$dir/$file";
+        unlink $path or die "$! $path" if -e $path;
+    }
     rmdir($dir) or die "$! $dir";
 
 }
@@ -1765,7 +1770,7 @@ sub shutdown_nodes {
     }
 }
 
-sub create_storage_pool($vm) {
+sub create_storage_pool($vm, $dir=undef) {
     if (!ref($vm)) {
         $vm = rvd_back->search_vm($vm);
     }
@@ -1775,7 +1780,9 @@ sub create_storage_pool($vm) {
     my $capacity = 1 * 1024 * 1024;
 
     my $pool_name = new_pool_name();
-    my $dir = "/var/tmp/$pool_name";
+    if (!$dir) {
+        $dir = "/var/tmp/$pool_name";
+    }
 
     mkdir $dir if ! -e $dir;
 

t/vm/s30_storage.t

+use warnings;
+use strict;
+
+use Carp qw(confess);
+use Data::Dumper;
+use File::Copy;
+use Test::More;
+
+no warnings "experimental::signatures";
+use feature qw(signatures);
+
+use lib 't/lib';
+use Test::Ravada;
+
+
+###########################################################
+
+sub test_storage_pools($vm) {
+    my $req = Ravada::Request->check_storage(
+        uid => user_admin->id
+    );
+    my $req_cleanup = Ravada::Request->cleanup(
+        after_request_ok => $req->id
+    );
+    wait_request( debug => 1);
+    is($req->status,'done');
+    is($req->error, '');
+
+    is($req_cleanup->status,'done');
+    is($req_cleanup->error, '');
+}
+
+sub _add_fstab($vm) {
+    my $dir = "/mnt/".base_domain_name();
+    if (!-e $dir) {
+        mkdir $dir;
+    }
+    my %fstab = Ravada::_list_mnt($vm,"s");
+    return $dir if $fstab{$dir};
+
+    copy("/etc/fstab","/etc/fstab.tst_rvd_backup") or die $!;
+
+    open my $fstab,">>","/etc/fstab" or die $!;
+    print $fstab "bogus.ravada:$dir $dir nfs rw,_netdev 0 0\n";
+    close $fstab;
+
+    return $dir;
+}
+
+sub remove_fstab($vm, $dir) {
+    my $file = "$dir/check_storage";# or die "$!";
+    unlink $file or die "$! $file" if -e $file;
+    copy("/etc/fstab.tst_rvd_backup","/etc/fstab")
+}
+
+sub test_storage_pools_fail($vm) {
+    return if $vm->type ne 'KVM';
+    my $dir = _add_fstab($vm);
+
+    create_storage_pool($vm, $dir);
+
+    my $req = Ravada::Request->check_storage(
+        uid => user_admin->id
+        ,retry => 2
+    );
+    my $req_cleanup = Ravada::Request->cleanup(
+        after_request_ok => $req->id
+    );
+    is($req_cleanup->after_request_ok,$req->id);
+    wait_request( debug => 1, check_error => 0);
+    is($req->status,'done');
+    like($req->error, qr/not mounted/);
+
+    is($req_cleanup->status,'done');
+    like($req_cleanup->error, qr/not mounted/);
+
+    remove_fstab($vm, $dir);
+    $vm->refresh_storage_pools();
+}
+
+sub _clean_local {
+    my $dir = "/mnt/".base_domain_name();
+    my $file = "$dir/check_storage";# or die "$!";
+    unlink $file or die "$! $file" if -e $file;
+}
+###########################################################
+
+_clean_local();
+clean();
+
+for my $vm_name (vm_names() ) {
+    ok($vm_name);
+    SKIP: {
+        my $vm = rvd_back->search_vm($vm_name);
+        my $msg = "SKIPPED: No virtual managers found";
+        if ($vm && $vm_name =~ /kvm/i && $>) {
+            $msg = "SKIPPED: Test must run as root";
+            $vm = undef;
+        }
+
+        skip($msg,10)   if !$vm;
+        test_storage_pools($vm);
+        test_storage_pools_fail($vm);
+    }
+}
+
+end();
+done_testing();