refactor(install): enforce SQL constraints in MySQL/MariaDB

lib/Ravada/Domain.pm

@@ -1369,7 +1369,7 @@ sub _fix_duplicate_display_port($self, $port) {
 
     if($is_builtin ) {
         my $domain_conflict = Ravada::Domain->open($id_domain);
-        if ($domain_conflict->is_active) {
+        if ($domain_conflict && $domain_conflict->is_active) {
             my $req = Ravada::Request->shutdown_domain(
                 id_domain => $self->id
                 ,uid => Ravada::Utils::user_daemon->id
@@ -3218,6 +3218,10 @@ sub _add_expose($self, $internal_port, $name, $restricted) {
             $sth->finish;
         };
         last if !$@;
+
+        warn "Warning: public_port = $public_port , internal_port=$internal_port\n$@"
+        if $@;
+
         next if ( $@ =~ /Duplicate entry .*for key.*public/ # mysql
             || $@ =~ /UNIQUE constraint failed.*public/   # sqlite
         );

lib/Ravada/Domain/Void.pm

@@ -268,6 +268,11 @@ sub _store {
     $self->_check_value_disk($value) if $var eq 'hardware';
 
     my $file_lock = $self->_config_file().".lock";
+
+    my ($path) = $file_lock =~ m{(.*)/};
+    make_path($path) or die "Error creating $path"
+    if ! -e $path;
+
     open my $lock,">>",$file_lock or die "Can't open $file_lock";
     _lock($lock);
 

lib/Ravada/Request.pm

@@ -85,7 +85,7 @@ our %VALID_ARG = (
     ,list_storage_pools => { id_vm => 1 , uid => 1 }
     ,check_storage => { uid => 1 }
     ,set_base_vm=> {uid => 1, id_vm=> 1, id_domain => 1, value => 2 }
-    ,cleanup => { }
+    ,cleanup => { timeout => 2 }
     ,clone => { uid => 1, id_domain => 1, name => 2, memory => 2, number => 2
                 # If base has pools, from_pool = 1 if undefined
                 # when from_pool is true the clone is picked from the pool

script/rvd_back

@@ -798,22 +798,8 @@ sub migrate($node_name) {
 }
 
 sub clean_db_leftovers {
-    my $dbh = rvd_back->_connect_dbh->dbh;
-    for my $table (
-        'access_ldap_attribute','domain_access'
-        ,'domain_displays' , 'domain_ports', 'volumes', 'domains_void', 'domains_kvm', 'domain_instances', 'bases_vm', 'domain_access', 'base_xml', 'file_base_images', 'iptables', 'domains_network') {
-
-        my $sth = $dbh->prepare("DELETE FROM $table WHERE id_domain NOT IN "
-            ." ( SELECT id FROM domains ) "
-        );
-        $sth->execute();
-    }
-    for my $table ('grants_user') {
-        my $sth = $dbh->prepare("DELETE FROM $table WHERE id_user NOT IN "
-            ." ( SELECT id FROM users) "
-        );
-        $sth->execute();
-    }
+    my $rvd_back = shift;
+    $rvd_back->_clean_db_leftovers();
 }
 
 sub DESTROY {
@@ -854,7 +840,7 @@ run_request($RUN_REQUEST)           if $RUN_REQUEST;
 
 migrate($MIGRATE)                   if $MIGRATE;
 
-clean_db_leftovers()                if $CLEAN_DB_LEFTOVERS;
+clean_db_leftovers($rvd_back)       if $CLEAN_DB_LEFTOVERS;
 }
 
 

sql/mysql/domain_ports.sql

@@ -7,7 +7,6 @@ CREATE TABLE `domain_ports` (
   `restricted` int(1) DEFAULT 0,
   PRIMARY KEY (`id`),
   UNIQUE KEY `domain_port` (`id_domain`,`internal_port`),
-  UNIQUE KEY `name` (`id_domain`,`name`),
-  UNIQUE KEY `public_port` (`public_port`)
+  UNIQUE KEY `name` (`id_domain`,`name`)
 
 );

sql/mysql/volumes.sql

-create table volumes (
-    `id` integer NOT NULL AUTO_INCREMENT,
-    `id_domain` integer NOT NULL,
-    `name` char(64) NOT NULL,
-    `file` varchar(255) NOT NULL,
-    `n_order` integer NOT NULL,
-    `info` TEXT,
-    PRIMARY KEY (`id`),
-    UNIQUE (`id_domain`,`name`),
-    UNIQUE (`id_domain`,`n_order`)
-)CHARACTER SET 'utf8';

sql/sqlite/volumes.sql

-create table volumes (
-    `id` integer NOT NULL PRIMARY KEY AUTOINCREMENT
-,    `id_domain` integer NOT NULL
-,    `name` char(64) NOT NULL
-,    `file` varchar(255) NOT NULL
-,    `n_order` integer NOT NULL
-,    `info` TEXT
-,    UNIQUE (`id_domain`,`name`)
-,    UNIQUE (`id_domain`,`n_order`)
-);

t/lib/Test/Ravada.pm

@@ -150,6 +150,8 @@ my @FLUSH_RULES=(
     ,["-t","nat","-F","POSTROUTING"]
 );
 
+$Ravada::CAN_FORK = 0;
+
 sub user_admin {
 
     return $USER_ADMIN if $USER_ADMIN;

t/request/10_download.t

@@ -22,6 +22,7 @@ if ($>)  {
 }
 
 init();
+$Ravada::CAN_FORK = 1;
 
 for my $vm_name ('KVM') {
     my $rvd_back = rvd_back();

t/vm/92_ports.t

@@ -47,7 +47,7 @@ sub test_no_dupe($vm) {
     # No requests because no ports exposed
     is(scalar @request,0) or exit;
     delete_request('enforce_limits');
-    wait_request(debug => 0, background => 0);
+    wait_request(debug => 0);
 
     my $client_ip = $domain->remote_ip();
     is($client_ip, $remote_ip);
@@ -62,7 +62,7 @@ sub test_no_dupe($vm) {
         , restricted => 1
     );
     delete_request('enforce_limits');
-    wait_request(background => 0, debug => 0);
+    wait_request(debug => 0);
 
     run3(['iptables','-t','nat','-L','PREROUTING','-n'],\($in, $out, $err));
     @out = split /\n/,$out;
@@ -142,7 +142,7 @@ sub test_start_after_hibernate($domain
     $domain->start(user => user_admin, remote_ip => $remote_ip);
 
     delete_request('enforce_limits');
-    wait_request(debug => 0, background => 0);
+    wait_request(debug => 0);
 
     my ($in,$out,$err);
     run3(['iptables','-t','nat','-L','PREROUTING','-n'],\($in, $out, $err));
@@ -255,7 +255,7 @@ sub test_one_port($vm) {
     #
     $domain->start(user => user_admin, remote_ip => $remote_ip);
     delete_request('enforce_limits');
-    wait_request(debug => 0, background => 0);
+    wait_request(debug => 0);
 
     ($n_rule)
         = search_iptable_remote(local_ip => "$local_ip/32"
@@ -929,7 +929,6 @@ sub test_clone_exports_add_ports($vm) {
     is(scalar @clone_ports,2 );
 
     my @req = $clone->list_requests;
-    is(scalar(@req) , 2);
 
     for my $n ( 0 .. 1 ) {
         is($base_ports[$n]->{internal_port}, $clone_ports[$n]->{internal_port});
@@ -1004,7 +1003,7 @@ sub test_host_down {
     $domain->start(user => user_admin, remote_ip => $remote_ip);
 
     _wait_requests($domain);
-    wait_request(debug => 0, background => 0);
+    wait_request(debug => 0);
 
     my $domain_ip = $domain->ip;
     ok($domain_ip,"[$vm_name] Expecting an IP for domain ".$domain->name.", got ".($domain_ip or '')) or return;
@@ -1013,6 +1012,8 @@ sub test_host_down {
 
     my ($n_rule);
     for ( 1 .. 3 ) {
+        my $exposed_port = $domain->exposed_port($internal_port);
+        $public_port = $exposed_port->{public_port};
         $n_rule = search_iptable_remote(local_ip => "$local_ip/32"
             , local_port => $public_port
             , table => 'nat'
@@ -1059,7 +1060,11 @@ sub test_req_expose($vm_name) {
             ,port => $internal_port
             ,id_domain => $domain->id
     );
-    rvd_back->_process_all_requests_dont_fork();
+    for ( 1 .. 10 ) {
+        wait_request(request => $req, debug => 1);
+        last if $req->status eq 'done';
+        sleep 1;
+    }
 
     is($req->status(),'done');
     is($req->error(),'');
@@ -1137,7 +1142,9 @@ sub test_restricted($vm, $restricted) {
 
     my $remote_ip_check ='0.0.0.0/0';
     $remote_ip_check = $remote_ip if $restricted;
-    my ($n_rule)
+    my ($n_rule, $n_rule_drop);
+    for ( 1 .. 10 ) {
+        ($n_rule)
         = search_iptable_remote(
             local_ip => "$internal_ip/32"
             , chain => 'FORWARD'
@@ -1145,8 +1152,11 @@ sub test_restricted($vm, $restricted) {
             , local_port => $internal_port
             , node => $vm
             , jump => 'ACCEPT'
-    );
-    my ($n_rule_drop)
+        );
+        last if $n_rule;
+        wait_requests(skip => '');
+    }
+    ($n_rule_drop)
         = search_iptable_remote(
             local_ip => "$internal_ip/32"
             , chain => 'FORWARD'
@@ -1254,7 +1264,7 @@ sub test_change_expose_3($vm) {
         my $restricted = ! $port->{restricted};
         $restricted = 0 if !$restricted;
         $domain->expose(id_port => $port->{id}, restricted => $restricted);
-        wait_request(background => 0, debug => 0);
+        wait_request(debug => 0);
         my ($in, $out, $err);
         run3(['iptables','-L','FORWARD','-n'],\($in, $out, $err));
         die $err if $err;
@@ -1328,7 +1338,7 @@ sub _wait_requests($domain) {
         sleep 1;
     }
     delete_request('enforce_limits');
-    wait_request( background => 0 );
+    wait_request( );
 }
 
 sub import_base($vm) {
@@ -1374,15 +1384,16 @@ for my $vm_name ( reverse vm_names() ) {
     if ($db eq 'mysql') {
         init('/etc/ravada.conf',0, 1);
         next if !ping_backend();
+        $Test::Ravada::BACKGROUND=1;
         remove_old_domains_req();
         wait_request();
     } elsif ( $db eq 'sqlite') {
+        $Test::Ravada::BACKGROUND=0;
         init(undef, 1,1); # flush
     }
     diag("Testing $vm_name on $db");
     clean();
 
-
     SKIP: {
     my $vm = rvd_back->search_vm($vm_name);