Unverified Commit 872ecd23 authored by Francesc Guasch's avatar Francesc Guasch Committed by GitHub
Browse files

Fix shared storage change hardware (#1521)

fix(backend): redefine instances on change hardware

* test(nodes): remove base shared storage
* test(nodes): shared storage
* refactor(backend): deal with gone volumes

issue #1340 
parent 5d6c757b
......@@ -206,13 +206,11 @@ around 'is_hibernated' => \&_around_is_hibernated;
around 'autostart' => \&_around_autostart;
before 'set_controller' => \&_pre_change_hardware;
before 'remove_controller' => \&_pre_change_hardware;
before 'change_hardware' => \&_pre_change_hardware;
around 'set_controller' => \&_around_change_hardware;
around 'change_hardware' => \&_around_change_hardware;
after 'set_controller' => \&_post_change_hardware;
after 'remove_controller' => \&_post_change_hardware;
after 'change_hardware' => \&_post_change_hardware;
before 'remove_controller' => \&_pre_remove_hardware;
after 'remove_controller' => \&_post_remove_hardware;
around 'name' => \&_around_name;
......@@ -357,6 +355,9 @@ sub _around_start($orig, $self, @arg) {
$error = $@;
last if !$error;
warn "WARNING: $error ".$self->_vm->name." ".$self->_vm->enabled if $error;
next if $error && ref($error) && $error->code == 1;# pool has asynchronous jobs running.
if ($error && $self->id_base && !$self->is_local && $self->_vm->enabled) {
$self->_request_set_base();
next;
......@@ -1822,8 +1823,14 @@ sub _after_remove_domain {
}
sub _remove_all_volumes($self) {
my $vm_local = $self->_vm;
$vm_local = $self->_vm->new( host => 'localhost' ) if !$self->is_local;
for my $vol (@{$self->{_volumes}}) {
next if $vol =~ /iso$/;
if (!$self->is_local) {
my ($dir) = $vol =~ m{(.*)/};
next if $vm_local->shared_storage($self->_vm, $dir);
}
$self->remove_volume($vol);
}
}
......@@ -1853,6 +1860,23 @@ sub _remove_domain_cascade($self,$user, $cascade = 1) {
}
}
sub _redefine_instances($self) {
my $domain_name = $self->name or confess "Unknown my self name $self ".Dumper($self->{_data});
my @instances = $self->list_instances();
for my $instance ( @instances ) {
next if $instance->{id_vm} == $self->_vm->id;
my $vm;
eval { $vm = Ravada::VM->open($instance->{id_vm}) };
die $@ if $@ && $@ !~ /I can't find VM/i;
next if !$vm || !$vm->is_active;
my $domain;
$@ = '';
eval { $domain = $vm->search_domain($domain_name) } if $vm;
warn $@ if $@;
$domain->copy_config($self);
}
}
sub _remove_ports_db($self) {
return if !$self->{_data}->{id};
my $sth = $$CONNECTOR->dbh->prepare("DELETE FROM domain_ports"
......@@ -2192,22 +2216,23 @@ sub _do_remove_base($self, $user) {
my $vm_local = $self->_vm->new( host => 'localhost' );
for my $vol ($self->list_volumes_info) {
next if !$vol->file || $vol->file =~ /\.iso$/;
my ($dir) = $vol->file =~ m{(.*)/};
next if !$self->is_local && $self->_vm->shared_storage($vm_local, $dir);
my $backing_file = $vol->backing_file;
next if !$backing_file;
# confess "Error: no backing file for ".$vol->file if !$backing_file;
if (!$self->is_local) {
my ($dir) = $backing_file =~ m{(.*/)};
if ( $self->_vm->shared_storage($vm_local, $dir) ) {
next;
}
next if $self->_vm->shared_storage($vm_local, $dir);
$self->_vm->remove_file($vol->file);
$self->_vm->remove_file($backing_file);
$self->_vm->refresh_storage_pools();
return ;
next;
}
$vol->block_commit();
unlink $vol->file or die "$! ".$vol->file;
my @stat = stat($backing_file);
my @stat = stat($backing_file) or confess "Error: missing $backing_file";
move($backing_file, $vol->file) or die "$! $backing_file -> ".$vol->file;
my $mask = oct(7777);
my $mode = $stat[2] & $mask;
......@@ -2219,8 +2244,11 @@ sub _do_remove_base($self, $user) {
for my $file ($self->list_files_base) {
next if $file =~ /\.iso$/i;
next if ! -e $file;
unlink $file or die "$! unlinking $file";
next if ! $self->_vm->file_exists($file);
my ($dir) = $file =~ m{(.*/)};
next if !$self->_vm->is_local && $self->_vm->shared_storage($vm_local, $dir);
$self->_vm->remove_file($file);
}
$self->storage_refresh() if $self->storage();
......@@ -4688,14 +4716,32 @@ sub needs_restart($self, $value=undef) {
return $self->_data('needs_restart',$value);
}
sub _pre_change_hardware($self, @) {
sub _pre_remove_hardware($self, @) {
if (!$self->_vm->is_local) {
my $vm_local = $self->_vm->new( host => 'localhost' );
$self->_set_vm($vm_local, 1);
}
}
sub _post_remove_hardware($self, $hardware, $index, $data=undef) {
if ($hardware eq 'disk' && ( defined $index || $data ) && $self->is_known() ) {
my $sth = $$CONNECTOR->dbh->prepare("DELETE FROM volumes WHERE id_domain=?");
$sth->execute($self->id);
}
$self->needs_restart(1) if $self->is_known && $self->_data('status') eq 'active';
}
sub _around_change_hardware($orig, $self, $hardware, $index=undef, $data=undef) {
my $real_id_vm;
if ($hardware eq 'disk' && !$self->_vm->is_local) {
$real_id_vm = $self->_vm->id;
my $vm_local = $self->_vm->new( host => 'localhost' );
$self->_set_vm($vm_local, 1);
}
$orig->($self, $hardware, $index, $data);
sub _post_change_hardware($self, $hardware, $index, $data=undef) {
if ($hardware eq 'disk' && ( defined $index || $data ) && $self->is_known() ) {
my $sth = $$CONNECTOR->dbh->prepare("DELETE FROM volumes WHERE id_domain=?");
$sth->execute($self->id);
......@@ -4703,10 +4749,16 @@ sub _post_change_hardware($self, $hardware, $index, $data=undef) {
}
$self->info(Ravada::Utils::user_daemon) if $self->is_known();
$self->_remove_domain_cascade(Ravada::Utils::user_daemon,1)
if $self->is_known() && !$self->is_base;
if ( $self->is_known() && !$self->is_base ) {
$self->_redefine_instances();
}
$self->needs_restart(1) if $self->is_known && $self->_data('status') eq 'active';
if ( $real_id_vm ) {
my $id_vm = $real_id_vm;
my $vm = Ravada::VM->open($id_vm);
$self->_set_vm($vm, 1);
}
}
=head2 Access restrictions
......
......@@ -2342,6 +2342,16 @@ sub _post_change_hardware($self, $doc) {
$self->info(Ravada::Utils::user_daemon);
}
sub copy_config($self, $domain) {
my $doc = XML::LibXML->load_xml(string => $domain->xml_description(Sys::Virt::Domain::XML_INACTIVE));
my ($uuid) = $doc->findnodes("/domain/uuid/text()");
confess "I cant'find /domain/uuid in ".$self->name if !$uuid;
$uuid->setData($self->domain->get_uuid_string);
my $new_domain = $self->_vm->vm->define_domain($doc->toString);
$self->domain($new_domain);
}
sub _change_xml_address($self, $doc, $address, $bus) {
my $type_def = $address->getAttribute('type');
return $self->_change_xml_address_ide($doc, $address, 1, 1) if $bus eq 'ide';
......
......@@ -455,14 +455,31 @@ Returns true if the file exists in this virtual manager storage
sub file_exists($self, $file) {
for my $pool ($self->vm->list_all_storage_pools ) {
$pool->refresh();
for my $vol ( $pool->list_all_volumes ) {
return 1 if $vol->get_path eq $file;
$self->_wait_storage( sub { $pool->refresh() } );
my @volumes = $self->_wait_storage( sub { $pool->list_all_volumes });
for my $vol ( @volumes ) {
my $found;
eval { $found = 1 if $vol->get_path eq $file };
# volume was removed in the nick of time
die $@ if $@ && ( !ref($@) || $@->code != 50);
return 1 if $found;
}
}
return 0;
}
sub _wait_storage($self, $sub) {
my @ret;
for ( 1 .. 10 ) {
eval { @ret=$sub->() };
last if !$@;
die $@ if !ref($@) || $@->code != 1;
warn "Warning: $@ [retrying $_]";
sleep 1;
};
return @ret;
}
=head2 dir_img
Returns the directory where disk images are stored in this Virtual Manager
......
......@@ -109,8 +109,9 @@ sub test_prepare_base {
eval {
$domain->remove_base( user_admin);
$domain->prepare_base( user_admin );
$@ = '';
};
is($@,'');
is(''.$@,'');
ok($domain->is_base);
my $name_clone = new_domain_name();
......
......@@ -121,7 +121,7 @@ sub test_remove_domain {
sub test_base {
my $domain = shift;
eval { $domain->prepare_base( user_admin ) };
is($@,'',"Prepare base") or exit;
is(''.$@,'',"Prepare base") or exit;
my @files_base = $domain->list_files_base();
is(scalar @files_base, 2);
......
......@@ -596,6 +596,7 @@ sub test_bases_node {
is($domain->base_in_vm($node->id), undef);
$domain->migrate($node);
is($domain->_vm->id, $node->id) or exit;
is($domain->base_in_vm($node->id), 1);
for my $file ( $domain->list_files_base ) {
......@@ -606,6 +607,9 @@ sub test_bases_node {
$domain->set_base_vm(vm => $node, value => 0, user => user_admin);
is($domain->base_in_vm($node->id), 0);
for my $file ( $domain->list_files_base ) {
ok($vm->file_exists($file)) or die "File $file doesn't exist in ".$vm->name;
}
$domain->set_base_vm(vm => $vm, value => 0, user => user_admin);
is($domain->is_base(),0);
......
......@@ -52,6 +52,7 @@ create_domain
hibernate_domain_internal
remote_node
remote_node_2
remote_node_shared
add_ubuntu_minimal_iso
create_ldap_user
connector
......@@ -727,7 +728,13 @@ sub _remove_old_disks_kvm {
next if $volume->get_name !~ /^${name}_\d+.*\.(img|raw|ro\.qcow2|qcow2|void|backup)$/;
eval { $volume->delete() };
warn $@ if $@;
if ($@) {
if ($@->code == 38 ) {
$vm->remove_file($volume->get_path);
} else {
warn "Error $@ removing ".$volume->get_name." in ".$vm->name if $@;
}
}
}
}
eval {
......@@ -1680,6 +1687,15 @@ sub remote_node_2($vm_name) {
return @nodes;
}
sub remote_node_shared($vm_name) {
my $remote_config = {
'name' => 'ztest-shared'
,'host' => '192.168.122.153'
,public_ip => '192.168.130.153'
};
return _do_remote_node($vm_name, $remote_config);
}
sub _do_remote_node($vm_name, $remote_config) {
my $vm = rvd_back->search_vm($vm_name);
......
......@@ -794,8 +794,7 @@ sub test_remove_base($vm, $node, $volatile) {
$base->remove_base_vm(node => $node, user => user_admin);
for my $file ( @volumes , @volumes0 ) {
my ($out, $err) = $node->run_command("ls $file");
ok(!$out, "Expecting no file '$file' in ".$node->name) or exit;
ok(!$node->file_exists($file));
ok(-e $file, "Expecting file '$file' in local") or exit;
}
isnt($base->_data('id_vm'), $node->id);
......
......@@ -185,14 +185,16 @@ sub test_change_hardware($vm, @nodes) {
."WHERE id_domain = ".$clone->id );
$sth->execute();
my ($count) = $sth->fetchrow;
is($count,1,"Expecting other instances removed when hardware changed") or exit;
is($count,1+scalar(@nodes),"Expecting other instances not removed when hardware $hardware removed");
for my $node (@nodes) {
my $clone2 = $node->search_domain($clone->name);
ok(!$clone2,"Expecting no clone ".$clone->name." in remote node ".$node->name) or exit;
ok($clone2,"Expecting clone ".$clone->name." in remote node ".$node->name
." when removing $hardware");
}
is($clone->_vm->is_local,1) or exit;
my $clone_fresh = Ravada::Domain->open($clone->id);
is($clone_fresh->_vm->is_local, 1) or exit;
for (@volumes) {
ok(-e $_,$_) or exit;
}
......
......@@ -12,20 +12,20 @@ use Test::Ravada;
no warnings "experimental::signatures";
use feature qw(signatures);
my $SHARED_SP = "pool_tst";
my $SHARED_SP = "pool_shared";
my $DIR_SHARED = "/home2/pool_shared";
init();
my $BASE_NAME = "zz-test-base-alpine";
my $BASE;
#################################################################################
sub test_shared($vm, $node) {
$vm->default_storage_pool_name($SHARED_SP);
my $domain = create_domain($vm);
my $storage_path = $vm->_storage_path($SHARED_SP);
is($vm->shared_storage($node, $storage_path),1,"Expecting $SHARED_SP shared") or exit;
for my $vol ($domain->list_disks) {
like($vol,qr(^$storage_path), $vol);
}
......@@ -43,6 +43,17 @@ sub test_shared($vm, $node) {
is($domain->base_in_vm($vm->id),1);
my @files_base = $domain->list_files_base();
for my $vol (@files_base) {
my $ok;
for ( 1 .. 5 ) {
$ok = -e $vol;
last if $ok;
sleep 1;
}
ok($ok,"Volume $vol should exist") or exit;
ok($node->file_exists($vol), "Volume $vol should exist in ".$node->name);
}
$req = Ravada::Request->set_base_vm(
uid => user_admin->id
......@@ -65,12 +76,173 @@ sub test_shared($vm, $node) {
last if $ok;
sleep 1;
}
ok($ok,"Volume $vol should exists");
ok($ok,"Volume $vol should exist") or exit;
ok($node->file_exists($vol), "Volume $vol should exist in ".$node->name);
}
$domain->remove(user_admin);
}
sub test_is_shared($vm, $node) {
is($vm->shared_storage($node,$DIR_SHARED),1) or exit;
is($node->shared_storage($vm,$DIR_SHARED),1) or exit;
my $sth = connector->dbh->prepare("SELECT * FROM storage_nodes "
." WHERE dir=?"
);
my $dir_shared = $DIR_SHARED;
$dir_shared.="/" unless $dir_shared =~ m{/$};
$sth->execute($dir_shared);
my $row = $sth->fetchrow_hashref;
is($row->{is_shared},1) or die Dumper($row);
}
sub _add_disk($domain) {
my $req = Ravada::Request->add_hardware(
uid => user_admin->id
,id_domain => $domain->id
,name=> 'disk'
,data => { size => 512 * 1024 }
);
wait_request(debug => 0);
is($req->error,'');
}
sub _change_ram($domain) {
my $mem = $domain->info(user_admin)->{memory};
my $new_mem = int($mem * 0.8 ) - 1;
my $req = Ravada::Request->change_hardware(
uid => user_admin->id
,id_domain => $domain->id
,hardware => 'memory'
,data => {memory => $new_mem }
);
wait_request(debug => 0);
is($req->error,'');
return $new_mem;
}
sub test_change_ram($vm, $node, $start=0, $prepare_base=0, $migrate=0) {
diag("start=$start , prepare_base=$prepare_base , migrate=$migrate");
my $base = $BASE->clone(name => new_domain_name, user => user_admin);
$base->spinoff();
my $domain = $base;
if ($prepare_base) {
$base->prepare_base(user_admin);
$base->set_base_vm(vm => $node, user => user_admin);
$domain = $base->clone(name => new_domain_name, user => user_admin);
}
req_migrate($node, $domain, $start) if $migrate;
my $new_mem = _change_ram($domain);
if ( $migrate ) {
my $domain2 = Ravada::Domain->open($domain->id);
is($domain2->_vm->id, $node->id);
}
_test_volumes_exist($domain);
my $domain_local = $vm->search_domain($domain->name);
is($domain_local->_vm->id,$vm->id);
my $mem2 = $domain_local->info(user_admin)->{memory};
is($mem2, $new_mem);
req_start($domain);
req_migrate($node, $domain, 1);
$domain = Ravada::Domain->open($domain->id);
$mem2 = $domain->info(user_admin)->{memory};
is($mem2, $new_mem);
_test_volumes_exist($domain);
$domain->remove(user_admin);
$base->remove(user_admin) if $base->id != $domain->id;
}
sub _test_volumes_exist($domain) {
my $domain_local = Ravada::Domain->open($domain->id);
for my $disk ( $domain_local->list_volumes ) {
ok(-e $disk,"Expecting ".$domain_local->name." $disk exist in "
.$domain_local->_vm->name) or confess;
}
if ($domain->id_base) {
my $base = Ravada::Domain->open($domain->id_base);
for my $disk ($base->list_files_base) {
ok(-e $disk,"Expecting $disk exist");
}
}
}
sub test_add_disk($vm, $node, $start=0, $prepare_base=0, $migrate=0) {
my $base = $BASE->clone(name => new_domain_name, user => user_admin);
$base->spinoff();
my $domain = $base;
if ($prepare_base) {
$base->prepare_base(user_admin);
$base->set_base_vm(vm => $node, user => user_admin);
$domain = $base->clone(name => new_domain_name, user => user_admin);
}
my $n = scalar($domain->list_volumes);
req_migrate($node, $domain, $start) if $migrate;
_add_disk($domain);
if ( $migrate ) {
my $domain2 = Ravada::Domain->open($domain->id);
is($domain2->_vm->id, $node->id);
}
_test_volumes_exist($domain);
my $domain_local = $vm->search_domain($domain->name);
is($domain_local->_vm->id,$vm->id);
is(scalar($domain_local->list_volumes),$n+1);
req_start($domain);
req_migrate($node, $domain, 1);
$domain = Ravada::Domain->open($domain->id);
is(scalar($domain->list_volumes),$n+1);
_test_volumes_exist($domain);
$domain->remove(user_admin);
# $base->remove(user_admin) if $base->id != $domain->id;
}
sub req_start($domain) {
my $req = Ravada::Request->start_domain(
id_domain => $domain->id
,uid => user_admin->id
);
wait_request();
is($req->status,'done');
is($req->error,'');
}
sub req_migrate($node, $domain, $start=0) {
my $req = Ravada::Request->migrate(
id_domain => $domain->id
,uid => user_admin->id
,shutdown => 1
,shutdown_timeout => 30
,start => 1
,id_node => $node->id
);
wait_request();
is($req->status,'done');
is($req->error,'');
my $domain2 = Ravada::Domain->open($domain->id);
is($domain2->_vm->id,$node->id);
}
sub import_base($vm) {
if ($vm->type eq 'KVM') {
$BASE = import_domain($vm->type, $BASE_NAME, 1);
confess "Error: domain $BASE_NAME is not base" unless $BASE->is_base;
} else {
$BASE = create_domain($vm);
}
}
#################################################################################
clean();
......@@ -107,7 +279,7 @@ the file "
skip($msg,10) if !$vm;
diag("Testing remote node in $vm_name");
my $node = remote_node($vm_name) or next;
my $node = remote_node_shared($vm_name) or next;
clean_remote_node($node);
ok($node->vm,"[$vm_name] expecting a VM inside the node") or do {
......@@ -122,7 +294,29 @@ the file "
skip($msg,10);
}
# set storage in shared dir
$vm->default_storage_pool_name($SHARED_SP);
my $storage_path = $vm->_storage_path($SHARED_SP);
is($vm->shared_storage($node, $storage_path),1,"Expecting $SHARED_SP shared") or exit;
import_base($vm);
test_is_shared($vm, $node);
test_shared($vm, $node);
for my $default_sp ( 0,1 ) {
$node->default_storage_pool_name('default') if !$default_sp;
$node->default_storage_pool_name($SHARED_SP) if $default_sp;
for my $start ( 0,1 ) {
for my $prepare_base ( 0,1 ) {
for my $migrate( 1, 0 ) {
test_change_ram($vm,$node, $start, $prepare_base, $migrate);
test_add_disk($vm,$node, $start, $prepare_base, $migrate);
}
}
}
}
NEXT:
clean_remote_node($node);
remove_node($node);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment