Commit 41013f00 authored by Francesc Guasch's avatar Francesc Guasch Committed by Fernando Verdugo
Browse files

Feat/865 volatile remote (#868)

* test(backend): start of volatile clones on remote

Volatile clones must be started on creation. Check it works:
- base has: volatile_clones enabled
- user is temporary

issue #865

* refactor(frontend): removed old call

* refactor(backend): moved balance vm code to VM

Also, make sure the VM is balanced on clones

issue #865

* refactor(backend): pass request to sync to issue messages

issue #865

* test(backend): allow mock VMs to add volumes

issue #865

* wip(backend): run clone volumes in remote too

issue #865

* feat(backend): balance VMs before creation if volatile

fixes issue #865

* test(nodes): machine can be created on remote now

* wip(backend): balance VM for volatiles before creation

issue #865

* wip(nodes): do not use disabled nodes

issue #865

* wip(nodes): do not check bad shutdown on volatiles

issue #865

* wip(nodes): allow volumes on remote nodes

issue #865

* test(nodes): clean failed tests

issue #865
parent 060e2861
......@@ -1380,11 +1380,20 @@ sub create_domain {
$vm = $self->search_vm($vm_name);
confess "ERROR: vm $vm_name not found" if !$vm;
}
my $base;
if ($id_base) {
my $base = Ravada::Domain->open($id_base)
$base = Ravada::Domain->open($id_base)
or confess "Unknown base id: $id_base";
$vm = $base->_vm;
}
my $user = Ravada::Auth::SQL->search_by_id($id_owner);
$request->status("creating machine") if $request;
if ( $base && $base->volatile_clones
|| $user->is_temporary ) {
$vm = $vm->balance_vm($base);
$request->status("creating machine on ".$vm->name);
}
confess "No vm found, request = ".Dumper(request => $request) if !$vm;
......@@ -1393,7 +1402,6 @@ sub create_domain {
confess "I can't find any vm ".Dumper($self->vm) if !$vm;
$request->status("creating") if $request;
my $domain;
delete $args{'at'};
eval { $domain = $vm->create_domain(%args)};
......@@ -1410,7 +1418,6 @@ sub create_domain {
if (!$error && $start) {
$request->status("starting") if $request;
eval {
my $user = Ravada::Auth::SQL->search_by_id($id_owner);
my $remote_ip;
$remote_ip = $request->defined_arg('remote_ip') if $request;
$domain->start(
......@@ -2135,7 +2142,7 @@ sub _cmd_create{
my $self = shift;
my $request = shift;
$request->status('creating domain');
$request->status('creating machine');
warn "$$ creating domain ".Dumper($request->args) if $DEBUG;
my $domain;
......
......@@ -223,7 +223,7 @@ sub BUILD {
}
sub _check_clean_shutdown($self) {
return if !$self->is_known || $self->readonly;
return if !$self->is_known || $self->readonly || $self->is_volatile;
if (( $self->_data('status') eq 'active' && !$self->is_active )
|| $self->_active_iptables(id_domain => $self->id)) {
......@@ -313,7 +313,7 @@ sub _start_preconditions{
$self->_set_vm($vm_local, 1);
}
$self->_balance_vm();
$self->rsync() if !$self->_vm->is_local();
$self->rsync(request => $request) if !$self->is_volatile && !$self->_vm->is_local();
}
$self->_check_free_vm_memory();
#TODO: remove them and make it more general now we have nodes
......@@ -356,32 +356,15 @@ sub _search_already_started($self) {
sub _balance_vm($self) {
return if $self->{_migrated};
return if !$self->id_base;
my $sth = $$CONNECTOR->dbh->prepare(
"SELECT id FROM vms where vm_type=?"
);
$sth->execute($self->_vm->type);
my %vm_list;
for my $vm ($self->_vm->list_nodes) {
next if !$vm->is_active();
next if !$vm->is_active || $vm->free_memory < $MIN_FREE_MEMORY;
$vm_list{$vm->id} = scalar($vm->list_domains(active => 1)).".".$vm->free_memory;
}
my @sorted_vm = sort { $vm_list{$a} <=> $vm_list{$b} } keys %vm_list;
my $base = Ravada::Domain->open($self->id_base);
for my $id (@sorted_vm) {
if ( $base->base_in_vm($id) ) {
return if $id == $self->_vm->id;
my $base;
$base = Ravada::Domain->open($self->id_base) if $self->id_base;
my $vm_free = Ravada::VM->open($id);
my $vm_free = $self->_vm->balance_vm($base);
return if !$vm_free;
$self->migrate($vm_free);
return $id;
}
}
return;
$self->migrate($vm_free) if $vm_free->id != $self->_vm->id;
return $vm_free->id;
}
sub _update_description {
......@@ -1504,20 +1487,26 @@ sub clone {
$self->prepare_base($user)
}
my $id_base = $self->id;
my @args_copy = ();
push @args_copy, ( memory => $memory ) if $memory;
push @args_copy, ( request => $request ) if $request;
my $clone = $self->_vm->create_domain(
my $vm = $self->_vm;
if ($self->volatile_clones ) {
$vm = $vm->balance_vm();
} elsif( !$vm->is_local ) {
for my $node ($self->_vm->list_nodes) {
$vm = $node if $node->is_local;
}
}
my $clone = $vm->create_domain(
name => $name
,id_base => $id_base
,id_base => $self->id
,id_owner => $uid
,vm => $self->vm
,_vm => $self->_vm
,@args_copy
);
die if $clone->_data('id_vm') ne $vm->id;
return $clone;
}
......@@ -2601,6 +2590,9 @@ sub rsync($self, @args) {
}
my $rsync = File::Rsync->new(update => 1);
for my $file ( @$files ) {
my ($path) = $file =~ m{(.*)/};
my ($out, $err) = $node->run_command("/bin/mkdir","-p",$path);
die $err if $err;
$request->status("syncing","Tranferring $file to ".$node->host)
if $request;
my $src = $file;
......@@ -2613,7 +2605,7 @@ sub rsync($self, @args) {
}
if ($rsync->err) {
$request->status("done",join(" ",@{$rsync->err})) if $request;
die $rsync->err;
confess $rsync->err;
}
$node->refresh_storage_pools();
}
......@@ -2630,7 +2622,7 @@ sub _rsync_volumes_back($self, $request=undef) {
$self->_vm->refresh_storage_pools();
}
sub _pre_migrate($self, $node) {
sub _pre_migrate($self, $node, $request = undef) {
$self->_check_equal_storage_pools($node);
......@@ -2659,7 +2651,7 @@ sub _pre_migrate($self, $node) {
$self->_set_base_vm_db($node->id,0);
}
sub _post_migrate($self, $node) {
sub _post_migrate($self, $node, $request = undef) {
$self->_set_base_vm_db($node->id,1) if $self->is_base;
$self->_vm($node);
$self->_update_id_vm();
......@@ -2743,7 +2735,7 @@ sub set_base_vm($self, %args) {
} elsif ($value) {
$request->status("working", "Syncing base volumes to ".$vm->host)
if $request;
$self->rsync(node => $vm, request => $request);
$self->migrate($vm, $request);
}
return $self->_set_base_vm_db($vm->id, $value);
}
......
......@@ -1679,7 +1679,7 @@ sub _check_machine($self,$doc) {
$os_type->setAttribute( machine => 'pc');
}
sub migrate($self, $node) {
sub migrate($self, $node, $request=undef) {
my $dom;
eval { $dom = $node->vm->get_domain_by_name($self->name) };
die $@ if $@ && $@ !~ /libvirt error code: 42/;
......@@ -1698,7 +1698,7 @@ sub migrate($self, $node) {
}
$self->_set_spice_ip(1,$node->ip);
$self->rsync($node);
$self->rsync(node => $node, request => $request);
return if $self->is_removed;
$self->domain->managed_save_remove
......
......@@ -259,8 +259,6 @@ Adds a new volume to the domain
sub add_volume {
my $self = shift;
confess "Wrong arguments " if scalar@_ % 1;
confess "ERROR: add_volume on for local"
if !$self->is_local();
my %args = @_;
......@@ -272,8 +270,6 @@ sub add_volume {
confess "Volume path must be absolute , it is '$args{path}'"
if $args{path} !~ m{^/};
my %valid_arg = map { $_ => 1 } ( qw( name size path vm type swap target));
for my $arg_name (keys %args) {
......@@ -290,20 +286,12 @@ sub add_volume {
my $data = $self->_load();
$args{target} = _new_target($data) if !$args{target};
$data->{device}->{$args{name}} = \%args;
my $disk = $self->_config_file;
open my $lock,">>","$disk.lock" or die "I can't open lock: $disk.log: $!";
_lock($lock);
eval { DumpFile($self->_config_file, $data) };
_unlock($lock);
chomp $@;
die "readonly=".$self->readonly." ".$@ if $@;
my $device = $data->{device};
$device->{$args{name}} = \%args;
return if -e $args{path};
$self->_store(device => $device);
open my $out,'>>',$args{path} or die "$! $args{path}";
print $out Dumper($data->{device}->{$args{name}});
close $out;
$self->_vm->write_file($args{path}, Dumper($data->{device}->{$args{name}}));
}
......@@ -538,10 +526,11 @@ sub hibernate($self, $user) {
sub type { 'Void' }
sub migrate($self, $node) {
sub migrate($self, $node, $request=undef) {
$self->rsync(
node => $node
, files => [$self->_config_file ]
,request => $request
);
$self->rsync($node);
......
......@@ -11,6 +11,7 @@ Ravada::VM - Virtual Managers library for Ravada
use Carp qw( carp croak cluck);
use Data::Dumper;
use File::Path qw(make_path);
use Hash::Util qw(lock_hash);
use IPC::Run3 qw(run3);
use JSON::XS;
......@@ -298,8 +299,10 @@ sub _around_create_domain {
my $base;
my $id_base = delete $args{id_base};
$base = Ravada::Domain->open($id_base) if $id_base;
if ($id_base) {
$base = $self->search_domain_by_id($id_base)
or confess "Error: I can't find domain $id_base on ".$self->name;
}
confess "ERROR: User ".$owner->name." is not allowed to create machines"
unless $owner->is_admin
|| $owner->can_create_machine()
......@@ -502,10 +505,6 @@ sub _check_create_domain {
my %args = @_;
confess "ERROR: Domains can only be created at localhost got ".$self->host
unless $self->host eq 'localhost'
|| $self->host eq '127.0.0.1';
$self->_check_readonly(@_);
$self->_check_require_base(@_);
......@@ -790,6 +789,8 @@ Returns a list of virtual machine manager nodes of the same type as this.
=cut
sub list_nodes($self) {
return @{$self->{_nodes}} if $self->{_nodes};
my $sth = $$CONNECTOR->dbh->prepare(
"SELECT id FROM vms WHERE vm_type=?"
);
......@@ -800,6 +801,7 @@ sub list_nodes($self) {
push @nodes,(Ravada::VM->open($id))
}
$self->{_nodes} = \@nodes;
return @nodes;
}
......@@ -853,7 +855,7 @@ Returns if the domain is active.
sub is_active($self) {
return $self->_do_is_active() if $self->is_local;
return $self->_cached_active if time - $self->_cached_active_time < 5;
return $self->_cached_active if time - $self->_cached_active_time < 60;
return $self->_do_is_active();
}
......@@ -969,7 +971,12 @@ sub write_file( $self, $file, $contents ) {
}
sub _write_file_local( $self, $file, $contents ) {
confess "TODO";
my ($path) = $file =~ m{(.*)/};
make_path($path) or die "$! $path"
if ! -e $path;
CORE::open(my $out,">",$file) or die "$! $file";
print $out $contents;
close $out or die "$! $file";
}
sub create_iptables_chain($self,$chain) {
......@@ -1035,5 +1042,22 @@ sub iptables_list($self) {
return $ret;
}
sub balance_vm($self, $base=undef) {
my %vm_list;
for my $vm ($self->list_nodes) {
next if !$vm->enabled();
next if !$vm->is_active || $vm->free_memory < $Ravada::Domain::MIN_FREE_MEMORY;
my $key = scalar($vm->list_domains(active => 1)).".".$vm->free_memory;
$vm_list{$key} = $vm;
}
my @sorted_vm = map { $vm_list{$_} } sort keys %vm_list;
for my $vm (@sorted_vm) {
next if $base && !$base->base_in_vm($vm->id);
return $vm;
}
return;
}
1;
......@@ -603,10 +603,9 @@ sub create_volume {
$doc->findnodes('/volume/allocation/text()')->[0]->setData($allocation);
$doc->findnodes('/volume/capacity/text()')->[0]->setData($capacity);
}
my $vol = $self->storage_pool->create_volume($doc->toString);
die "volume $img_file does not exists after creating volume "
.$doc->toString()
if ! -e $img_file;
my $vol = $self->storage_pool->create_volume($doc->toString)
or die "volume $img_file does not exists after creating volume on ".$self->name." "
.$doc->toString();
return $img_file;
......@@ -754,7 +753,7 @@ sub _domain_create_common {
};
if ($@) {
my $out;
warn $@;
warn $self->name."\n".$@;
my $name_out = "/var/tmp/$args{name}.xml";
warn "Dumping $name_out";
open $out,">",$name_out and do {
......@@ -762,7 +761,7 @@ sub _domain_create_common {
};
close $out;
warn "$! $name_out" if !$out;
die $@ if !$dom;
confess $@ if !$dom;
}
my $domain = Ravada::Domain::KVM->new(
......@@ -813,20 +812,14 @@ sub _create_disk_qcow2 {
sub _clone_disk($self, $file_base, $file_out) {
my @cmd = ('qemu-img','create'
my @cmd = ('/usr/bin/qemu-img','create'
,'-f','qcow2'
,"-b", $file_base
,$file_out
);
my ($in, $out, $err);
run3(\@cmd,\$in,\$out,\$err);
if (! -e $file_out) {
warn "ERROR: Output file $file_out not created at ".join(" ",@cmd)."\n$err\n$out\n";
exit;
}
my ($out, $err) = $self->run_command(@cmd);
die $err if $err;
}
sub _create_disk_raw {
......
......@@ -331,7 +331,6 @@
$scope.getReqs= function() {
$http.get('/requests.json').then(function(response) {
$scope.requests=response.data;
$scope.getSingleMachine();
});
};
$scope.getReqs();
......
......@@ -220,7 +220,7 @@ sub test_iptables($node, $remote_ip, $local_ip, $local_port) {
ok(scalar @line == 1,$node->type." iptables should found only 1 found $remote_ip -> $local_ip:$local_port ".Dumper(\@line));
}
sub test_domain_no_remote {
sub test_domain_on_remote {
my ($vm_name, $node) = @_;
my $domain;
......@@ -231,7 +231,7 @@ sub test_domain_no_remote {
,id_iso => 1
);
};
like($@,qr'.',"Expecting no domain in remote node by now");
is($@,'',"Expecting no domain in remote node by now");
$domain->remove(user_admin) if $domain;
}
......@@ -1142,6 +1142,7 @@ sub test_status($node) {
#############################################################
clean();
clean_remote();
$Ravada::Domain::MIN_FREE_MEMORY = 256 * 1024;
......@@ -1206,7 +1207,7 @@ SKIP: {
test_domain_already_started($vm_name, $node);
test_clone_not_in_node($vm_name, $node);
test_rsync_newer($vm_name, $node);
test_domain_no_remote($vm_name, $node);
test_domain_on_remote($vm_name, $node);
my $domain2 = test_domain($vm_name, $node);
test_remove_domain_from_local($vm_name, $node, $domain2) if $domain2;
......
......@@ -378,14 +378,17 @@ sub _remove_old_domains_kvm {
next if $domain->get_name !~ /^$base_name/;
my $domain_name = $domain->get_name;
eval {
$domain->shutdown() if $domain->is_active;
sleep 1;
eval { $domain->destroy() if $domain->is_active };
warn $@ if $@;
}
if $domain->is_active;
warn "WARNING: error $@ trying to shutdown ".$domain_name
if $@ && $@ !~ /error code: 42,/;
$domain->shutdown();
sleep 1 if $domain->is_active;
};
warn "WARNING: error $@ trying to shutdown ".$domain_name." on ".$vm->name
if $@ && $@ !~ /error code: (42|55),/;
eval { $domain->destroy() if $domain->is_active };
warn $@ if $@;
warn "WARNING: error $@ trying to shutdown ".$domain_name." on ".$vm->name
if $@ && $@ !~ /error code: (42|55),/;
eval {
$domain->managed_save_remove()
......
......@@ -29,7 +29,8 @@ sub test_reuse_vm($node) {
my $clone1 = $domain->clone(name => new_domain_name, user => user_admin);
my $clone2 = $domain->clone(name => new_domain_name, user => user_admin);
is($clone1->_vm, $clone2->_vm);
is($clone1->_vm, $clone2->_vm, $clone1->_vm->name);
is($clone1->_vm->id, $clone2->_vm->id);
$clone1->migrate($node);
is($clone1->_data('id_vm'), $node->id);
......@@ -144,60 +145,184 @@ sub test_iptables_close($vm, $node) {
$domain->prepare_base(user_admin);
$domain->set_base_vm(vm => $node, user => user_admin) if !$domain->base_in_vm($node->id);
my $clone1 = $domain->clone(name => new_domain_name, user => user_admin);
$clone1->migrate($node) if $clone1->_vm->id != $node->id;
my $remote_ip1 = '1.1.1.1';
$clone1->start(user => user_admin, remote_ip => $remote_ip1);
my $clone2 = $domain->clone(name => new_domain_name, user => user_admin);
$clone2->migrate($vm) if $clone2->_vm->id != $vm->id;
my $remote_ip2 = '2.2.2.2';
$clone2->start(user => user_admin, remote_ip => $remote_ip2);
my ($local_ip1, $local_port1)
= $clone1->display(user_admin) =~ m{(\d+\.\d+\.\d+\.\d+)\:(\d+)};
my ($local_ip2, $local_port2)
= $clone2->display(user_admin) =~ m{(\d+\.\d+\.\d+\.\d+)\:(\d+)};
is($local_port1, $local_port2);
my ($remote_ip1, $remote_ip2) = ('1.1.1.1','2.2.2.2');
my ($clone_local, $clone_remote) = _create_2_clones_same_port($vm, $node, $domain
, $remote_ip1, $remote_ip2);
isnt($clone_local->_vm->id, $clone_remote->_vm->id);
my ( $local_port1 ) = $clone_local->display(user_admin)=~ m{://.*:(\d+)};
my ( $local_port2 ) = $clone_remote->display(user_admin)=~ m{://.*:(\d+)};
my @found = search_iptable_remote(
node => $vm
,remote_ip => $remote_ip1
,local_port => $local_port2
);
is(scalar @found,1,$vm->name." $remote_ip2:$local_port2".Dumper(\@found)) or exit;
@found = search_iptable_remote(
node => $node
,remote_ip => $remote_ip2
,local_port => $local_port2
);
is(scalar @found,1,$vm->name." $remote_ip2:$local_port2".Dumper(\@found)) or exit;
$clone1->shutdown_now(user_admin);
$clone_local->shutdown_now(user_admin);
@found = search_iptable_remote(
node => $node
node => $vm
,remote_ip => $remote_ip1
,local_port => $local_port1
);
is(scalar @found,0,$node->name." $remote_ip1:$local_port1".Dumper(\@found));
@found = search_iptable_remote(
node => $vm
node => $node
,remote_ip => $remote_ip2
,local_port => $local_port2
);
is(scalar @found,1,$vm->name." $remote_ip2:$local_port2".Dumper(\@found));
$clone2->remove(user_admin);
$clone1->remove(user_admin);
for my $clone0 ( $domain->clones) {
my $clone = Ravada::Domain->open($clone0->{id});
$clone->remove(user_admin);
}
$domain->remove(user_admin);
}
sub _create_2_clones_same_port($vm, $node, $base, $ip_local, $ip_remote) {
my $clone_local = $base->clone(name => new_domain_name, user => user_admin);
$clone_local->migrate($vm) if $clone_local->_vm->id != $vm->id;
my $clone_remote= $base->clone(name => new_domain_name, user => user_admin);
$clone_remote->migrate($node);
$clone_local->start(user => user_admin, remote_ip => $ip_local);
$clone_remote->start(user => user_admin, remote_ip => $ip_remote);
for (1 .. 10 ) {
my ($port_local) = $clone_local->display(user_admin) =~ m{://.*:(\d+)};
my ($port_remote) = $clone_remote->display(user_admin) =~ m{://.*:(\d+)};
return($clone_local, $clone_remote) if $port_local == $port_remote;
my $clone3 = $base->clone(name => new_domain_name, user => user_admin);
if ($port_local < $port_remote) {
$clone3->migrate($vm) if $clone3->_vm->id != $vm->id;
$clone_local = $clone3;
$clone_local->start(user => user_admin, remote_ip => $ip_local);
} else {
$clone3->migrate($node) if $clone3->_vm->id != $node->id;
$clone_remote = $clone3;
$clone_remote->start(user => user_admin, remote_ip => $ip_remote);
}
}
die;
}
sub test_volatile($vm, $node) {
my $base = create_domain($vm);
$base->prepare_base(user_admin);
$base->set_base_vm(user => user_admin, node => $node);
$base->volatile_clones(1);
my @clones;
for ( 1 .. 4 ) {
my $clone = $base->clone(user => user_admin, name => new_domain_name);
$clone->start(user_admin) if !$clone->is_active;
push @clones,($clone);
last if $clone->_vm->id == $node->id;
}
is($clones[-1]->_vm->id, $node->id);
for (@clones) {
$_->remove(user_admin);
}
$base->remove(user_admin);
}
sub test_volatile_req($vm, $node) {
my $base = create_domain($vm);
$base->prepare_base(user_admin);
$base->set_base_vm(user => user_admin, node => $node);
$base->volatile_clones(1);
ok($base->base_in_vm($node->id));