Unverified Commit 3bcdece3 authored by Francesc Guasch's avatar Francesc Guasch Committed by GitHub
Browse files

Fix node out of memory (#1301)

fix(backend): checks node has enough memory

* refactor(test): now it only lists bases
* sets a minimal default memory if unset
* defaults to localhost if node full
* test fill memory

issue #1297
parent 4e7e78ba
......@@ -1309,7 +1309,7 @@ sub _upgrade_tables {
$self->_upgrade_table('vms','is_active',"int DEFAULT 0");
$self->_upgrade_table('vms','enabled',"int DEFAULT 1");
$self->_upgrade_table('vms','min_free_memory',"text DEFAULT NULL");
$self->_upgrade_table('vms','min_free_memory',"int DEFAULT 600000");
$self->_upgrade_table('vms', 'max_load', 'int not null default 10');
$self->_upgrade_table('vms', 'active_limit','int DEFAULT NULL');
$self->_upgrade_table('vms', 'base_storage','varchar(64) DEFAULT NULL');
......@@ -3885,11 +3885,14 @@ sub import_domain {
my $self = shift;
my %args = @_;
my $vm_name = $args{vm} or die "ERROR: mandatory argument vm required";
my $name = $args{name} or die "ERROR: mandatory argument domain name required";
my $user_name = $args{user} or die "ERROR: mandatory argument user required";
my $vm_name = delete $args{vm} or die "ERROR: mandatory argument vm required";
my $name = delete $args{name} or die "ERROR: mandatory argument domain name required";
my $user_name = delete $args{user} or die "ERROR: mandatory argument user required";
my $spinoff_disks = delete $args{spinoff_disks};
$spinoff_disks = 1 if !defined $spinoff_disks;
my $import_base = delete $args{import_base};
confess "Error : unknown args ".Dumper(\%args) if keys %args;
my $vm = $self->search_vm($vm_name) or die "ERROR: unknown VM '$vm_name'";
my $user = Ravada::Auth::SQL->new(name => $user_name);
......@@ -3899,7 +3902,7 @@ sub import_domain {
eval { $domain = $self->search_domain($name) };
die "ERROR: Domain '$name' already in RVD" if $domain;
return $vm->import_domain($name, $user, $spinoff_disks);
return $vm->import_domain($name, $user, $spinoff_disks, $import_base);
}
sub _cmd_enforce_limits($self, $request=undef) {
......
......@@ -311,9 +311,17 @@ sub _around_start($orig, $self, @arg) {
for (;;) {
eval { $self->_start_checks(@arg) };
my $error = $@;
if ($error && $error =~/base file not found/ && !$self->_vm->is_local) {
$self->_request_set_base();
next;
if ($error) {
if ( $error =~/base file not found/ && !$self->_vm->is_local) {
$self->_request_set_base();
next;
} elsif ($error =~ /No free memory/) {
warn $error;
die $error if $self->is_local;
my $vm_local = $self->_vm->new( host => 'localhost' );
$self->migrate($vm_local);
next;
}
}
die $error if $error;
if (!defined $listen_ip) {
......@@ -903,13 +911,17 @@ sub _check_has_clones {
sub _check_free_vm_memory {
my $self = shift;
return if !$self->_vm->min_free_memory;
my $vm_free_mem = $self->_vm->free_memory;
return if $vm_free_mem > $self->_vm->min_free_memory;
my $domain_memory = $self->info(Ravada::Utils::user_daemon)->{memory};
my $min_free_memory = ($self->_vm->min_free_memory or $MIN_FREE_MEMORY)+$domain_memory;
return if $vm_free_mem > $min_free_memory;
$self->_data(status => 'down');
my $msg = "Error: No free memory in ".$self->_vm->name.". Only "._gb($vm_free_mem)." out of "
._gb($self->_vm->min_free_memory)." GB required.\n";
._gb($min_free_memory)." GB required.\n";
die $msg;
}
......@@ -1777,7 +1789,8 @@ sub _remove_domain_cascade($self,$user, $cascade = 1) {
eval { $vm = Ravada::VM->open($instance->{id_vm}) };
die $@ if $@ && $@ !~ /I can't find VM/i;
my $domain;
eval { $domain = $vm->search_domain($domain_name) };
$@ = '';
eval { $domain = $vm->search_domain($domain_name) } if $vm;
warn $@ if $@;
$domain->remove($user, $cascade) if $domain;
$sth_delete->execute($instance->{id});
......
......@@ -780,6 +780,7 @@ sub change_hardware($self, $hardware, $index, $data) {
my $hardware_def = $self->_value('hardware');
my $devices = $hardware_def->{$hardware};
confess "Error: $hardware not found ".Dumper($hardware_def) if !$devices;
die "Error: Missing hardware $hardware\[$index], only ".scalar(@$devices)." found"
if $index > scalar(@$devices);
......
......@@ -495,9 +495,9 @@ sub _check_duplicate_name($self, $name) {
sub _around_import_domain {
my $orig = shift;
my $self = shift;
my ($name, $user, $spinoff) = @_;
my ($name, $user, $spinoff, $import_base) = @_;
my $domain = $self->$orig($name, $user);
my $domain = $self->$orig($name, $user, $spinoff);
$domain->_insert_db(name => $name, id_owner => $user->id);
......@@ -506,9 +506,25 @@ sub _around_import_domain {
if $ENV{TERM} && $0 !~ /\.t$/;
$domain->spinoff();
}
if ($import_base) {
$self->_import_base($domain);
}
return $domain;
}
sub _import_base($self, $domain) {
my @img;
for my $vol ( $domain->list_volumes_info ) {
next if !$vol->file;
next if !$vol->backing_file;
push @img,[$vol->backing_file, $vol->info->{target}];
}
return if !@img;
$domain->_prepare_base_db(@img);
$domain->_post_prepare_base( Ravada::Utils::user_daemon());
}
############################################################
#
......@@ -1535,7 +1551,7 @@ sub balance_vm($self, $base=undef) {
$vm->enabled(0) if !$vm->is_local();
next;
}
next if $free_memory < $Ravada::Domain::MIN_FREE_MEMORY;
next if $free_memory < $min_memory;
my $n_active = $vm->_count_domains(status => 'active')
+ $vm->_count_domains(status => 'starting');
......
......@@ -2212,7 +2212,7 @@ Imports a KVM domain in Ravada
=cut
sub import_domain($self, $name, $user) {
sub import_domain($self, $name, $user, $spinoff=1) {
my $domain_kvm;
eval { $domain_kvm = $self->vm->get_domain_by_name($name) };
......
......@@ -321,7 +321,7 @@ sub search_volume_path_re {
}
sub import_domain($self, $name, $user) {
sub import_domain($self, $name, $user, $backing_file) {
my $file = $self->dir_img."/$name.yml";
......
......@@ -36,7 +36,6 @@ sub test_new_domain {
ok($@,"Expecting failed because we ran out of free RAM");
return;
}
warn $vm->type." ".$freemem;
ok(!$@,"Domain $name not created: $@");
ok($domain,"Domain not created") or return;
......@@ -126,8 +125,8 @@ for my $vm_name (vm_names()) {
SKIP: {
my $msg = "SKIPPED test: No $vm_name backend found";
my $vm = $RVD_BACK->search_vm($vm_name);
$msg = "SKIPPED: todo review overcommitting issue #1164";
$vm = undef;
# $msg = "SKIPPED: todo review overcommitting issue #1164";
#$vm = undef;
if ($vm_name eq 'KVM' && $>) {
$msg = "SKIPPED test: $vm_name must be run from root";
......
......@@ -31,6 +31,7 @@ require Exporter;
@EXPORT = qw(base_domain_name new_domain_name rvd_back remove_old_disks remove_old_domains create_user user_admin wait_request rvd_front init init_vm clean new_pool_name new_volume_name
create_domain
import_domain
test_chain_prerouting
find_ip_rule
search_id_iso
......@@ -174,6 +175,20 @@ sub vm_names {
confess;
}
sub import_domain($vm, $name, $import_base=0) {
diag("importing domain $name");
my $t0 = time;
my $domain = $RVD_BACK->import_domain(
vm => $vm
,name => $name
,user => user_admin->name
,spinoff_disks => 0
,import_base => $import_base
);
diag("imported $name ".(time - $t0));
return $domain;
}
sub create_domain {
my $vm_name = shift;
my $user = (shift or $USER_ADMIN);
......
......@@ -92,7 +92,7 @@ sub test_bases($t, $bases) {
$n_machines += $n_clones;
my @machines = list_machines($t);
is( scalar(@machines), $n_machines, Dumper(\@machines)) or exit;
is( scalar(@machines), scalar(@$bases), Dumper(\@machines)) or exit;
}
}
......
......@@ -12,6 +12,7 @@ use Test::Ravada;
no warnings "experimental::signatures";
use feature qw(signatures);
my $BASE_NAME = "zz-test-base";
use_ok('Ravada');
init();
......@@ -209,12 +210,16 @@ sub test_iptables_close($vm, $node) {
_remove_domain($domain);
}
sub _remove_domain($domain) {
sub _remove_clones($domain) {
_remove_domain($domain,0);
}
sub _remove_domain($domain, $remove_base=0) {
for my $clone0 ( $domain->clones) {
my $clone = Ravada::Domain->open($clone0->{id});
$clone->remove(user_admin);
}
$domain->remove(user_admin);
$domain->remove(user_admin) if $remove_base;
}
......@@ -843,6 +848,52 @@ sub test_base_unset($vm, $node) {
_remove_domain($base);
}
sub test_fill_memory($vm, $node, $migrate) {
#TODO: Void VMs
return if $vm->type eq 'Void';
diag("Testing fill memory ".$vm->type.", migrate=$migrate");
my $base = rvd_back->search_domain($BASE_NAME);
$base = import_domain('KVM', $BASE_NAME, 1) if !$base;
if (!$base) {
diag("SKIPPING: base $BASE_NAME must be installed to test");
return;
}
$base->prepare_base(user_admin) if !$base->is_base;
$base->set_base_vm(vm => $node, user => user_admin);
wait_request();
my $master_free_memory = $vm->free_memory;
my $node_free_memory = $node->free_memory;
my $error;
my %nodes;
my @clones;
for ( 1 .. 100 ) {
my $clone_name = new_domain_name();
my $req = Ravada::Request->create_domain(
name => $clone_name
,id_owner => user_admin->id
,id_base => $base->id
);
wait_request(debug => 0);
is($req->error, '');
is($req->status,'done');
push @clones,($clone_name);
my $clone = rvd_back->search_domain($clone_name) or last;
ok($clone,"Expecting clone $clone_name") or exit;
$clone->migrate($node) if $migrate;
eval { $clone->start(user_admin) };
$error = $@;
like($error, qr/(^$|No free memory)/);
last if $error;
$nodes{$clone->_vm->name}++;
}
ok(exists $nodes{$vm->name},"Expecting some clones to node ".$vm->name." ".$vm->id);
ok(exists $nodes{$node->name},"Expecting some clones to node ".$node->name." ".$node->id);
_remove_clones($base);
}
##################################################################################
clean();
......@@ -881,6 +932,8 @@ for my $vm_name ( 'Void', 'KVM') {
};
is($node->is_local,0,"Expecting ".$node->name." ".$node->ip." is remote" ) or BAIL_OUT();
test_fill_memory($vm, $node, 0); # balance
test_fill_memory($vm, $node, 1); # migrate
test_create_active($vm, $node);
test_base_unset($vm,$node);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment