Commit c53053cd authored by Francesc Guasch's avatar Francesc Guasch
Browse files

Merge branch 'develop' of https://github.com/UPC/ravada into develop

parents 75defdf0 0bd884c4
......@@ -3,7 +3,7 @@ package Ravada;
use warnings;
use strict;
our $VERSION = '0.9.0';
our $VERSION = '0.10.0';
use Carp qw(carp croak);
use Data::Dumper;
......@@ -1406,6 +1406,11 @@ sub _sql_insert_defaults($self){
,name => 'debug'
,value => 0
}
,{
id_parent => $id_backend
,name => 'delay_migrate_back'
,value => 600
}
]
);
my %field = ( settings => 'name' );
......@@ -1542,6 +1547,8 @@ sub _upgrade_tables {
$self->_upgrade_screenshots();
}
$self->_upgrade_table('domains','shared_storage','varchar(254)');
$self->_upgrade_table('domains','post_shutdown','int not null default 0');
$self->_upgrade_table('domains_network','allowed','int not null default 1');
......@@ -2394,6 +2401,7 @@ sub process_requests {
$self->_wait_pids();
$self->_kill_stale_process();
$self->_kill_dead_process();
my $sth = $CONNECTOR->dbh->prepare("SELECT id,id_domain FROM requests "
." WHERE "
......@@ -2424,8 +2432,9 @@ sub process_requests {
for my $req (sort { $a->priority <=> $b->priority } @reqs) {
next if $req eq 'refresh_vms' && scalar@reqs > 2;
next if !$req->id;
warn "[$request_type] $$ executing request ".$req->id." ".$req->status()." retry=".($req->retry or '<UNDEF>')." "
warn "[$request_type] $$ executing request id=".$req->id." ".$req->status()." retry=".($req->retry or '<UNDEF>')." "
.$req->command
." ".Dumper($req->args) if $DEBUG || $debug;
......@@ -2567,13 +2576,40 @@ sub _kill_stale_process($self) {
"SELECT id,pid,command,start_time "
." FROM requests "
." WHERE start_time<? "
." AND command = 'refresh_vms'"
." AND ( command = 'refresh_vms' or command = 'screenshot' or command = 'set_time' "
." OR command = 'open_exposed_ports' "
.") "
." AND status <> 'done' "
." AND pid IS NOT NULL "
." AND start_time IS NOT NULL "
);
$sth->execute(time - $TIMEOUT_STALE_PROCESS);
while (my ($id, $pid, $command, $start_time) = $sth->fetchrow) {
if ($pid == $$ ) {
warn "HOLY COW! I should kill pid $pid stale for ".(time - $start_time)
." seconds, but I won't because it is myself";
my $request = Ravada::Request->open($id);
$request->status('done',"Stale process pid=$pid");
next;
}
my $request = Ravada::Request->open($id);
$request->stop();
}
$sth->finish;
}
sub _kill_dead_process($self) {
my $sth = $CONNECTOR->dbh->prepare(
"SELECT id,pid,command,start_time "
." FROM requests "
." WHERE start_time<? "
." AND status = 'working' "
." AND pid IS NOT NULL "
);
$sth->execute(time - 2);
while (my ($id, $pid, $command, $start_time) = $sth->fetchrow) {
next if -e "/proc/$pid";
if ($pid == $$ ) {
warn "HOLY COW! I should kill pid $pid stale for ".(time - $start_time)
." seconds, but I won't because it is myself";
......@@ -2581,10 +2617,12 @@ sub _kill_stale_process($self) {
}
my $request = Ravada::Request->open($id);
$request->stop();
warn "stopping ".$request->id." ".$request->command;
}
$sth->finish;
}
sub _domain_working {
my $self = shift;
my ($id_domain, $id_request) = @_;
......@@ -2663,7 +2701,6 @@ sub _execute {
}
$request->status('working','') unless $request->status() eq 'waiting';
$request->pid($$);
$request->start_time(time);
$request->error('');
if ($dont_fork || !$CAN_FORK) {
......@@ -3140,7 +3177,7 @@ sub _cmd_start {
$self->_remove_unnecessary_downs($domain);
my @args = ( user => $user );
my @args = ( user => $user, request => $request );
push @args, ( remote_ip => $request->defined_arg('remote_ip') )
if $request->defined_arg('remote_ip');
......@@ -3470,6 +3507,7 @@ sub _cmd_force_shutdown {
die "Unknown domain '$id_domain'\n" if !$domain;
my $user = Ravada::Auth::SQL->search_by_id( $uid);
die "Error: unknown user id=$uid in request= ".$request->id if !$user;
$domain->force_shutdown($user,$request);
......@@ -3734,7 +3772,7 @@ sub _migrate_base($self, $domain, $node, $uid, $request) {
, id_vm => $node->id
, uid => $uid
);
$request->after_request($req_base);
$request->after_request($req_base->id) if $req_base;
die "Base ".$base->name." still not prepared in node ".$node->name.". Retry\n";
}
......@@ -3779,6 +3817,23 @@ sub _cmd_migrate($self, $request) {
}
sub _cmd_rsync_back($self, $request) {
my $uid = $request->args('uid');
my $id_domain = $request->args('id_domain') or die "ERROR: Missing id_domain";
my $domain = Ravada::Domain->open($id_domain);
return if $domain->is_active;
my $user = Ravada::Auth::SQL->search_by_id($uid);
die "Error: user ".$user->name." not allowed to migrate domain ".$domain->name
unless $user->is_operator;
my $node = Ravada::VM->open($request->args('id_node'));
$domain->_rsync_volumes_back($node, $request);
}
sub _clean_requests($self, $command, $request=undef, $status='requested') {
my $query = "DELETE FROM requests "
." WHERE command=? "
......@@ -3886,6 +3941,8 @@ sub _refresh_active_domain($self, $domain, $active_domain) {
$domain->info(Ravada::Utils::user_daemon) if $is_active;
$active_domain->{$domain->id} = $is_active;
$domain->_post_shutdown()
if $domain->_data('status') eq 'shutdown' && !$domain->_data('post_shutdown');
}
sub _refresh_down_domains($self, $active_domain, $active_vm) {
......@@ -3911,6 +3968,8 @@ sub _refresh_down_domains($self, $active_domain, $active_vm) {
$req->status('done');
}
}
$domain->_post_shutdown()
if $domain->_data('status') eq 'shutdown' && !$domain->_data('post_shutdown');
}
}
......@@ -4078,6 +4137,7 @@ sub _req_method {
,start_node => \&_cmd_start_node
,connect_node => \&_cmd_connect_node
,migrate => \&_cmd_migrate
,rsync_back => \&_cmd_rsync_back
#users
,post_login => \&_cmd_post_login
......@@ -4354,6 +4414,12 @@ sub _cmd_open_exposed_ports($self, $request) {
$domain->open_exposed_ports();
}
=head2 set_debug_value
Sets debug global variable from setting
=cut
sub set_debug_value($self) {
$DEBUG = $self->setting('backend/debug');
}
......
......@@ -38,6 +38,8 @@ our %PROPAGATE_FIELD = map { $_ => 1} qw( run_timeout shutdown_disconnected);
our $TIME_CACHE_NETSTAT = 60; # seconds to cache netstat data output
our $RETRY_SET_TIME=10;
our $DEBUG_RSYNC = 0;
_init_connector();
requires 'name';
......@@ -235,6 +237,7 @@ sub _check_clean_shutdown($self) {
return if !$self->is_known || $self->readonly || $self->is_volatile;
if (( $self->_data('status') eq 'active' && !$self->is_active )
|| ($self->_data('status') eq 'shutdown' && !$self->_data('post_shutdown'))
|| $self->_active_iptables(id_domain => $self->id)) {
$self->_post_shutdown();
}
......@@ -296,6 +299,7 @@ sub _vm_disconnect {
sub _around_start($orig, $self, @arg) {
$self->_data( 'post_shutdown' => 0);
$self->_start_preconditions(@arg);
my %arg;
......@@ -305,6 +309,7 @@ sub _around_start($orig, $self, @arg) {
$arg{user} = $arg[0];
}
my $request = delete $arg{request};
my $listen_ip = delete $arg{listen_ip};
my $remote_ip = $arg{remote_ip};
......@@ -319,7 +324,7 @@ sub _around_start($orig, $self, @arg) {
warn $error;
die $error if $self->is_local;
my $vm_local = $self->_vm->new( host => 'localhost' );
$self->migrate($vm_local);
$self->migrate($vm_local, $request);
next;
}
}
......@@ -397,7 +402,7 @@ sub _start_checks($self, @args) {
my $vm = $vm_local;
my ($id_vm, $request);
if (!scalar(@args) % 2) {
if (!(scalar(@args) % 2)) {
my %args = @args;
# We may be asked to start the machine in a specific id_vmanager
......@@ -435,9 +440,22 @@ sub _start_checks($self, @args) {
if ($id_vm) {
$self->_set_vm($vm);
} else {
$self->_balance_vm();
$self->_balance_vm($request);
}
if ( !$self->is_volatile && !$self->_vm->is_local() ) {
my $args = {
uid => Ravada::Utils::user_daemon->id
,id_domain => $self->id_base
,id_vm => $self->_vm->id
};
my $req;
$req = Ravada::Request->set_base_vm(%$args)
unless Ravada::Request::_duplicated_request(undef
,'set_base_vm', encode_json($args));
$self->rsync(request => $request);
}
$self->rsync(request => $request) if !$self->is_volatile && !$self->_vm->is_local();
}
$self->_check_free_vm_memory();
#TODO: remove them and make it more general now we have nodes
......@@ -497,7 +515,7 @@ sub _search_already_started($self, $fast = 0) {
return keys %started;
}
sub _balance_vm($self) {
sub _balance_vm($self, $request=undef) {
return if $self->{_migrated};
my $base;
......@@ -509,7 +527,7 @@ sub _balance_vm($self) {
return if !$vm_free;
last if $vm_free->id == $self->_vm->id;
eval { $self->migrate($vm_free) };
eval { $self->migrate($vm_free, $request) };
last if !$@;
if ($@ && $@ =~ /file not found/i) {
$base->_set_base_vm_db($vm_free->id,0) unless $vm_free->is_local;
......@@ -825,7 +843,7 @@ sub _pre_prepare_base($self, $user, $request = undef ) {
$self->_post_remove_base();
if (!$self->is_local) {
my $vm_local = Ravada::VM->open( type => $self->vm );
$self->migrate($vm_local);
$self->migrate($vm_local, $request);
}
if ($self->id_base ) {
$self->spinoff();
......@@ -1158,7 +1176,11 @@ sub _data($self, $field, $value=undef, $table='domains') {
$field_id = 'id_domain';
}
if (defined $value) {
if (defined $value &&
( !exists $self->{$data}->{$field}
|| !defined $self->{$data}->{$field}
|| $self->{$data}->{$field} ne $value )
) {
confess "Domain ".$self->name." is not in the DB"
if !$self->is_known();
......@@ -1972,6 +1994,9 @@ sub is_locked {
." WHERE id_domain=? AND status <> 'done'"
." AND command <> 'open_iptables' "
." AND command <> 'set_time'"
." AND command <> 'rsync_back'"
." AND command <> 'refresh_machine'"
." AND command <> 'screenshot'"
);
$sth->execute($self->id);
my ($id, $at_time) = $sth->fetchrow;
......@@ -2436,8 +2461,10 @@ sub _post_shutdown {
my $is_active = $self->is_active;
$self->_data(status => 'shutdown')
if $self->is_known && !$self->is_volatile && !$is_active;
if ( $self->is_known && !$self->is_volatile && !$is_active ) {
$self->_data(status => 'shutdown');
$self->_data(post_shutdown => 1);
}
if ($self->is_known && $self->id_base) {
my @disks = $self->list_disks();
......@@ -2484,8 +2511,14 @@ sub _post_shutdown {
# only if not volatile
my $request;
$request = $arg{request} if exists $arg{request};
$self->_rsync_volumes_back( $request )
if !$self->is_local && !$is_active && !$self->is_volatile;
if ( !$self->is_local && !$self->is_volatile && $self->has_non_shared_storage()) {
my $req = Ravada::Request->rsync_back(
uid => Ravada::Utils::user_daemon->id
,id_domain => $self->id
,id_node => $self->_vm->id
,at => time + Ravada::setting(undef,"/backend/delay_migrate_back")
);
}
$self->needs_restart(0) if $self->is_known()
&& $self->needs_restart()
......@@ -2792,6 +2825,7 @@ sub _open_exposed_port($self, $internal_port, $name, $restricted) {
sub _open_iptables_state($self) {
my $local_net = $self->ip;
return if !$local_net;
$local_net =~ s{(.*)\.\d+}{$1.0/24};
$self->_vm->iptables_unique(
......@@ -3921,7 +3955,7 @@ sub rsync($self, @args) {
or confess "No Connection to ".$self->_vm->host;
}
my $vm_local = $self->_vm->new( host => 'localhost' );
my $rsync = File::Rsync->new(update => 1, sparse => 1);
my $rsync = File::Rsync->new(update => 1, sparse => 1, times => 1);
for my $file ( @$files ) {
my ($path) = $file =~ m{(.*)/};
my ($out, $err) = $node->run_command("/bin/mkdir","-p",$path);
......@@ -3935,9 +3969,15 @@ sub rsync($self, @args) {
} else {
next if $vm_local->shared_storage($node, $path);
}
$request->status("syncing","Tranferring $file to ".$node->host)
if $request;
next if _check_stat($file, $vm_local, $node);
my $msg = $self->_msg_log_rsync($file, $node, "rsync", $request);
$request->status("syncing",$msg) if $request;
warn "$msg\n" if $DEBUG_RSYNC;
my $t0 = time;
$rsync->exec(src => $src, dest => $dst);
warn "Domain::rsync ".(time - $t0)." seconds $file" if $DEBUG_RSYNC;
}
if ($rsync->err) {
$request->status("done",join(" ",@{$rsync->err})) if $request;
......@@ -3948,13 +3988,40 @@ sub rsync($self, @args) {
$node->refresh_storage_pools();
}
sub _rsync_volumes_back($self, $request=undef) {
my $rsync = File::Rsync->new(update => 1);
sub _check_stat($file, $vm1, $vm2) {
return if !$vm2->file_exists($file);
my @cmd = ("stat","-c",'"%A %s %y"',$file);
my ($out1, $err1) = $vm1->run_command(@cmd);
my ($out2, $err2) = $vm2->run_command(@cmd);
$out1 =~ s/^"(.*)"$/$1/;
$out2 =~ s/^"(.*)"$/$1/;
warn "$file\n$out1\n$out2\n" if $DEBUG_RSYNC;
return $out1 eq $out2;
}
sub _msg_log_rsync($self, $file, $node, $sub, $request) {
my $msg = '';
$msg .= " [req=".$request->id.",".$request->command."] " if $request;
$msg .="Domain::$sub ".$self->name.". Tranferring $file to ".$node->host;
return $msg;
}
sub _rsync_volumes_back($self, $node, $request=undef) {
my $rsync = File::Rsync->new(update => 1, sparse => 1, times => 1);
my $vm_local = $self->_vm->new( host => 'localhost' );
for my $file ( $self->list_volumes() ) {
my ($dir) = $file =~ m{(.*)/.*};
next if $vm_local->shared_storage($self->_vm,$dir);
$rsync->exec(src => 'root@'.$self->_vm->host.":".$file ,dest => $file );
next if $vm_local->shared_storage($node, $dir);
my $msg = $self->_msg_log_rsync($file, $node, "rsync_back", $request);
$request->status("syncing",$msg) if $request;
warn "$msg\n" if $DEBUG_RSYNC;
my $t0 = time;
$rsync->exec(src => 'root@'.$node->host.":".$file ,dest => $file );
warn "Domain::rsync_volumes_back ".(time - $t0)." seconds $file" if $DEBUG_RSYNC;
if ( $rsync->err ) {
$request->status("done",join(" ",@{$rsync->err})) if $request;
last;
......@@ -5183,6 +5250,43 @@ sub list_instances($self) {
return @instances;
}
=head2 has_non_shared_storage
Return wether this virtual machine has non shared storage volumes
=cut
sub has_non_shared_storage($self, $node=$self->_vm->new(host => 'localhost')) {
my $id1 = $self->_vm->id;
my $id2 = $node->id;
confess "Error: both nodes are the same ".$self->_vm->name
." and ".$node->name
if $id1 == $id2;
my $nodes_id = join(",",sort ($id1,$id2));
my $shared_storage_cache = $self->_data('shared_storage');
my $shared_storage = {};
$shared_storage = decode_json($shared_storage_cache)
if $shared_storage_cache;
my $has_non_shared;
if ($shared_storage && exists $shared_storage->{$nodes_id}) {
$has_non_shared = $shared_storage->{$nodes_id};
return $has_non_shared if defined $has_non_shared;
}
for my $file ( $self->list_volumes ) {
my ($dir) = $file =~ m{(.*)/};
$has_non_shared = !$self->_vm->shared_storage($node, $dir);
last if $has_non_shared
}
$shared_storage->{$nodes_id}= $has_non_shared;
$self->_data('shared_storage' => encode_json($shared_storage));
return $has_non_shared;
}
sub _base_in_nodes($self) {
my $base = Ravada::Front::Domain->open($self->id_base);
confess "Error: no id_base ".($self->id_base or '<NULL>')
......
......@@ -100,6 +100,7 @@ our %VALID_ARG = (
,refresh_machine => { id_domain => 1, uid => 1 }
,rebase => { uid => 1, id_base => 1, id_domain => 1 }
,set_time => { uid => 1, id_domain => 1 }
,rsync_back => { uid => 1, id_domain => 1, id_node => 1 }
# ports
,expose => { uid => 1, id_domain => 1, port => 1, name => 2, restricted => 2, id_port => 2}
,remove_expose => { uid => 1, id_domain => 1, port => 1}
......@@ -143,6 +144,8 @@ our %CMD_NO_DUPLICATE = map { $_ => 1 }
qw(
set_base_vm
remove_base_vm
rsync_back
cleanup
);
our $TIMEOUT_SHUTDOWN = 120;
......@@ -152,31 +155,40 @@ our $CONNECTOR;
our %COMMAND = (
long => {
limit => 4
,priority => 4
,priority => 10
} #default
,huge => {
limit => 1
,commands => ['download']
,priority => 5
# list from low to high priority
,disk_low_priority => {
limit => 2
,commands => ['rsync_back','check_storage', 'refresh_vms']
,priority => 30
}
,disk => {
limit => 1
,commands => ['prepare_base','remove_base','set_base_vm','rebase_volumes'
, 'remove_base_vm'
, 'screenshot'
, 'migrate'
, 'cleanup'
]
,priority => 6
,priority => 20
}
,important=> {
limit => 20
,priority => 1
,commands => ['clone','start','start_clones','shutdown_clones','create','open_iptables','list_network_interfaces','list_isos']
,huge => {
limit => 1
,commands => ['download']
,priority => 15
}
,secondary => {
limit => 50
,priority => 2
,commands => ['shutdown','shutdown_now', 'manage_pools']
,priority => 4
,commands => ['shutdown','shutdown_now', 'manage_pools','enforce_limits', 'set_time']
}
,important=> {
limit => 20
,priority => 1
,commands => ['clone','start','start_clones','shutdown_clones','create','open_iptables','list_network_interfaces','list_isos','ping_backend','refresh_machine']
}
);
lock_hash %COMMAND;
......@@ -356,12 +368,37 @@ sub start_domain {
confess "ERROR: choose either id_domain or name "
if $args->{id_domain} && $args->{name};
_remove_low_priority_requests($args->{id_domain} or $args->{name});
my $self = {};
bless($self,$class);
return $self->_new_request(command => 'start' , args => $args);
}
sub _remove_low_priority_requests($id_domain) {
_init_connector() if !$CONNECTOR || !$$CONNECTOR;
if ($id_domain !~ /^\d+$/) {
$id_domain = _search_domain_id(undef,$id_domain);
}
for my $command (sort @{$COMMAND{disk_low_priority}->{commands}}) {
my $sth = $$CONNECTOR->dbh->prepare("SELECT id FROM requests "
." WHERE command=? AND id_domain=? "
." AND status <> 'done' "
);
$sth->execute($command, $id_domain);
while ( my ($id_request) = $sth->fetchrow ) {
my $req = Ravada::Request->open($id_request);
$req->stop();
warn "Stopping request $id_request";
}
}
}
=head2 start_clones
Requests to start the clones of a base
......@@ -517,6 +554,7 @@ sub new_request($self, $command, @args) {
}
sub _duplicated_request($self=undef, $command=undef, $args=undef) {
_init_connector() if !$CONNECTOR || !$$CONNECTOR;
my $args_d;
if ($self) {
......@@ -530,7 +568,7 @@ sub _duplicated_request($self=undef, $command=undef, $args=undef) {
delete $args_d->{uid};
delete $args_d->{at};
my $sth = $$CONNECTOR->dbh->prepare(
"SELECT id,args FROM requests WHERE status='requested'"
"SELECT id,args FROM requests WHERE (status <> 'done')"
." AND command=?"
);
$sth->execute($command);
......@@ -713,6 +751,14 @@ sub _search_domain_name {
return $sth->fetchrow;
}
sub _search_domain_id($self,$domain_name) {
my $sth = $$CONNECTOR->dbh->prepare("SELECT id FROM domains where name=?");
$sth->execute($domain_name);
return $sth->fetchrow;
}
sub _send_message {
my $self = shift;
my $status = shift;
......@@ -1047,7 +1093,10 @@ sub count_requests($self) {
sub requests_limit($self, $type = $self->type) {
confess "Error: no requests of type $type" if !exists $COMMAND{$type};
return $COMMAND{$type}->{limit};
my $limit = $COMMAND{$type}->{limit};
return $limit;
}
=head2 domain_autostart
......