Unverified Commit e9c54f1a authored by Francesc Guasch's avatar Francesc Guasch Committed by GitHub
Browse files

Feature #953 wol (#965)

* feature(backend): start and stop remote node via WOL

Added new field in the VM to store MAC address

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* feature(nodes): start and stop remote node

issue #953

* feature(backend): start and stop remote node via WOL

Added new field in the VM to store MAC address

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* feature(nodes): start and stop remote node

issue #953

* wip(frontend): shutdown and start nodes request

issue #953

* feature(backend): start and stop remote node via WOL

Added new field in the VM to store MAC address

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* feature(nodes): start and stop remote node

issue #953

* feature(backend): start and stop remote node via WOL

Added new field in the VM to store MAC address

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* feature(nodes): start and stop remote node

issue #953

* wip(frontend): shutdown and start nodes request

issue #953

* feature(backend): check node connection

issue #953

* feature(frontend): show status of node connection

issue #953

* refactor(nodes): ping node with all IPs

* wip(backend): show error if start request failed

issue #953

* refactor(frontend): removed debug

* refactor(frontend): show error messages prettier

issue #953

* refactor(frontend): check node status after operations

issue #953

* refactor(requests): deal with stopping requests and set done

issue #953

* refactor(nodes): ping all known addresses

isse #953

* refactor(frontend): ignore connect node messages

issue #953

* refactor(nodes): balance better when many start at once

issue #953

* refactor(frontend): do not bother so much for status

issue #953

* feature(backend): start and stop remote node via WOL

Added new field in the VM to store MAC address

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* feature(nodes): start and stop remote node

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* wip(frontend): shutdown and start nodes request

issue #953

* Feature #731 start all (#954)

* 731_start_all: feat: New tab clones created. issue #731

* 731_start_all: feat: Request to start all clones of a base implemented. issue #731

* feat: test created. issue [#731].

* test(start): check clones are started

issue #731

* wip(frontend): merged bs4.1 intro clones tab

* refactor(backend): set starting domains to help balance

Count starting domains as if already started when balancing.
This way the load is best spread among nodes.

issue #731

* wip(backend): add new valid status *starting*

issue #731

* Replaced hardcoded footer with footer variable (#962)

* feature(backend): start and stop remote node via WOL

Added new field in the VM to store MAC address

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* feature(backend): start and stop remote node via WOL

Added new field in the VM to store MAC address

issue #953

* test(backend): check mac and stop node

issue #953

* test(backend): test wol

issue #953

* feature(nodes): start and stop remote node

issue #953

* feature(nodes): start and stop remote node

issue #953

* wip(frontend): shutdown and start nodes request

issue #953

* feature(backend): check node connection

issue #953

* feature(frontend): show status of node connection

issue #953

* refactor(nodes): ping node with all IPs

* wip(backend): show error if start request failed

issue #953

* refactor(frontend): removed debug

* refactor(frontend): show error messages prettier

issue #953

* refactor(frontend): check node status after operations

issue #953

* refactor(requests): deal with stopping requests and set done

issue #953

* refactor(nodes): ping all known addresses

isse #953

* refactor(frontend): ignore connect node messages

issue #953

* refactor(nodes): balance better when many start at once

issue #953

* refactor(frontend): do not bother so much for status

issue #953

* wip(nodes): remove all remote bases

issue #953

* test(nodes): test remove remote bases
parent 719ecfbd
......@@ -1183,6 +1183,7 @@ sub _upgrade_tables {
$self->_upgrade_table('vms','security','varchar(255) default NULL');
$self->_upgrade_table('grant_types','enabled','int not null default 1');
$self->_upgrade_table('vms','mac','char(18)');
}
......@@ -1979,6 +1980,86 @@ sub process_requests {
}
$sth->finish;
$self->_timeout_requests();
}
sub _date_now($seconds = 0) {
confess "Error, can't search what changed in the future "
if $seconds > 0;
my @now = localtime(time + $seconds);
$now[4]++;
for (0 .. 4) {
$now[$_] = "0".$now[$_] if length($now[$_])<2;
}
my $time_recent = ($now[5]+=1900)."-".$now[4]."-".$now[3]
." ".$now[2].":".$now[1].":".$now[0];
return $time_recent;
}
sub _timeout_requests($self) {
my $sth = $CONNECTOR->dbh->prepare(
"SELECT id,pid, start_time, date_changed "
." FROM requests "
." WHERE ( status = 'working' or status = 'stopping' )"
." AND date_changed >= ? "
." ORDER BY date_req "
);
$sth->execute(_date_now(-30));
my @requests;
while (my ($id, $pid, $start_time) = $sth->fetchrow()) {
my $req = Ravada::Request->open($id);
my $timeout = $req->defined_arg('timeout') or next;
next if time - $start_time <= $timeout;
warn "request ".$req->pid." ".$req->command." timeout";
push @requests,($req);
}
$sth->finish;
$self->_kill_requests(@requests);
}
sub _kill_requests($self, @requests) {
for my $req (@requests) {
warn "killing request ".$req->id." ".$req->command." pid: ".$req->pid;
$req->status('stopping');
my @procs = $self->_process_sons($req->pid);
if ( @procs) {
for my $current (@procs) {
my ($pid, $cmd) = @$current;
my $signal = 15;
$signal = 9 if $cmd =~ /<defunct>$/;
warn "sending $signal to $pid $cmd";
kill($signal, $pid);
}
} else {
if ($req->pid == $$) {
warn "I'm not gonna kill myself $$";
} else {
kill(15, $req->pid);
}
$req->status('done');
}
}
}
sub _process_sons($self, $pid) {
my @process;
my $cmd = "ps -eo 'ppid pid cmd'";
open my $ps,'-|', $cmd or die "$! $cmd";
while (my $line = <$ps>) {
warn "looking for $pid in ".$line if $line =~ /$pid/;
my ($pid_son, $cmd) = $line =~ /^\s*$pid\s+(\d+)\s+(.*)/;
next if !$pid_son;
warn "$cmd\n";
push @process,[$pid_son, $cmd] if $pid_son;
}
return @process;
}
=head2 process_long_requests
......@@ -2107,8 +2188,11 @@ sub _execute {
my $sub = $self->_req_method($request->command);
confess "Unknown command ".$request->command
if !$sub;
if (!$sub) {
$request->error("Unknown command ".$request->command);
$request->status('done');
return;
}
$request->pid($$);
$request->start_time(time);
......@@ -2732,6 +2816,55 @@ sub _cmd_change_curr_memory($self, $request) {
$domain->set_memory($memory);
}
sub _cmd_shutdown_node($self, $request) {
my $id_node = $request->args('id_node');
my $node = Ravada::VM->open($id_node);
$node->shutdown();
}
sub _cmd_start_node($self, $request) {
my $id_node = $request->args('id_node');
my $node = Ravada::VM->open($id_node);
$node->start();
}
sub _cmd_connect_node($self, $request) {
my $backend = $request->defined_arg('backend');
my $hostname = $request->defined_arg('hostname');
my $id_node = $request->defined_arg('id_node');
my $node;
if ($id_node) {
$node = Ravada::VM->open($id_node);
$hostname = $node->host;
} else {
$node = Ravada::VM->open( type => $backend
, host => $hostname
, store => 0
);
}
die "I can't ping $hostname\n"
if ! $node->ping();
$request->error("Ping ok. Trying to connect to $hostname");
my ($out, $err);
eval {
($out, $err) = $node->run_command('/bin/true');
};
$err = $@ if $@ && !$err;
warn "out: $out" if $out;
if ($err) {
warn $err;
$err =~ s/(.*?) at lib.*/$1/s;
chomp $err;
$err .= "\n";
die $err if $err;
}
$node->connect() && $request->error("Connection OK");
}
sub _clean_requests($self, $command, $request=undef) {
my $query = "DELETE FROM requests "
." WHERE command=? "
......@@ -2962,6 +3095,10 @@ sub _req_method {
,remove_hardware => \&_cmd_remove_hardware
,change_max_memory => \&_cmd_change_max_memory
,change_curr_memory => \&_cmd_change_curr_memory
# Virtual Managers or Nodes
,shutdown_node => \&_cmd_shutdown_node
,start_node => \&_cmd_start_node
,connect_node => \&_cmd_connect_node
#users
,post_login => \&_cmd_post_login
......
......@@ -307,19 +307,18 @@ sub _start_preconditions{
return if $self->_search_already_started();
# if it is a clone ( it is not a base )
if ($self->id_base) {
$self->status('starting');
# $self->_set_last_vm(1)
if ( !$self->is_local && ( !$self->_vm->enabled || !$self->_vm->ping) ) {
my $vm_local = $self->_vm->new( host => 'localhost' );
$self->_set_vm($vm_local, 1);
}
$self->_balance_vm();
$self->status('starting');
$self->rsync(request => $request) if !$self->is_volatile && !$self->_vm->is_local();
} elsif (!$self->is_local) {
my $vm_local = $self->_vm->new( host => 'localhost' );
$self->_set_vm($vm_local, 1);
}
$self->status('starting');
$self->_check_free_vm_memory();
#TODO: remove them and make it more general now we have nodes
#$self->_check_cpu_usage($request);
......@@ -1485,7 +1484,18 @@ sub _post_remove_base {
my $self = shift;
$self->_remove_base_db(@_);
$self->_post_remove_base_domain();
$self->_set_base_vm_db($self->_vm->id,1);
$self->_remove_all_bases();
}
sub _remove_all_bases($self) {
my $sth = $$CONNECTOR->dbh->prepare("SELECT id_vm FROM bases_vm "
." WHERE id_domain=? AND enabled=1"
);
$sth->execute($self->id);
while ( my ($id_vm) = $sth->fetchrow ) {
$self->remove_base_vm( id_vm => $id_vm );
}
}
sub _pre_shutdown_domain {}
......@@ -1710,7 +1720,9 @@ sub _around_is_active($orig, $self) {
|| !$self->is_known
|| (defined $self->_data('id_vm') && (defined $self->_vm) && $self->_vm->id != $self->_data('id_vm'));
my $status = 'shutdown';
my $status = $self->_data('status');
$status = 'shutdown' if $status eq 'active';
$status = 'active' if $is_active;
$status = 'hibernated' if !$is_active && !$self->is_removed && $self->is_hibernated;
$self->_data(status => $status);
......
......@@ -614,14 +614,14 @@ sub start {
$self->_set_spice_ip($set_password);
$self->status('starting');
eval { $self->domain->create() };
if ( $@ && $@ !~ /already running/i ) {
my $error = $@;
if ( $error && $error !~ /already running/i ) {
if ( $self->domain->has_managed_save_image ) {
$request->status("removing saved image") if $request;
$self->domain->managed_save_remove();
$self->domain->create();
} elsif ( $request ) {
$request->error($@);
die $@ if $@;
} else {
die $error;
}
}
}
......
......@@ -845,6 +845,7 @@ sub list_requests($self, $id_domain_req=undef, $seconds=60) {
|| $command eq 'ping_backend'
|| $command eq 'cleanup'
|| $command eq 'screenshot'
|| $command eq 'connect_node'
;
next if ( $command eq 'force_shutdown'
|| $command eq 'start'
......
......@@ -171,19 +171,4 @@ sub _get_driver_sound {
}
=pod
sub get_info {
my $self = shift;
my $doc = XML::LibXML->load_xml(string => $self->_data_extra('xml'));
my $info;
$info->{max_mem} = ($doc->findnodes('/domain/memory'))[0]->textContent;
$info->{memory} = ($doc->findnodes('/domain/currentMemory'))[0]->textContent;
return $info;
}
=cut
1;
......@@ -82,9 +82,16 @@ our %VALID_ARG = (
,change_max_memory => {uid => 1, id_domain => 1, ram => 1}
,enforce_limits => { timeout => 2, _force => 2 }
,refresh_machine => { id_domain => 1 }
# Virtual Managers or Nodes
,refresh_vms => { _force => 2 }
,shutdown_node => { id_node => 1, at => 2 }
,start_node => { id_node => 1, at => 2 }
,connect_node => { backend => 2, hostname => 2, id_node =>2, timeout => 2 }
#users
,post_login => { uid => 1, locale => 2 }
);
our %CMD_SEND_MESSAGE = map { $_ => 1 }
......@@ -95,6 +102,7 @@ our %CMD_SEND_MESSAGE = map { $_ => 1 }
change_max_memory change_curr_memory
add_hardware remove_hardware set_driver
set_base_vm
shutdown_node start_node
);
our $TIMEOUT_SHUTDOWN = 120;
......@@ -109,7 +117,7 @@ our %COMMAND = (
}
,priority => {
limit => 20
,commands => ['clone','start','create_domain','open_iptables']
,commands => ['clone','start','start_clones','create_domain','open_iptables']
}
);
lock_hash %COMMAND;
......@@ -1076,6 +1084,8 @@ sub refresh_vms {
return if done_recently(undef,60,'refresh_vms') || _requested('refresh_vms');
}
$args->{timeout} = 120 if ! $args->{timeout};
my $self = {};
bless($self,$class);
......@@ -1385,22 +1395,67 @@ sub refresh_machine {
}
sub shutdown_node {
my $proto = shift;
my $class = ref($proto) || $proto;
my $args = _check_args('shutdown_node', @_ );
my $self = {};
bless($self, $class);
my $req = _new_request($self
, command => 'shutdown_node'
, args => $args
);
return $req;
sub post_login {
return _new_request_generic('post_login',@_);
}
sub _new_request_generic {
my $command = shift;
sub start_node{
my $proto = shift;
my $class = ref($proto) || $proto;
my $args = _check_args($command, @_ );
my $args = _check_args('start_node', @_ );
my $self = {};
bless($self, $class);
my $req = _new_request($self
, command => 'start_node'
, args => $args
);
return $req;
}
sub connect_node {
my $proto = shift;
my $class = ref($proto) || $proto;
my $args = _check_args('connect_node', @_ );
$args->{timeout} = 10 if !exists $args->{timeout};
return _new_request($self
, command => 'connect_node'
, args => $args
);
}
sub post_login {
return _new_request_generic('post_login',@_);
}
sub _new_request_generic {
my $command = shift;
my $proto = shift;
my $class = ref($proto) || $proto;
my $args = _check_args($command, @_ );
my $req = _new_request($self
,command => $command
,args => $args
......
......@@ -19,7 +19,7 @@ use Socket qw( inet_aton inet_ntoa );
use Moose::Role;
use Net::DNS;
use Net::Ping;
use Net::SSH2;
use Net::SSH2 qw(LIBSSH2_FLAG_SIGPIPE);
use IO::Socket;
use IO::Interface;
use Net::Domain qw(hostfqdn);
......@@ -41,6 +41,10 @@ our $MIN_MEMORY_MB = 128 * 1024;
our $SSH_TIMEOUT = 20 * 1000;
our %SSH;
our $ARP = `which arp`;
chomp $ARP;
# domain
requires 'create_domain';
requires 'search_domain';
......@@ -82,6 +86,11 @@ has 'readonly' => (
,default => 0
);
has 'store' => (
isa => 'Bool'
, is => 'rw'
, default => 1
);
############################################################
#
# Method Modifiers definition
......@@ -97,6 +106,7 @@ before 'create_volume' => \&_connect;
around 'import_domain' => \&_around_import_domain;
around 'ping' => \&_around_ping;
around 'connect' => \&_around_connect;
#############################################################
#
......@@ -159,6 +169,9 @@ sub BUILD {
my $id = delete $args->{id};
my $host = delete $args->{host};
my $name = delete $args->{name};
my $store = delete $args->{store};
$store = 1 if !defined $store;
delete $args->{readonly};
delete $args->{security};
delete $args->{public_ip};
......@@ -169,7 +182,7 @@ sub BUILD {
lock_hash(%$args);
confess "ERROR: Unknown args ".join (",", keys (%$args)) if keys %$args;
return if !$store;
if ($id) {
$self->_select_vm_db(id => $id)
} else {
......@@ -203,7 +216,6 @@ sub _open_type {
my $vm = $proto->new(%args);
eval { $vm->vm };
warn $@ if $@;
return if $@;
return $vm;
......@@ -218,7 +230,23 @@ sub _check_readonly {
sub _connect {
my $self = shift;
$self->connect();
my $result = $self->connect();
if ($result) {
$self->is_active(1);
} else {
$self->is_active(0);
}
return $result;
}
sub _around_connect($orig, $self) {
my $result = $self->$orig();
if ($result) {
$self->is_active(1);
} else {
$self->is_active(0);
}
return $result;
}
sub _pre_create_domain {
......@@ -583,7 +611,7 @@ sub id {
}
sub _data($self, $field, $value=undef) {
if (defined $value) {
if (defined $value && $self->store ) {
$self->{_data}->{$field} = $value;
my $sth = $$CONNECTOR->dbh->prepare(
"UPDATE vms set $field=?"
......@@ -597,6 +625,8 @@ sub _data($self, $field, $value=undef) {
# _init_connector();
return $self->{_data}->{$field} if exists $self->{_data}->{$field};
return if !$self->store();
$self->{_data} = $self->_select_vm_db( name => $self->name);
confess "No DB info for VM ".$self->name if !$self->{_data};
......@@ -619,6 +649,7 @@ sub _do_select_vm_db {
}
}
confess Dumper(\%args) if !keys %args;
my $sth = $$CONNECTOR->dbh->prepare(
"SELECT * FROM vms WHERE ".join(" AND ",map { "$_=?" } sort keys %args )
);
......@@ -642,6 +673,8 @@ sub _select_vm_db {
sub _insert_vm_db {
my $self = shift;
return if !$self->store();
my $sth = $$CONNECTOR->dbh->prepare(
"INSERT INTO vms (name, vm_type, hostname, public_ip)"
." VALUES(?, ?, ?, ?)"
......@@ -682,7 +715,7 @@ sub default_storage_pool_name {
$sth->execute($value,$id);
$self->{_data}->{default_storage} = $value;
}
$self->_select_vm_db();
$self->_select_vm_db() if $self->store();
return $self->_data('default_storage');
}
......@@ -835,24 +868,32 @@ Returns if the virtual manager connection is available
sub ping($self, $option=undef) {
confess "ERROR: option unknown" if defined $option && $option ne 'debug';
return 1 if $self->is_local();
my $debug = 0;
$debug = 1 if defined $option && $option eq 'debug';
return 1 if $self->is_local();
return $self->_do_ping($self->host, $debug);
}
sub _do_ping($self, $host, $debug=0) {
my $p = Net::Ping->new('tcp',2);
my $ping_ok;
eval { $ping_ok = $p->ping($self->host) };
warn $@ if $@;
eval { $ping_ok = $p->ping($host) };
confess $@ if $@;
warn "$@ pinging host $host" if $@;
$self->_store_mac_address() if $ping_ok && $self;
return 1 if $ping_ok;
$p->close();
return if $>; # icmp ping requires root privilege
warn "trying icmp" if $debug;
$p= Net::Ping->new('icmp',2);
eval { $ping_ok = $p->ping($self->host) };
eval { $ping_ok = $p->ping($host) };
warn $@ if $@;
$self->_store_mac_address() if $ping_ok && $self;
return 1 if $ping_ok;
return 0;
......@@ -976,6 +1017,21 @@ sub run_command($self, @command) {
return ($out, $err);
}
sub run_command_nowait($self, @command) {
return $self->_run_command_local(@command) if $self->is_local();
my $chan = $self->_ssh_channel() or die "ERROR: No SSH channel to host ".$self->host;
my $command = join(" ",@command);
$chan->exec($command);# or $self->{_ssh}->die_with_error;
$chan->send_eof();
return;
}
sub _run_command_local($self, @command) {
my ( $in, $out, $err);
my ($exec) = $command[0];
......@@ -1073,9 +1129,15 @@ sub iptables_list($self) {
return $ret;
}
sub _random_list(@list) {
return @list if rand(5) < 2;
return reverse @list if rand(5) < 2;
return (sort { $a cmp $b } @list);
}
sub balance_vm($self, $base=undef) {
my %vm_list;
for my $vm ($self->list_nodes) {
for my $vm (_random_list($self->list_nodes)) {
next if !$vm->enabled();
next if !$vm->is_active();
......@@ -1090,7 +1152,6 @@ sub balance_vm($self, $base=undef) {
last if $key =~ /^[01]+\./; # don't look for other nodes when this one is empty !
}
my @sorted_vm = map { $vm_list{$_} } sort keys %vm_list;
for my $vm (@sorted_vm) {
next if $base && !$base->base_in_vm($vm->id);
return $vm;
......@@ -1126,6 +1187,58 @@ sub shutdown_domains($self) {
$sth->finish;
}
sub _store_mac_address($self, $force=0 ) {
return if !$force && $self->_data('mac');
die "Error: I can't find arp" if !$ARP;
my %done;
for my $ip ($self->host,$self->ip, $self->public_ip) {
next if !$ip || $done{$ip}++;
CORE::open (my $arp,'-|',"$ARP -n "