Unverified Commit 237d8130 authored by Francesc Guasch's avatar Francesc Guasch Committed by GitHub
Browse files

789 torture (#849)

* test(backend): torture test backend

- create domains and clones from debian preseed
- start and stop many instances
- prepare bases

issue #789

* wip(test): fixed typo

issue #789

* fix(backend): allow clone even if many requests pending

fixes issue #790

* test(backend): add acpi to start domains quickly

issue #789

* test(backend): create clones from the clones

issue #789

* test(requests): many clone requests to same base

issue #790

* fix(requests): allow many clones to same base

issue #790

* test(users): create admin user if it doesn't exist

issue #789

* test(backend): check real memory for void domains

issue #789

* test(backend): test void domains too

issue #789

* wip(test): stress test clones

issue #789

* wip(test): let install debian in the background

test void in the meanwhile

issue #789

* test(backend): run test if env set

Either TEST_STRESS , TEST_STRESS_KVM or TEST_STRESS_Void
should be set to run the tests

issue #789

* test(backend): create random requests, restart and hibernate

issue #789

* test(backend): allow run all Void tests from non root

issue #789

* test(backend): partition a real debian machine

* test(backend): test random requests

issue #789

* test(requests): autostart_domain right call for easier tests

issue #789

* test(backend): added verbose mode

issue #789

* test(backend): lock void domains on update

issue #789

* test(iptables): clean jump
issue #789

* fix(backend): connect to KVM before listing volumes

issue #789

* test(backend): store id_vm on create domain

issue #789

* test(backend): test random requests

issue #789

* fix(frontend): list trivial recent process if failed

those process results were hidden

issue #789

* fix(frontend): properly show error message

issue #789

* test(backend): test more requests

issue #789

* test(backend): ignore error base removed

issue #789

* refactor(backend): clean at arg from request

* refactor(backend): network arg not used

issue #789

* fix(backend): do not create iptables jump on shutdown

Sometimes an extra RAVADA chain was added to iptables

issue #789

* test: accept fail because host too loaded

* refactor(requests): autostart domain fixed duplicate

issue #789

* test(requests): test all requests one by one

and test with each argument removed too

issue #789

* fix(iptables): add jump only when creating chain

issue #789

* test(shutdown): fix shutdown arguments

issue #789

* test(requests): check requests on each vm

issue #789

* test(backend): check memory added to mock machines

issue #789

* test(backend): return default info if no memory defined

* refactor(iptables): retry if can't acquire iptables cmd

issue #789

* test: create a mock admin user and remove it

issue #789

* test: allow error when no screen at copy screen

* fix(iptables): try better to aquire iptables exe

Under heavy load it may be busy

issue #789

* test(backend): make VM random in tests

issue #789

* refactor(backend): always process ping backend

issue #789

* test: don't use machines that will be removed

issue #789

* test: no need to specify vm

* test: stress with not so many clones at first

* wip(test): next if request removed

* refactor(requests): store creation date

issue #789

* wip(test): remove done requests

issue #789

* wip(test): stress test

* refactor(backend): check correct owner

* fixed error in merge

* test(backend): do not test downloads in stress

* wip(test stress): cope with normal requests problems

* wip(test stress): cope with normal req error

issue #789

* refactor(backend): check user passed exists

issue #789

* refactor(backend): store vm id

issue #789

* test(stress): use only test domains and clones

issue #789

* test(backend): check start and create at once

issue #789

* wip(backend): load db changes before using storage fields

issue #841

* check iptables jumps count

issue #789

* doc: both flavours of autostart domain

* test: clean user at the end

issue #789

* refactor(backend): make sure all requests args are used

issue #789
parent 6d389b89
......@@ -84,6 +84,7 @@ GetOptions ( help => \$help
,all => \$ALL
,list => \$LIST
,debug => \$DEBUG
,verbose => \$VERBOSE
,'no-fork'=> \$NOFORK
,'start=s' => \$START_DOMAIN
,'config=s'=> \$FILE_CONFIG
......@@ -130,6 +131,7 @@ my %CONFIG;
%CONFIG = ( config => $FILE_CONFIG ) if $FILE_CONFIG;
$Ravada::DEBUG=1 if $DEBUG;
$Ravada::VERBOSE=1 if $VERBOSE;
$Ravada::CAN_FORK=0 if $NOFORK;
###################################################################
......
......@@ -67,6 +67,7 @@ $FILE_CONFIG = undef if ! -e $FILE_CONFIG;
our $CONNECTOR;
our $CONFIG = {};
our $DEBUG;
our $VERBOSE;
our $CAN_FORK = 1;
our $CAN_LXC = 0;
......@@ -1298,18 +1299,15 @@ sub create_domain {
my %args = @_;
my $request = $args{request};
%args = %{$request->args} if $request;
my $start = $args{start};
my $vm_name = $args{vm};
my $id_base = $args{id_base};
my $request = $args{request};
my $id_owner = $args{id_owner};
my $vm;
if ($request) {
%args = %{$request->args};
$vm_name = $request->defined_arg('vm') if $request->defined_arg('vm');
$id_base = $request->defined_arg('id_base') if $request->defined_arg('id_base');
}
if ($vm_name) {
$vm = $self->search_vm($vm_name);
confess "ERROR: vm $vm_name not found" if !$vm;
......@@ -1329,6 +1327,7 @@ sub create_domain {
$request->status("creating") if $request;
my $domain;
delete $args{'at'};
eval { $domain = $vm->create_domain(%args)};
my $error = $@;
......@@ -1734,9 +1733,12 @@ sub process_requests {
next if $@ && $@ =~ /I can't find id/;
die $@ if $@;
if ( ($long_commands &&
if (
$req->command ne 'ping_backend'
&&( ($long_commands &&
(!$short_commands && !$LONG_COMMAND{$req->command}))
||(!$long_commands && $LONG_COMMAND{$req->command})
||(!$long_commands && $LONG_COMMAND{$req->command})
)
) {
warn "[$debug_type,$long_commands,$short_commands] $$ skipping request "
.$req->command if $DEBUG;
......@@ -1761,11 +1763,11 @@ sub process_requests {
$req->status("done");
}
}
next if !$DEBUG && !$debug;
next if !$DEBUG && !$debug && !$VERBOSE;
sleep 1;
warn "req ".$req->id." , command: ".$req->command." , status: ".$req->status()
." , error: '".($req->error or 'NONE')."'\n" if $DEBUG;
." , error: '".($req->error or 'NONE')."'\n" if $DEBUG || $VERBOSE;
sleep 1 if $DEBUG;
}
$sth->finish;
......@@ -2180,9 +2182,11 @@ sub _cmd_clone($self, $request) {
push @args, ( memory => $request->args('memory'))
if $request->defined_arg('memory');
my $user = Ravada::Auth::SQL->search_by_id($request->args('uid'))
or die "Error: User missing, id: ".$request->args('uid');
push @args,(user => $user);
$domain->clone(
name => $request->args('name')
,user => Ravada::Auth::SQL->search_by_id($request->args('uid'))
,@args
);
......
......@@ -1430,6 +1430,7 @@ sub _post_shutdown {
$self->needs_restart(0) if $self->is_known()
&& $self->needs_restart()
&& !$self->is_active;
_test_iptables_jump();
}
sub _around_is_active($orig, $self) {
......@@ -1500,6 +1501,8 @@ sub add_volume_swap {
sub _remove_iptables {
my $self = shift;
return if $>;
my %args = @_;
my $user = delete $args{user};
......@@ -1509,7 +1512,7 @@ sub _remove_iptables {
confess "ERROR: Unknown args ".Dumper(\%args) if keys %args;
my $ipt_obj = _obj_iptables();
my $ipt_obj = _obj_iptables(0);
my $sth = $$CONNECTOR->dbh->prepare(
"UPDATE iptables SET time_deleted=?"
......@@ -1527,6 +1530,21 @@ sub _remove_iptables {
}
}
sub _test_iptables_jump {
my @cmd = ('iptables','-L','INPUT');
my ($in, $out, $err);
run3(\@cmd, \$in, \$out, \$err);
my $count = 0;
for my $line ( split /\n/,$out ) {
$count++ if $line =~ /^RAVADA /;
}
return if !$count || $count == 1;
warn "Expecting 0 or 1 RAVADA iptables jump, got: " .($count or 0);
}
sub _remove_temporary_machine {
my $self = shift;
......@@ -1628,7 +1646,8 @@ sub _add_iptable {
,$local_ip, 'filter', $IPTABLES_CHAIN, 'ACCEPT',
,{'protocol' => 'tcp', 's_port' => 0, 'd_port' => $local_port});
($rv, $out_ar, $errs_ar) = $ipt_obj->append_ip_rule(@iptables_arg);
($rv, $out_ar, $errs_ar) = $ipt_obj->append_ip_rule(@iptables_arg)
if !$>;
$self->_log_iptable(iptables => \@iptables_arg, @_);
......@@ -1637,7 +1656,8 @@ sub _add_iptable {
,$local_ip, 'filter', $IPTABLES_CHAIN, 'DROP',
,{'protocol' => 'tcp', 's_port' => 0, 'd_port' => $local_port});
($rv, $out_ar, $errs_ar) = $ipt_obj->append_ip_rule(@iptables_arg);
($rv, $out_ar, $errs_ar) = $ipt_obj->append_ip_rule(@iptables_arg)
if !$>;
$self->_log_iptable(iptables => \@iptables_arg, %args);
......@@ -1691,7 +1711,7 @@ sub open_iptables {
$self->_add_iptable(%args);
}
sub _obj_iptables {
sub _obj_iptables($create_chain=1) {
my %opts = (
'use_ipv6' => 0, # can set to 1 to force ip6tables usage
......@@ -1710,9 +1730,20 @@ sub _obj_iptables {
### iptables commands (default is 0).
);
my $ipt_obj = IPTables::ChainMgr->new(%opts)
or die "[*] Could not acquire IPTables::ChainMgr object";
my $ipt_obj;
my $error;
for ( 1 .. 10 ) {
eval {
$ipt_obj = IPTables::ChainMgr->new(%opts)
or warn "[*] Could not acquire IPTables::ChainMgr object";
};
$error = $@;
last if !$error;
sleep 1;
}
confess $error if $error;
return $ipt_obj if !$create_chain;
my $rv = 0;
my $out_ar = [];
my $errs_ar = [];
......@@ -1721,15 +1752,28 @@ sub _obj_iptables {
($rv, $out_ar, $errs_ar) = $ipt_obj->chain_exists('filter', $IPTABLES_CHAIN);
if (!$rv) {
$ipt_obj->create_chain('filter', $IPTABLES_CHAIN);
}
($rv, $out_ar, $errs_ar) = $ipt_obj->add_jump_rule('filter','INPUT', 1, $IPTABLES_CHAIN);
warn join("\n", @$out_ar) if $out_ar->[0] && $out_ar->[0] !~ /already exists/;
}
_add_jump($ipt_obj);
# set the policy on the FORWARD table to DROP
# $ipt_obj->set_chain_policy('filter', 'FORWARD', 'DROP');
return $ipt_obj;
}
sub _add_jump($ipt_obj) {
my $out = `iptables -L INPUT -n`;
my $count = 0;
for my $line ( split /\n/,$out ) {
next if $line !~ /^[A-Z]+ /;
$count++;
return if $line =~ /^RAVADA/;
}
my ($rv, $out_ar, $errs_ar)
= $ipt_obj->add_jump_rule('filter','INPUT', 1, $IPTABLES_CHAIN);
warn join("\n", @$out_ar) if $out_ar->[0] && $out_ar->[0] !~ /already exists/;
}
sub _log_iptable {
my $self = shift;
if (scalar(@_) %2 ) {
......
......@@ -5,6 +5,7 @@ use strict;
use Carp qw(cluck croak);
use Data::Dumper;
use Fcntl qw(:flock SEEK_END);
use File::Copy;
use Hash::Util qw(lock_keys);
use IPC::Run3 qw(run3);
......@@ -114,12 +115,40 @@ sub _store {
$data->{$var} = $value;
open my $lock,">>","$disk.lock" or die "I can't open lock: $disk.log: $!";
_lock($lock);
eval { DumpFile($disk, $data) };
_unlock($lock);
chomp $@;
confess $@ if $@;
}
sub _lock {
my ($fh) = @_;
flock($fh, LOCK_EX) or die "Cannot lock - $!\n";
}
sub _unlock {
my ($fh) = @_;
flock($fh, LOCK_UN) or die "Cannot unlock - $!\n";
}
sub _value{
my $self = shift;
my ($var) = @_;
my ($disk) = $self->_config_file();
my $data = {} ;
$data = LoadFile($disk) if -e $disk;
return $data->{$var};
}
sub shutdown {
my $self = shift;
$self->_store(is_active => 0);
......@@ -232,7 +261,11 @@ sub add_volume {
$args{target} = _new_target($data);
$data->{device}->{$args{name}} = \%args;
my $disk = $self->_config_file;
open my $lock,">>","$disk.lock" or die "I can't open lock: $disk.log: $!";
_lock($lock);
eval { DumpFile($self->_config_file, $data) };
_unlock($lock);
chomp $@;
die "readonly=".$self->readonly." ".$@ if $@;
......@@ -345,9 +378,9 @@ sub can_screenshot { return $CONVERT; }
sub get_info {
my $self = shift;
my $info = $self->_value('info');
$self->_set_default_info()
if !$info->{memory};
$info = $self->_value('info');
if (!$info->{memory}) {
$info = $self->_set_default_info();
}
lock_keys(%$info);
return $info;
}
......@@ -366,7 +399,7 @@ sub _set_default_info {
for my $name ( sort keys %controllers) {
$self->set_controller($name,2);
}
return $info;
}
sub set_max_memory {
......@@ -459,6 +492,7 @@ sub clean_swap_volumes {
}
}
sub can_hibernate { return 1};
sub hybernate {
my $self = shift;
$self->_store(is_active => 0);
......
......@@ -10,6 +10,7 @@ Ravada::Front - Web Frontend library for Ravada
=cut
use Carp qw(carp);
use DateTime;
use Hash::Util qw(lock_hash);
use JSON::XS;
use Moose;
......@@ -50,6 +51,7 @@ our $DIR_SCREENSHOTS = "/var/www/img/screenshots";
our %VM;
our $PID_FILE_BACKEND = '/var/run/rvd_back.pid';
our $LOCAL_TZ = DateTime::TimeZone->new(name => 'local');
###########################################################################
#
# method modifiers
......@@ -743,15 +745,28 @@ sub list_requests($self, $id_domain_req=undef, $seconds=60) {
, $error, $id_domain, $domain, $date));
while ( $sth->fetch) {
next if $command eq 'force_shutdown'
my $epoch_date_changed;
if ($date_changed) {
my ($y,$m,$d,$hh,$mm,$ss) = $date_changed =~ /(\d{4})-(\d\d)-(\d\d) (\d+):(\d+):(\d+)/;
if ($y) {
$epoch_date_changed = DateTime->new(year => $y, month => $m, day => $d
,hour => $hh, minute => $mm, second => $ss
,time_zone => $LOCAL_TZ
)->epoch;
}
}
next if ( $command eq 'force_shutdown'
|| $command eq 'start'
|| $command eq 'shutdown'
|| $command eq 'screenshot'
|| $command eq 'hibernate'
|| $command eq 'ping_backend'
|| $command eq 'ping_backend')
|| $command eq 'enforce_limits'
|| $command eq 'refresh_vms'
|| $command eq 'refresh_storage';
|| $command eq 'refresh_storage'
&& time - $epoch_date_changed > 5
&& $status eq 'done'
&& !$error;
next if $id_domain_req && defined $id_domain && $id_domain != $id_domain_req;
my $args;
$args = decode_json($j_args) if $j_args;
......
......@@ -45,7 +45,7 @@ our %VALID_ARG = (
,id_template => 2
,memory => 2
,disk => 2
,network => 2
# ,network => 2
,remote_ip => 2
,start => 2
}
......@@ -58,13 +58,13 @@ our %VALID_ARG = (
,shutdown_domain => { name => 2, id_domain => 2, uid => 1, timeout => 2, at => 2 }
,force_shutdown_domain => { id_domain => 1, uid => 1, at => 2 }
,screenshot_domain => { id_domain => 1, filename => 2 }
,autostart_domain => { id_domain => 1 , uid => 1, value => 2 }
,domain_autostart => { id_domain => 1 , uid => 1, value => 2 }
,copy_screenshot => { id_domain => 1, filename => 2 }
,start_domain => {%$args_manage, remote_ip => 1, name => 2, id_domain => 2 }
,rename_domain => { uid => 1, name => 1, id_domain => 1}
,set_driver => {uid => 1, id_domain => 1, id_option => 1}
,hybernate=> {uid => 1, id_domain => 1}
,download => {uid => 2, id_iso => 1, id_vm => 2, delay => 2, verbose => 2}
,download => {uid => 2, id_iso => 1, id_vm => 2, verbose => 2}
,refresh_storage => { id_vm => 2 }
,clone => { uid => 1, id_domain => 1, name => 1, memory => 2 }
,change_owner => {uid => 1, id_domain => 1}
......@@ -77,7 +77,7 @@ our %VALID_ARG = (
our %CMD_SEND_MESSAGE = map { $_ => 1 }
qw( create start shutdown prepare_base remove remove_base rename_domain screenshot download
autostart_domain hibernate hybernate
domain_autostart hibernate hybernate
change_owner
change_max_memory change_curr_memory
add_hardware remove_hardware set_driver
......@@ -507,6 +507,11 @@ sub _new_request {
$self->{id} = $self->_last_insert_id();
$sth = $$CONNECTOR->dbh->prepare(
"UPDATE requests set date_req=date_changed"
." WHERE id=?");
$sth->execute($self->{id});
return $self->open($self->{id});
}
......@@ -1094,7 +1099,8 @@ sub domain_autostart {
my $proto = shift;
my $class = ref($proto) || $proto;
my $args = _check_args('autostart_domain', @_ );
my $args = _check_args('domain_autostart', @_ );
$args->{value} = 1 if !exists $args->{value};
my $self = {};
bless($self, $class);
......@@ -1105,6 +1111,16 @@ sub domain_autostart {
);
}
=head2 autostart_domain
Deprecated for domain_autostart
=cut
sub autostart_domain {
return domain_autostart(@_);
}
=head2 refresh_vms
Refreshes cached information of the VMs.
......
......@@ -138,7 +138,7 @@ sub _around_create_domain {
my %args = @_;
my $id_owner = delete $args{id_owner} or confess "ERROR: Missing id_owner";
my $owner = Ravada::Auth::SQL->search_by_id($id_owner);
my $owner = Ravada::Auth::SQL->search_by_id($id_owner) or confess "Unknown user id: $id_owner";
my $base;
my $id_base = delete $args{id_base};
......@@ -480,6 +480,7 @@ sub default_storage_pool_name {
$sth->execute($value,$id);
$self->{_data}->{default_storage} = $value;
}
$self->_select_vm_db();
return $self->_data('default_storage');
}
......@@ -505,6 +506,7 @@ sub base_storage_pool {
$sth->execute($value,$id);
$self->{_data}->{base_storage} = $value;
}
$self->_select_vm_db();
return $self->_data('base_storage');
}
......@@ -530,6 +532,7 @@ sub clone_storage_pool {
$sth->execute($value,$id);
$self->{_data}->{clone_storage} = $value;
}
$self->_select_vm_db();
return $self->_data('clone_storage');
}
......
......@@ -250,6 +250,7 @@ sub search_volume_re($self,$pattern,$refresh=0) {
confess "'$pattern' doesn't look like a regexp to me ".ref($pattern)
if !ref($pattern) || ref($pattern) ne 'Regexp';
$self->_connect();
$self->_refresh_storage_pools() if $refresh;
my @volume;
......@@ -660,7 +661,9 @@ sub _domain_create_from_iso {
my ($domain, $spice_password)
= $self->_domain_create_common($xml,%args);
$domain->_insert_db(name=> $args{name}, id_owner => $args{id_owner});
$domain->_insert_db(name=> $args{name}, id_owner => $args{id_owner}
, id_vm => $self->id
);
$domain->_set_spice_password($spice_password)
if $spice_password;
......@@ -849,7 +852,9 @@ sub _domain_create_from_base {
my ($domain, $spice_password)
= $self->_domain_create_common($xml,%args, is_volatile => $base->volatile_clones);
$domain->_insert_db(name=> $args{name}, id_base => $base->id, id_owner => $args{id_owner});
$domain->_insert_db(name=> $args{name}, id_base => $base->id, id_owner => $args{id_owner}
, id_vm => $self->id
);
$domain->_set_spice_password($spice_password);
$domain->xml_description();
return $domain;
......@@ -1134,7 +1139,7 @@ sub _cache_filename($url) {
$file =~ s/__+/_/g;
my ($user) = getpwuid($>);
my $dir = "/var/tmp/ravada_cache/$user";
my $dir = "/var/tmp/$user/ravada_cache/";
make_path($dir) if ! -e $dir;
return "$dir/$file";
}
......
......@@ -180,7 +180,21 @@ sub ping { return 1 }
sub is_active { return 1 }
sub free_memory { return 1000 * 1000}
sub free_memory {
my $self = shift;
open my $mem,'<',"/proc/meminfo" or die "$! /proc/meminfo";
my $memory = <$mem>;
close $mem;
chomp $memory;
$memory =~ s/.*?(\d+).*/$1/;
for my $domain ( $self->list_domains(active => 1) ) {
next if !$domain->is_active;
$memory -= $domain->get_info->{memory};
}
return $memory;
}
#########################################################################3
1;
......@@ -328,13 +328,13 @@ for my $vm_name ( qw(Void KVM)) {
SKIP: {
my $msg = "SKIPPED: No $vm_name found";
if ($vm && $vm_name =~ /kvm/i && $>) {
$msg = "SKIPPED: Test must run as root";
$msg = "SKIPPED $vm_name: Test must run as root";
$vm = undef;
}
diag($msg) if !$vm;
skip($msg,10) if !$vm;
diag("Testing requests with ".(ref $vm or '<UNDEF>'));
diag("Testing $vm_name requests with ".(ref $vm or '<UNDEF>'));
test_requests_by_domain($vm_name);
my $domain_iso0 = test_req_create_domain_iso($vm_name);
......
......@@ -226,7 +226,7 @@ for my $vm_name (qw(KVM Void)) {
SKIP: {
my $msg = "SKIPPED: Virtual manager $vm_name not found";
if ($vmm && $>) {
if ($vmm && $vm_name eq 'KVM' && $>) {
$msg = "SKIPPED: Test must run as root";
$vmm = undef;
}
......
......@@ -12,8 +12,6 @@ use lib 't/lib';
use Test::Ravada;
use Sys::Statistics::Linux;
my $BACKEND = 'KVM';
use_ok('Ravada');
my $test = Test::SQL::Data->new( config => 't/etc/sql.conf');
......@@ -78,13 +76,18 @@ my $vm;
remove_old_domains();
remove_old_disks();
for my $vm_name (vm_names()) {
SKIP: {
my $msg = "SKIPPED test: No KVM backend found";
my $vm = $RVD_BACK->search_vm('KVM');
my $msg = "SKIPPED test: No $vm_name backend found";
my $vm = $RVD_BACK->search_vm($vm_name);
if ($vm_name eq 'KVM' && $>) {
$msg = "SKIPPED test: $vm_name must be run from root";
$vm = undef;
}
diag($msg) if !$vm;
skip $msg,10 if !$vm;
use_ok("Ravada::Domain::$BACKEND");
use_ok("Ravada::Domain::$vm_name");
my $freemem = _check_free_memory();
my $n_domains = int($freemem)+2;
......@@ -114,6 +117,7 @@ SKIP: {
$_->remove($USER);
}
};
}
remove_old_domains();
remove_old_disks();
......
This diff is collapsed.
......@@ -139,7 +139,7 @@ for my $vm_name (reverse sort @VMS) {
SKIP: {
my $msg = "SKIPPED test: No $vm_name VM found ";
if ($vm && $>) {
if ($vm && $vm_name eq 'KVM' && $>) {
$msg = "SKIPPED: Test must run as root";
$vm = undef;
}
......
......@@ -115,9 +115,11 @@ sub base_pool_name {
}
sub new_domain_name {
my $post = (shift or '');