Merge pull request #178 from UPC/156_md5

156 md5

etc/xml/zesty-amd64.xml

+<domain type='kvm'>
+  <name>yakkety64</name>
+  <uuid>1805fb4f-ca45-a946-efbf-94124e760484</uuid>
+  <memory unit='KiB'>1048576</memory>
+  <currentMemory unit='KiB'>1048576</currentMemory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='x86_64' machine='pc-i440fx-xenial'>hvm</type>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>restart</on_crash>
+  <devices>
+    <emulator>/usr/bin/kvm-spice</emulator>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='raw'/>
+      <source file='/var/lib/libvirt/images/yakkety64.img'/>
+      <target dev='vda' bus='virtio'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
+      <boot order='1'/>
+    </disk>
+    <disk type='file' device='cdrom'>
+      <driver name='qemu' type='raw'/>
+      <source file='/var/lib/libvirt/images/ubuntu-16.10-desktop-amd64.iso'/>
+      <target dev='hdc' bus='ide'/>
+      <readonly/>
+      <address type='drive' controller='0' bus='1' target='0' unit='0'/>
+      <boot order='2'/>
+    </disk>
+    <controller type='usb' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'/>
+    <controller type='ide' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
+    </controller>
+    <interface type='network'>
+      <mac address='52:54:00:a7:49:36'/>
+      <source network='default'/>
+      <model type='virtio'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+    <serial type='pty'>
+      <target port='0'/>
+    </serial>
+    <console type='pty'>
+      <target type='serial' port='0'/>
+    </console>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <graphics type='spice' autoport='yes' listen='10.1.36.68'>
+      <listen type='address' address='10.1.36.68'/>
+	  <image compression='auto_glz'/>
+      <jpeg compression='auto'/>
+      <zlib compression='auto'/>
+      <playback compression='on'/>
+      <streaming mode='filter'/>
+    </graphics>
+    <sound model='ich6'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+    </sound>
+    <video>
+      <model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </video>
+    <redirdev bus='usb' type='spicevmc'>
+    </redirdev>      
+    <memballoon model='virtio'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
+    </memballoon>
+  </devices>
+</domain>

etc/xml/zesty-volume.xml

+<volume type='file'>
+  <name>__name__.img</name>
+  <key>/var/lib/libvirt/images/__name__.img</key>
+  <source>
+  </source>
+  <capacity unit='bytes'>9442450944</capacity>
+  <allocation unit='bytes'>5395775488</allocation>
+  <target>
+    <path>/var/lib/libvirt/images/__name__.img</path>
+    <format type='qcow2'/>
+    <permissions>
+      <mode>0600</mode>
+      <owner>0</owner>
+      <group>0</group>
+    </permissions>
+    <timestamps>
+      <atime>1461924534.956408726</atime>
+      <mtime>1461672183.623804946</mtime>
+      <ctime>1461672193.267892180</ctime>
+    </timestamps>
+  </target>
+</volume>

lib/Ravada.pm

@@ -3,7 +3,7 @@ package Ravada;
 use warnings;
 use strict;
 
-our $VERSION = '0.2.2';
+our $VERSION = '0.2.3';
 
 use Carp qw(carp croak);
 use Data::Dumper;
@@ -164,6 +164,8 @@ sub _upgrade_tables {
     $self->_upgrade_table('file_base_images','target','varchar(64) DEFAULT NULL');
     $self->_upgrade_table('vms','vm_type',"char(20) NOT NULL DEFAULT 'KVM'");
     $self->_upgrade_table('requests','at_time','int(11) DEFAULT NULL');
+    $self->_upgrade_table('iso_images','md5_url','varchar(255)');
+    $self->_upgrade_table('iso_images','file_re','char(64)');
 }
 
 

lib/Ravada/VM/KVM.pm

@@ -766,9 +766,59 @@ sub _search_iso {
     $sth->execute($id_iso);
     my $row = $sth->fetchrow_hashref;
     die "Missing iso_image id=$id_iso" if !keys %$row;
+
+    $self->_fetch_filename($row)    if $row->{file_re};
+    $self->_fetch_md5($row)         if !$row->{md5} && $row->{md5_url};
+
     return $row;
 }
 
+sub _fetch_filename {
+    my $self = shift;
+    my $row = shift;
+    my $ua = new LWP::UserAgent;
+    my $req = HTTP::Request->new( GET => $row->{url});
+    my $res = $ua->request($req);
+
+    confess "No file_re" if !$row->{file_re};
+
+    die $res->status_line if !$res->is_success;
+    my $file;
+    my $lines = '';
+    for my $line (split/\n/,$res->content) {
+        next if $line !~ /iso"/;
+        $lines .= "$line\n";
+        my ($found) = $line =~ qr/"($row->{file_re})"/;
+        next if !$found;
+        $file=$found if $found;
+    }
+    die "No ".qr($row->{file_re})." found on $row->{url}"   if !$file;
+    $row->{url} .= $file;
+}
+
+sub _fetch_md5 {
+    my $self = shift;
+    my $row = shift;
+
+    my $ua = new LWP::UserAgent;
+    my $req = HTTP::Request->new( GET => $row->{md5_url});
+    my $res = $ua->request($req);
+
+    die $res->error_line if !$res->is_success;
+
+    my ($file) = $row->{url} =~ m{.*/(.*)};
+    confess "No file for $row->{url}"   if !$file;
+
+    for my $line (split/\n/,$res->content) {
+        my ($md5) = $line =~ m{^\s*(.*?)\s+.*?$file};
+        next if !$md5;
+        $row->{md5} = $md5;
+        return;
+    }
+
+    die "No MD5 for $file in $row->{md5_url}\n".$res->content;
+}
+
 ###################################################################################
 #
 # XML methods

sql/data/insert_iso_images.sql

@@ -10,62 +10,78 @@ VALUES('Debian Jessie 32 bits netinst'
    );
 
 INSERT INTO iso_images
-(name,description,arch,xml,xml_volume,url,md5)
+(name,description,arch,xml,xml_volume,url,file_re,md5_url)
 VALUES('Ubuntu Trusty 32 bits','Ubuntu 14.04 LTS Trusty 32 bits'
     ,'i386'
     ,'trusty-i386.xml'
     ,'trusty-volume.xml'
-    ,'http://releases.ubuntu.com/16.04/ubuntu-16.04-desktop-i386.iso'
-	,'610c4a399df39a78866f9236b8c658da'
+    ,'http://releases.ubuntu.com/14.04/'
+    ,'ubuntu-14.04.*-desktop-i386.iso'
+    ,'http://releases.ubuntu.com/14.04/MD5SUMS'
 	);
 
 INSERT INTO iso_images
-(name,description,arch,xml,xml_volume,url,md5)
+(name,description,arch,xml,xml_volume,url, file_re, md5_url)
 VALUES('Ubuntu Trusty 64 bits','Ubuntu 14.04.1 LTS Trusty 64 bits'
     ,'amd64'
     ,'trusty-amd64.xml'
     ,'trusty-amd64-volume.xml'
-    ,'http://releases.ubuntu.com/16.04/ubuntu-16.04.1-desktop-amd64.iso'
-    ,'17643c29e3c4609818f26becf76d29a3'
+    ,'http://releases.ubuntu.com/16.04/'
+    ,'ubuntu-16.04.*-desktop-amd64.iso'
+    ,'http://releases.ubuntu.com/16.04/MD5SUMS'
 	);
 
 INSERT INTO iso_images
-(name,description,arch,xml,xml_volume,url,md5)
+(name,description,arch,xml,xml_volume,url, file_re, md5_url)
 VALUES('Ubuntu Xenial Xerus 32 bits','Ubuntu 16.04 LTS Xenial Xerus 32 bits'
     ,'i386'
     ,'xenial-i386.xml'
     ,'xenial-volume.xml'
-    ,'http://releases.ubuntu.com/16.04/ubuntu-16.04-desktop-i386.iso'
-    ,'610c4a399df39a78866f9236b8c658da'
+    ,'http://releases.ubuntu.com/16.04/'
+    ,'ubuntu-16.04.*desktop-i386.iso'
+    ,'http://releases.ubuntu.com/16.04/MD5SUMS'
     );
 
 INSERT INTO iso_images
-(name,description,arch,xml,xml_volume,url,md5)
+(name,description,arch,xml,xml_volume,url, file_re, md5_url)
 VALUES('Ubuntu Xenial Xerus 64 bits','Ubuntu 16.04 LTS Xenial Xerus 64 bits'
     ,'amd64'
     ,'xenial64-amd64.xml'
     ,'xenial64-volume.xml'
-    ,'http://releases.ubuntu.com/16.04/ubuntu-16.04.2-desktop-amd64.iso'
-    ,'1400884cec8e40a1a876b2678f81494b'
+    ,'http://releases.ubuntu.com/16.04/'
+    ,'ubuntu-16.04.*-desktop-amd64.iso'
+    ,'http://releases.ubuntu.com/16.04/MD5SUMS'
     );
 
 INSERT INTO iso_images
-(name,description,arch,xml,xml_volume,url,md5)
+(name,description,arch,xml,xml_volume,url, file_re, md5_url)
 VALUES('Ubuntu Yakkety Yak 64 bits',' Ubuntu 16.10 Yakkety Yak 64 bits'
     ,'amd64'
     ,'yakkety64-amd64.xml'
     ,'yakkety64-volume.xml'
-    ,'http://releases.ubuntu.com/16.10/ubuntu-16.10-desktop-amd64.iso'
-    ,'3f50877c05121f7fd8544bef2d722824'
+    ,'http://releases.ubuntu.com/16.10/'
+    ,'ubuntu-16.10.*desktop-amd64.iso'
+    ,'http://releases.ubuntu.com/16.10/MD5SUMS'
     );
 INSERT INTO iso_images
-(name,description,arch,xml,xml_volume,url,md5)
+(name,description,arch,xml,xml_volume,url,file_re,md5_url)
 VALUES('Debian Jessie 64 bits netinst'
     ,'Debian 8.5.0 Jessie 64 bits (netsinst)'
     ,'amd64'
     ,'jessie-amd64.xml'
     ,'jessie-volume.xml'
-    ,'http://cdimage.debian.org/cdimage/archive/8.5.0/amd64/iso-cd/debian-8.5.0-amd64-netinst.iso'
-    ,'3e1b9029a0cf188730646c379d15073f'
+    ,'http://cdimage.debian.org/cdimage/archive/8.5.0/amd64/iso-cd/'
+    ,'debian-8.5.0-amd64-netinst.iso'
+    ,'http://cdimage.debian.org/cdimage/archive/8.5.0/amd64/iso-cd/MD5SUMS'
    );
+INSERT INTO iso_images
+(name,description,arch,xml,xml_volume,url, file_re, md5_url)
+VALUES('Ubuntu Zesty Zapus',' Ubuntu 17.04 Zesty Zapus 64 bits'
+    ,'amd64'
+    ,'zesty-amd64.xml'
+    ,'zesty-volume.xml'
+    ,'http://releases.ubuntu.com/17.04/'
+    ,'ubuntu-17.04.*desktop-amd64.iso'
+    ,'http://releases.ubuntu.com/17.04/MD5SUMS'
+    );
 

sql/mysql/iso_images.sql

 CREATE TABLE `iso_images` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
+  `file_re` char(64),
   `name` char(64) NOT NULL,
   `description` varchar(255),
   `arch` char(8),
   `xml`  varchar(64),
   `xml_volume` varchar(64),
   `url` varchar(255),
-  `md5` char(32),
+  `md5` varchar(32),
+  `md5_url` varchar(255),
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)
 );

sql/sqlite/insert_iso_dsl.sql

 INSERT INTO iso_images
-(name,description,arch,xml,xml_volume,url,md5)
+(name,description,arch,xml,xml_volume,url,file_re,md5,md5_url)
 VALUES('DSL 4.11 rc2'
     ,'Damn Small Linux 4.11 rc2'
     ,'i386'
     ,'dsl-i386.xml'
     ,'dsl-volume.xml'
-    ,'http://distro.ibiblio.org/damnsmall/release_candidate/dsl-4.11.rc2.iso'
-    ,'4b61c7892eefc23b71d4931bdcc3f1f1'
+    ,'http://distro.ibiblio.org/damnsmall/release_candidate/'
+    ,'dsl-4.11..*.iso'
+    ,''
+    ,'http://distro.ibiblio.org/damnsmall/release_candidate/dsl-4.11.rc2.iso.md5.txt'
 );
 

sql/sqlite/iso_images.sql

 CREATE TABLE `iso_images` (
   `id` integer NOT NULL PRIMARY KEY AUTOINCREMENT
+,  `file_re` char(64)
 ,  `name` char(64) NOT NULL
 ,  `description` varchar(255)
 ,  `arch` char(8)
 ,  `xml`  varchar(64)
 ,  `xml_volume` varchar(64)
 ,  `url` varchar(255)
-,  `md5` char(32)
+,  `md5` varchar(32)
+,  `md5_url` varchar(255)
 ,  UNIQUE (`name`)
 );

t/etc/sql.conf

@@ -18,3 +18,4 @@ sql:
   - ../../sql/data/insert_domain_drivers_types.sql
   - ../../sql/sqlite/vms.sql
   - ../../sql/sqlite/base_xml.sql
+  - ../../sql/data/insert_iso_images.sql

t/kvm/m10_md5.t

+use warnings;
+use strict;
+
+use Carp qw(confess);
+use Data::Dumper;
+use IPC::Run3;
+use Test::More;
+use Test::SQL::Data;
+
+use lib 't/lib';
+use Test::Ravada;
+
+my $test = Test::SQL::Data->new(config => 't/etc/sql.conf');
+
+use_ok('Ravada');
+my %ARG_CREATE_DOM = (
+      kvm => [ id_iso => 1 ]
+);
+
+my @VMS = reverse keys %ARG_CREATE_DOM;
+init($test->connector);
+my $USER = create_user("foo","bar");
+
+############################################3
+sub test_create_domain {
+    my $vm_name = shift;
+
+    my $vm = rvd_back->search_vm($vm_name);
+    ok($vm,"I can't find VM $vm_name") or return;
+
+    my $name = new_domain_name();
+
+    ok($ARG_CREATE_DOM{lc($vm_name)}) or do {
+        diag("VM $vm_name should be defined at \%ARG_CREATE_DOM");
+        return;
+    };
+    my @arg_create = @{$ARG_CREATE_DOM{$vm_name}};
+
+    my $domain;
+    eval { $domain = $vm->create_domain(name => $name
+                    , id_owner => $USER->id
+                    , @{$ARG_CREATE_DOM{$vm_name}})
+    };
+
+    ok($domain,"No domain $name created with ".ref($vm)." ".($@ or '')) or exit;
+    ok($domain->name
+        && $domain->name eq $name,"Expecting domain name '$name' , got "
+        .($domain->name or '<UNDEF>')
+        ." for VM $vm_name"
+    );
+
+    return $domain;
+
+}
+
+sub test_remove_domain {
+    my ($vm_name, $domain) = @_;
+
+    my @volumes = $domain->list_volumes();
+    ok(scalar@volumes,"Expecting some volumes, got :".scalar@volumes);
+
+    for my $file (@volumes) {
+        ok(-e $file,"Expecting volume $file exists, got : ".(-e $file or 0));
+    }
+    $domain->remove($USER);
+    for my $file (@volumes) {
+        ok(!-e $file,"Expecting no volume $file exists, got : ".(-e $file or 0));
+    }
+
+}
+
+sub test_isos {
+    my $vm = shift;
+
+    diag("testing isos");
+    my $sth = $test->connector->dbh->prepare(
+        "SELECT * FROM iso_images"
+    );
+    $sth->execute;
+
+    while (my $row = $sth->fetchrow_hashref) {
+        my $iso;
+        $iso = $vm->_search_iso($row->{id});
+
+        ok($iso,"Expecting a ISO description");
+
+        like($iso->{url},qr{.iso}) or exit;
+        like($iso->{url},qr($row->{file_re})) or exit   if $row->{file_re};
+#        diag($iso->{file_re}." -> ".$iso->{url})   if $row->{file_re};
+    }
+    $sth->finish;
+}
+
+#######################################################
+#
+
+clean();
+
+my $vm_name = 'kvm';
+my $vm = rvd_back->search_vm($vm_name);
+
+SKIP: {
+
+    my $msg = "SKIPPED: No virtual managers found";
+    if ($vm && $vm_name =~ /kvm/i && $>) {
+        $msg = "SKIPPED: Test must run as root";
+        $vm = undef;
+    }
+
+    skip($msg,10)   if !$vm;
+
+    my $domain = test_create_domain($vm_name );
+    test_remove_domain($vm_name, $domain);
+
+    test_isos($vm);
+}
+
+clean();
+
+done_testing();
+

t/vm/55_rename.t

@@ -222,7 +222,7 @@ for my $vm_name (qw( Void KVM )) {
         test_rename_twice($vm_name);
 
         my $domain_name = test_create_domain($vm_name);
-        test_rename_domain($vm_name, $domain_name)  or return;
+        test_rename_domain($vm_name, $domain_name)  or next;
         test_create_domain($vm_name, $domain_name);
     
         $domain_name = test_create_domain($vm_name);