Refactor: iso downloading (#1677)

refactor(frontend): improve ISO download in new machine form

lib/Ravada.pm

@@ -2293,6 +2293,7 @@ sub _upgrade_tables {
     $self->_upgrade_table('iso_images','min_disk_size','int (11) DEFAULT NULL');
     $self->_upgrade_table('iso_images','options','varchar(255)');
     $self->_upgrade_table('iso_images','has_cd','int (1) DEFAULT "1"');
+    $self->_upgrade_table('iso_images','downloading','int (1) DEFAULT "0"');
 
     $self->_upgrade_table('users','language','char(40) DEFAULT NULL');
     if ( $self->_upgrade_table('users','is_external','int(11) DEFAULT 0')) {
@@ -3200,6 +3201,13 @@ sub clean_old_requests {
     $self->_clean_requests('cleanup');
 }
 
+sub _clean_interrupted_downloads($self) {
+    my $sth = $CONNECTOR->dbh->prepare("UPDATE iso_images "
+        ." SET downloading=0 WHERE downloading=1"
+    );
+    $sth->execute();
+}
+
 =head2 process_requests
 
 This is run in the ravada backend. It processes the commands requested by the fronted
@@ -3443,7 +3451,7 @@ sub _kill_dead_process($self) {
         "SELECT id,pid,command,start_time "
         ." FROM requests "
         ." WHERE start_time<? "
-        ." AND status = 'working' "
+        ." AND ( status like 'working%' OR status like 'downloading%') "
         ." AND pid IS NOT NULL "
     );
     $sth->execute(time - 2);

lib/Ravada/Front.pm

@@ -521,7 +521,7 @@ sub list_vm_types {
 
     return $self->{cache}->{vm_types} if $self->{cache}->{vm_types};
 
-    my $result = [@VM_TYPES];
+    my $result = [sort @VM_TYPES];
 
     $self->{cache}->{vm_types} = $result if $result->[0];
 

lib/Ravada/Request.pm

@@ -756,6 +756,8 @@ sub _validate_create_domain($self) {
 
     $self->_validate_clone($id_base, $id_owner) if $id_base;
 
+    $self->_check_downloading();
+
     return if $owner->is_admin
             || $owner->can_create_machine()
             || ($id_base && $owner->can_clone);
@@ -764,6 +766,65 @@ sub _validate_create_domain($self) {
 
 }
 
+sub _check_downloading($self) {
+    my $id_iso = $self->defined_arg('id_iso');
+    my $iso_file = $self->defined_arg('iso_file');
+
+    return if !$id_iso && !$iso_file;
+
+    my $sth = $$CONNECTOR->dbh->prepare("SELECT id,downloading,device "
+        ." FROM iso_images "
+        ." WHERE (id=? or device=?) "
+    );
+    $sth->execute($id_iso,$iso_file);
+    my ($id_iso2,$downloading, $device) = $sth->fetchrow;
+    return if !$downloading && $device;
+
+    my $req_download = _search_request('download', id_iso => $id_iso2);
+    if (!$device && !$req_download) {
+        $req_download = Ravada::Request->download(
+            id_iso => $id_iso2
+            ,uid => Ravada::Utils::user_daemon->id
+        );
+    }
+    if (! $req_download) {
+        _mark_iso_downloaded($id_iso2);
+    } else {
+        $self->after_request($req_download->id);
+    }
+}
+
+sub _mark_iso_downloaded($id_iso) {
+    my $sth = $$CONNECTOR->dbh->prepare("UPDATE iso_images "
+        ." set downloading=0 "
+        ." WHERE id=?"
+    );
+    $sth->execute($id_iso);
+}
+
+sub _search_request($command,%fields) {
+    my $sth= $$CONNECTOR->dbh->prepare(
+        "SELECT id, args FROM requests WHERE command = ?"
+        ." AND status <> 'done' "
+    );
+    $sth->execute($command);
+
+    while ( my ($id, $args_json) = $sth->fetchrow ) {
+        return Ravada::Request->open($id) if !keys %fields;
+
+        my $args = decode_json($args_json);
+        my $found=1;
+        for my $key (keys %fields) {
+            if ( $args->{$key} ne $fields{$key} ) {
+                $found = 0;
+                last;
+            }
+        }
+        return Ravada::Request->open($id) if $found;
+    }
+
+}
+
 sub _validate_clone($self
                 , $id_base= $self->args('id_domain')
                 , $uid=$self->args('uid')) {

lib/Ravada/VM/KVM.pm

@@ -1129,6 +1129,14 @@ sub _fix_pci_slots {
 
 }
 
+sub _set_iso_downloading($self, $iso,$value) {
+    my $sth = $$CONNECTOR->dbh->prepare(
+        "UPDATE iso_images SET downloading=?"
+        ." WHERE id=?"
+    );
+    $sth->execute($value,$iso->{id});
+}
+
 sub _iso_name($self, $iso, $req=undef, $verbose=1) {
 
     my $iso_name;
@@ -1156,7 +1164,10 @@ sub _iso_name($self, $iso, $req=undef, $verbose=1) {
                  ." from $iso->{url}. It may take several minutes"
         )   if $req;
         _fill_url($iso);
+
+        $self->_set_iso_downloading($iso,1);
         my $url = $self->_download_file_external($iso->{url}, $device, $verbose, $test);
+        $self->_set_iso_downloading($iso,0);
         $req->output($url) if $req;
         $self->_refresh_storage_pools();
         die "Download failed, file $device missing.\n"
@@ -1287,6 +1298,7 @@ sub _download_file_external($self, $url, $device, $verbose=1, $test=0) {
     confess "ERROR: wget missing"   if !$WGET;
 
     return $self->_download_file_external_headers($url)    if $test;
+    return $url if -e $device;
 
     my @cmd = ($WGET,'-nv',$url,'-O',$device);
     my ($in,$out,$err) = @_;
@@ -1462,6 +1474,11 @@ sub _fetch_filename {
     if (@found) {
         $row->{device} = $found[0]->get_path;
         ($row->{filename}) = $found[0]->get_path =~ m{.*/(.*)};
+        my $sth = $$CONNECTOR->dbh->prepare(
+            "UPDATE iso_images SET device=?"
+            ." WHERE id=?"
+        );
+        $sth->execute($row->{device}, $row->{id});
         return;
     } else {
         @found = $self->_search_url_file($row->{url}, $row->{file_re}) if !@found;

lib/Ravada/WebSocket.pm

@@ -28,6 +28,7 @@ my %SUB = (
                   list_alerts => \&_list_alerts
                   ,list_bases => \&_list_bases
                   ,list_isos  => \&_list_isos
+                  ,list_iso_images  => \&_list_iso_images
                   ,list_nodes => \&_list_nodes
                ,list_machines => \&_list_machines
           ,list_machines_tree => \&_list_machines_tree
@@ -106,6 +107,14 @@ sub _list_isos($rvd, $args) {
     return $rvd->iso_file($type);
 }
 
+sub _list_iso_images($rvd, $args) {
+    my ($type) = $args->{channel} =~ m{/(.*)};
+    $type = 'KVM' if !defined $type;
+
+    my $images=$rvd->list_iso_images($type);
+    return $images;
+}
+
 sub _list_nodes($rvd, $args) {
     my ($type) = $args->{channel} =~ m{/(.*)};
     my @nodes = $rvd->list_vms($type);

public/css/admin.css

@@ -82,3 +82,18 @@ span.request{
 .comment {
     color: gray
 }
+
+span.info {
+    color: black;
+    text-width: italic;
+}
+
+input.ng-invalid {
+  background-color: pink;
+  border: 1px solid red;
+}
+
+input.ng-invalid.ng-touched {
+    border: 1px solid red;
+}
+

public/css/main.css

@@ -359,4 +359,8 @@ div.tab button.active {
 
 input.ng-invalid {
   background-color: pink;
+
+span.info {
+    color: black;
+    text-width: italic;
 }

public/js/admin.js

@@ -69,12 +69,18 @@ ravadaApp.directive("solShowMachine", swMach)
 
   function newMachineCtrl($scope, $http) {
 
-    $http.get('/list_vm_types.json').then(function(response) {
-            $scope.backends = response.data;
-            $scope.backend = response.data[0];
-            $scope.loadTemplates();
-        $scope.list_machine_types($scope.backend);
-    });
+      $scope.init = function(url) {
+          $scope.url = url;
+          $scope.images = [];
+          subscribe_list_isos(url);
+          subscribe_list_images(url);
+          subscribe_list_machines(url);
+          $http.get('/list_vm_types.json').then(function(response) {
+              $scope.backends = response.data;
+              $scope.backend = response.data[0];
+              $scope.loadTemplates(url);
+          });
+      }
 
       $scope.list_machine_types = function(backend) {
           $http.get('/list_machine_types.json?vm_type='+backend).then(function(response) {
@@ -84,25 +90,60 @@ ravadaApp.directive("solShowMachine", swMach)
       };
 
       $scope.loadTemplates = function() {
-        $http.get('/list_images.json',{
-          params: {
-            backend: $scope.backend
-          }
-        }).then(function(response) {
-                $scope.images = response.data;
-        });
+          $scope.list_machine_types($scope.backend);
+          subscribe_list_images($scope.backend);
       }
+
+      /*
       $http.get('/iso_file.json').then(function(response) {
               $scope.isos = response.data;
       });
+      */
 
+      subscribe_list_isos = function() {
+          var ws = new WebSocket($scope.url);
+          ws.onopen = function(event) { ws.send('list_isos') };
+          ws.onmessage = function(event) {
+              var data = JSON.parse(event.data);
+              $scope.$apply(function () {
+                  $scope.isos = data;
+              });
+          }
+      };
 
+      subscribe_list_machines = function() {
+          var ws = new WebSocket($scope.url);
+          ws.onopen = function(event) { ws.send('list_machines') };
+          ws.onmessage = function(event) {
+              var data = JSON.parse(event.data);
+              $scope.$apply(function () {
+                  $scope.base = data;
+              });
+          }
+      };
+
+
+
+      subscribe_list_images = function(backend) {
+          var ws = new WebSocket($scope.url);
+          ws.onopen = function(event) { ws.send('list_iso_images/'+backend) };
+          ws.onmessage = function(event) {
+              var data = JSON.parse(event.data);
+              $scope.$apply(function () {
+                  $scope.images = data;
+              });
+          }
+      };
+
+
+      /*
       $http.get('/list_lxc_templates.json').then(function(response) {
               $scope.templates_lxc = response.data;
       });
-      $scope.iso_download=function(id_iso) {
-            $http.get('/iso/download/'+id_iso+'.json').then(function() {
-                window.location.href = '/admin/machines';
+      */
+      $scope.iso_download=function(iso) {
+            iso.downloading=1;
+            $http.get('/iso/download/'+iso.id+'.json').then(function() {
             });
       };
       $scope.name_duplicated = false;
@@ -118,17 +159,17 @@ ravadaApp.directive("solShowMachine", swMach)
       $scope.iso = { arch: 'unknown' };
       $scope.machine_types = { };
 
-      $scope.change_iso = function(id) {
-          $scope.id_iso_id = id.id;
-          if (id.min_disk_size != null) {
-            $scope.min_size = id.min_disk_size;
+      $scope.change_iso = function(iso) {
+          $scope.id_iso_id = iso.id;
+          if (iso.min_disk_size != null) {
+            $scope.min_size = iso.min_disk_size;
             $scope.showMinSize = true;
           }
           else {
             $scope.showMinSize = false;
             $scope.min_size = 1;
           }
-          return (id.device != null) ? id.device : "";
+          return (iso.device != null) ? iso.device : "";
       };
 
       $scope.onIdIsoSelected = function() {
@@ -214,13 +255,8 @@ ravadaApp.directive("solShowMachine", swMach)
               }, 3000);
           }
         );
-
       };
 
-      $http.get('/list_machines.json').then(function(response) {
-              $scope.base = response.data;
-      });
-
       $scope.swap = {
           enabled: true
           ,value: 1
@@ -313,7 +349,6 @@ ravadaApp.directive("solShowMachine", swMach)
                           n_active_current++;
                           if (!mach.show) {
                               $scope.n_active_hidden++;
-                              console.log(mach.name);
                           }
                       }
                   }

script/rvd_back

@@ -260,6 +260,7 @@ sub done_request {
 sub clean_old_requests {
     my $ravada = Ravada->new( %CONFIG );
     $ravada->clean_old_requests();
+    $ravada->_clean_interrupted_downloads();
 }
 
 sub autostart_machines {

script/rvd_front

@@ -2455,10 +2455,21 @@ sub admin {
     $c->render( template => 'main/admin_'.$page);
 };
 
+sub _search_id_iso($name) {
+    my $sth = $RAVADA->_dbh
+        ->prepare("SELECT id FROM iso_images WHERE name=?");
+    $sth->execute($name);
+    my ($id) = $sth->fetchrow;
+    return $id;
+}
+
 sub new_machine {
     my $c = shift;
     my @error ;
     if ($c->param('submit')) {
+        if (!$c->param('id_iso_id') && $c->param('id_iso')) {
+            $c->param('id_iso_id', _search_id_iso($c->param('id_iso')));
+        }
         $c->param('id_iso', $c->param('id_iso_id')) if $c->param('id_iso_id');
         push @error,("Name is mandatory")   if !$c->param('name');
         push @error,("Invalid name '".$c->param('name')."'"
@@ -2519,12 +2530,12 @@ sub req_new_domain {
            name => $name
         ,id_iso => $c->param('id_iso')
         ,id_template => $c->param('id_template')
-        ,iso_file => $c->param('iso_file')
         ,vm=> $vm
         ,id_owner => $USER->id
         ,swap => $swap
         ,data => $data
     );
+    $args{iso_file} => $c->param('iso_file') if defined $c->param('iso_file');
     $args{options}->{uefi} = 1 if $bios && $bios eq 'UEFI';
     $args{options}->{machine} = $machine if $machine;
     $args{memory} = int($c->param('memory')*1024*1024)  if $c->param('memory');

t/17_templates.t

@@ -79,7 +79,7 @@ sub _check_html_lint($url, $content, $option = {}) {
             || $error->errtext =~ /Unknown attribute "image.* for tag <div/
             || $error->errtext =~ /Unknown attribute "ipaddress"/
             || $error->errtext =~ /Unknown attribute "sizes" for tag .link/
-            || $error->errtext =~ /Unknown attribute "(uib|typeahead).*?" for tag .input/
+            || $error->errtext =~ /Unknown attribute "(autocomplete|uib|typeahead).*?" for tag .input/
          ) {
              next;
          }

templates/main/new_machine.html.ep

@@ -3,7 +3,7 @@
     %= include 'bootstrap/header'
     <body id="page-top" data-spy="scroll" data-target=".navbar-fixed-top" role="document">
         <div id="wrapper">
-            <div ng-controller="new_machine" ng-init="name='<%= $name %>'">
+            <div ng-controller="new_machine" ng-init="name='<%= $name %>';init('<%= url_for('ws_subscribe')->to_abs %>')">
 %=              include 'bootstrap/navigation'
                 <div id="page-wrapper">
 %=                  include 'ng-templates/new_machine'

templates/ng-templates/new_machine.html.ep

@@ -15,7 +15,7 @@
         <ul class="nav nav-tabs" id="myTab" role="tablist">
             <li class="nav-item"><a class="nav-link active" href="#fromtemplate" data-toggle="tab"><%=l 'From Template' %></a></li>
 %           if ( $_user->is_admin ) {
-            <li class="nav-item" ng-show="base"><a class="nav-link" href="#frommachine" data-toggle="tab"><%=l 'From Machine' %></a></li>
+            <li class="nav-item"><a class="nav-link" href="#frommachine" data-toggle="tab"><%=l 'From Machine' %></a></li>
 %           }
         </ul>
         <div class="tab-content" id="myTabContent">

templates/ng-templates/new_machine_other.html.ep

@@ -6,7 +6,10 @@
                         <input class="form-control" type="hidden" name="id_base" value="{{src_machine.id}}">
                         <label for="src_machine" class="col-xl-3 col-form-label"><%=l 'Source Machine' %></label>
                         <div class="col-lg-9">
+                            <i ng-hide="base"
+                                class="fas fa-sync-alt fa-spin"></i>
                             <select class="form-control"
+                                ng-show="base"
                                 name ="src_machine"
                                 ng-model="src_machine"
                                 ng-options="item.name for item in base | orderBy:'name' track by item.id "
@@ -18,7 +21,7 @@
                 <div ng-show="src_machine">
                     <div class="form-group row">
                         <div class="col-xl-3">
-                            <h5><strong><%=l 'Destination Machine' %></strong></h5>
+                            <h5><strong><%=l 'New Machine' %></strong></h5>
                         </div>
                     </div>
                     <div class="form-group row">

templates/ng-templates/new_machine_template.html.ep

 <div class="tab-pane fade show active" id="fromtemplate" role="tabpanel">
     <div class="card-body">
-        <div ng-hide="isos && isos.length">
+        <div ng-show="!backends">
             Loading ... <i class="fas fa-sync-alt fa-spin"></i>
         </div>
         <form name="new_machineForm" role="form" method="post"
+                class="needs-validation"
                 action="/new_machine.html" novalidate ng-cloak="1"
-                ng-show="isos && isos.length">
+                ng-show="backends">
             <div class="form-group row">
                 <label for="backend" class="col-xl-3 col-form-label"><%=l 'Backend' %> <a
                   title="Choose the virtualization type of the Virtual Machine." ng-show="backends.length > 1"><i class="fa fa-info-circle"></i></a></label>
@@ -24,24 +25,31 @@
                 <div class="from-group row">
                     <label for="id_iso" class="col-xl-3 col-form-label"><%=l 'Select Template' %> <a
                       title="Choose the OS you want to install."><i class="fa fa-info-circle"></i></a></label>
-                    <div class="col-lg-9">
+                    <div class="col-lg-9" ng-show="images">
                         <input type="hidden" name="id_iso_id" ng-value="id_iso_id">
                         <input type="text" class="form-control" placeholder="<%=l 'Type the template name' %>"
                                    name="id_iso"
                                    ng-model="id_iso" 
                                    required=""
+                                   autocomplete="off"
+                                   ng-show="images.length"
+                                   ng-class="{'ng-invalid': !id_iso.id }"
                                    uib-typeahead="item as item.name for item in getVisualizableObjects($viewValue, images, 'name')"
                                    typeahead-min-length="0"
                                    typeahead-on-select="onIdIsoSelected()">
+                        <i ng-hide="images.length"
+                            class="fas fa-sync-alt fa-spin"></i>
+
                         <div class="mb-2" ng-show="id_iso.description" >
                             <small ng-bind-html="id_iso.description">{{id_iso.description}}</small>
                         </div>
                         <div ng-show="id_iso.name && ( !id_iso.device && id_iso.url )
+                                && !id_iso.downloading
                                 && (iso_file == '<NONE>' || !iso_file) ">
                             <font color="#500000"><%=l 'This ISO image has not been already downloaded. This may take many minutes, even hours until the file is fetched from Internet.' %></font>
-                            <a type="button" class="btn btn-warning"
-                                ng-click="iso_download(id_iso.id)"
-                            > <%=l "Download now" %></a>
+                            <input type="checkbox" name="_download_"
+                                ng-click="iso_download(id_iso)"
+                            /> <%=l "Download now" %>
                         </div>
                     </div>
                 </div>
@@ -54,10 +62,10 @@
                             <input type="text" class="form-control" placeholder="<%=l 'Type the ISO pathname' %>"
                                    name="iso_file"
                                    ng-model="iso_file" 
-                                   ng-hide="refresh_working || !isos"
-                                   required=""
+                                   ng-hide="refresh_working || !isos ||id_iso.downloading"
                                    uib-typeahead="item for item in getVisualizableObjects($viewValue, isos)"
                                    typeahead-min-length="0"/>
+                        <span class="info" ng-show="id_iso.downloading">This ISO is being downloaded. The virtual machine will be created after.</span>
                         </div>
                 </div>
             </div>
@@ -210,12 +218,16 @@
                 <div ng-show="new_machineForm.name.$error.pattern" class="alert alert-warning" role="alert">
                         <strong><%=l 'Error' %></strong> <%=l 'The machine name must contain only alphabetic, numbers, undercores and dashes.' %>
                 </div>
+                <div ng-show="id_iso && !id_iso.id" class="alert alert-warning"
+                    role="alert">
+                    <strong><%=l 'Error' %></strong> <%=l 'Invalid Template' %>
+                </div>
             </div>
 
             <div class="form-group row">
                 <button type="reset" class="btn btn-outline-secondary mr-2" onclick = "location='/admin/machines'"><%=l 'Cancel' %></button>
                 <input type="submit" class="btn btn-primary" name="submit" value="<%=l 'Create' %>"
-                     ng-disabled="new_machineForm.$invalid || name_duplicated">
+                     ng-disabled="new_machineForm.$invalid || name_duplicated || (id_iso && !id_iso.id)">
             </div>
         </form>
     </div>