Commit 68d8280d authored by Mickaël Desfrênes's avatar Mickaël Desfrênes
Browse files

defer post upload treatment

parent e65fbf71

resources/helpers.py

+34 −26
Original line number Diff line number Diff line
@@ -14,6 +14,8 @@ from subprocess import call 
from django.db.utils import IntegrityError

from unidecode import unidecode

import logging

from typing import Union

from resources.tasks import post_upload_treatment



logger = logging.getLogger(__name__)
@@ -96,21 +98,25 @@ def get_file_type_from_extension(extension: str) -> models.FileType: 
    return file_type





def make_ocr(f: models.File, refresh: bool = False):

def make_ocr(f: Union[models.File, int], refresh: bool = False):

    """

    Performs OCR on file if OCR output not yet available.

    Use refresh = True to force OCR.

    """

    if type(f) == int:

        f = models.File.objects.get(pk=f)

    f.make_ocr(refresh)





def make_iiif(f: models.File):

def make_iiif(f: Union[models.File, int]):

    """

    Make a Tiled TIF ready for IIIF server.



    Will silently return if not a image format

    or if tiled TIF already exists.

    """

    if type(f) == int:

        f = models.File.objects.get(pk=f)

    if f.should_have_iiif():

        iiif_destination_dir = "{}{}{}{}{}{}".format(

            settings.IIIF_DIR,
@@ -169,13 +175,15 @@ def delete_exif_metas(f: models.File) -> int: 
    return deleted_counter





def set_exif_metas(f: models.File) -> int:

def set_exif_metas(f: Union[models.File, int]) -> int:

    """

    Will run Exiftool on given models.File.local_path().

    Extracted data is then converted to Metadatas.

    ExifTool MetadataSet and Metadatas always have public_project

    as project.

    """

    if type(f) == int:

        f = models.File.objects.get(pk=f)

    added_counter = 0

    exiftools_metas = file_metadatas(f.local_path())

    if "ExifTool:Error" not in exiftools_metas:
@@ -217,7 +225,9 @@ def _copy_uploaded_file_to_destination( 
    return False





def handle_local_file(file_path: str, project: models.Project) -> int:

def handle_local_file(

    file_path: str, project: models.Project, defer_post_treatment: bool = False

) -> int:

    file_hash = _file_hash256(file_path)

    destination_file = models.hash_to_local_path(file_hash)

    os.makedirs(os.path.dirname(destination_file), exist_ok=True)
@@ -238,16 +248,12 @@ def handle_local_file(file_path: str, project: models.Project) -> int: 
            post_treated=False,

        )



        # Run exiftool on any file type (image or not)

        # and create metadatas with extracted info

        if defer_post_treatment:

            post_upload_treatment(file.pk)

        else:

            set_exif_metas(file)



        # Prepare for IIIF if needed

            make_iiif(file)



            make_ocr(file)



        # mark treatments as ok

            file.post_treated = True

            file.save()
@@ -259,7 +265,10 @@ def handle_local_file(file_path: str, project: models.Project) -> int: 




def handle_uploaded_file(

    uploaded_file: File, project: models.Project, force_file_name=None

    uploaded_file: File,

    project: models.Project,

    force_file_name=None,

    defer_post_treatment: bool = False,

) -> int:

    upload_hash = hash_upload(uploaded_file)

    destination_file = models.hash_to_local_path(upload_hash)
@@ -282,11 +291,10 @@ def handle_uploaded_file( 
        except IntegrityError:

            raise ConcurrencyError()



        # Run exiftool on any file type (image or not)

        # and create metadatas with extracted info

        if defer_post_treatment:

            post_upload_treatment(file.pk)

        else:

            set_exif_metas(file)



        # Prepare for IIIF if needed

            make_iiif(file)

            make_ocr(file)

            file.post_treated = True

resources/tasks.py

0 → 100644
+14 −0
Original line number Diff line number Diff line 
from huey.contrib.djhuey import db_task





@db_task()

def post_upload_treatment(file_id: int):

    from resources.models import File

    from resources.helpers import make_ocr, make_iiif, set_exif_metas



    f = File.objects.get(file_id)

    set_exif_metas(f)

    make_iiif(f)

    make_ocr(f)

    f.post_treated = True

    f.save()

rpc/views.py

+4 −1
Original line number Diff line number Diff line
@@ -305,7 +305,10 @@ def upload_partial(request: HttpRequest) -> HttpResponse: 
            try:

                with transaction.atomic():

                    file_id = handle_uploaded_file(

                        File(f), project, force_file_name=file_name

                        File(f),

                        project,

                        force_file_name=file_name,

                        defer_post_treatment=True,

                    )

                    f.close()

                    _silent_rmdir(partials_dir)