Commit 57f2f677 authored by Mickaël Desfrênes's avatar Mickaël Desfrênes
Browse files

independent tasks for post treatment

parent cd830aab

resources/helpers.py

+10 −15
Original line number Diff line number Diff line 
import shutil



import exiftool

import hashlib

from django.core.files.base import File

from jama import settings

from resources import models

from django.contrib.auth.models import User

import re

import os

import unicodedata
@@ -15,7 +13,7 @@ from django.db.utils import IntegrityError 
from unidecode import unidecode

import logging

from typing import Union

from resources.tasks import post_upload_treatment

from resources.tasks import exif_task, iiif_task, ocr_task



logger = logging.getLogger(__name__)
@@ -225,9 +223,7 @@ def _copy_uploaded_file_to_destination( 
    return False





def handle_local_file(

    file_path: str, project: models.Project, defer_post_treatment: bool = False

) -> int:

def handle_local_file(file_path: str, project: models.Project) -> int:

    file_hash = _file_hash256(file_path)

    destination_file = models.hash_to_local_path(file_hash)

    os.makedirs(os.path.dirname(destination_file), exist_ok=True)
@@ -248,9 +244,6 @@ def handle_local_file( 
            post_treated=False,

        )



        if defer_post_treatment:

            post_upload_treatment(file.pk)

        else:

        set_exif_metas(file)

        make_iiif(file)

        make_ocr(file)
@@ -298,7 +291,9 @@ def handle_uploaded_file( 
    file.dump_infos_to_fs()

    logger.info("Added from upload: {} ({})".format(file.title, file.id))

    # always re-run post treatment, in a separate task

    post_upload_treatment(file.pk)

    iiif_task(file.pk)

    exif_task(file.pk)

    ocr_task(file.pk)

    return file.id

resources/tasks.py

+38 −9
Original line number Diff line number Diff line 
from huey.contrib.djhuey import db_task

import logging



logger = logging.getLogger(__name__)





@db_task()

def post_upload_treatment(file_id: int):

def exif_task(file_id: int):

    from resources.models import File

    from resources.helpers import make_ocr, make_iiif, set_exif_metas

    from resources.helpers import set_exif_metas



    try:

        f = File.objects.get(pk=file_id)

        set_exif_metas(f)

    except Exception as e:

        logger.warning("exit_task({}) failed".format(file_id))

        logger.info(repr(e))





@db_task()

def iiif_task(file_id: int):

    from resources.models import File

    from resources.helpers import make_iiif



    try:

        f = File.objects.get(pk=file_id)

        make_iiif(f)

    except Exception as e:

        logger.warning("iiif_task({}) failed".format(file_id))

        logger.info(repr(e))





@db_task()

def ocr_task(file_id: int):

    from resources.models import File

    from resources.helpers import make_ocr



    try:

        f = File.objects.get(pk=file_id)

        make_ocr(f)

    f.post_treated = True

    f.save()
 
    except Exception as e:

        logger.warning("ocr_task({}) failed".format(file_id))

        logger.info(repr(e))

rpc/views.py

+3 −3
Original line number Diff line number Diff line 
from django.db import connection

from django.http import HttpRequest, HttpResponse, JsonResponse, FileResponse

from django.http import HttpRequest, HttpResponse, JsonResponse

from django.views.decorators.http import require_http_methods

from django.views.decorators.csrf import csrf_exempt

from django.contrib.auth.models import User
@@ -31,7 +31,7 @@ from .fileresponse import RangedFileResponse 
from django.views.decorators.gzip import gzip_page

import logging

from functools import lru_cache

from resources.tasks import post_upload_treatment





importlib = __import__("importlib")
@@ -183,6 +183,7 @@ def _get_rpc_methods() -> dict: 
@transaction.atomic

@csrf_exempt

def rpc(request: HttpRequest) -> HttpResponse:

    req_id = None

    if request.method == "GET":

        return render(request, "docs.html", {"methods": _get_rpc_methods()})

    if request.method != "POST":
@@ -256,7 +257,6 @@ def upload_partial(request: HttpRequest) -> HttpResponse: 
            file_instance.title = file_name

            file_instance.original_name = file_name

            file_instance.save()

            post_upload_treatment(file_instance.pk)

            logger.info(

                "User {}({}) respawned file {}".format(

                    user.username, user.pk, file_instance.pk