Commit 6e3c10be authored by Mickaël Desfrênes's avatar Mickaël Desfrênes
Browse files

pipelines first pass

parent 9bf8e8e8

src/resources/helpers.py

+1 −1
Original line number Diff line number Diff line
@@ -218,7 +218,7 @@ def make_iiif(f: Union[models.File, int], force: bool = False): 
        )

        call(["chmod", "775", iiif_destination_file])

        f.tiled = True

        f.save()

        models.File.objects.filter(pk=f.pk).update(tiled=True)





def delete_exif_metas(f: models.File) -> int:

src/resources/migrations/0044_project_collections_pipelines_and_more.py

0 → 100644
+22 −0
Original line number Diff line number Diff line 
# Generated by Django 5.2.5 on 2025-10-27 07:41



from django.db import migrations, models





class Migration(migrations.Migration):

    dependencies = [

        ("resources", "0043_alter_collection_title_alter_file_original_name_and_more"),

    ]



    operations = [

        migrations.AddField(

            model_name="project",

            name="collections_pipelines",

            field=models.JSONField(null=True),

        ),

        migrations.AddField(

            model_name="project",

            name="resources_pipelines",

            field=models.JSONField(null=True),

        ),

    ]

src/resources/models.py

+128 −4
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ from django.contrib.auth.models import User 
import os

from jama import settings

import hashlib

from typing import Iterator, Union, List

from typing import Iterator, Union, Callable, List, Dict

from django.db.models.signals import post_save

from django.dispatch import receiver

from subprocess import call
@@ -15,6 +15,8 @@ import logging 
from PIL import Image, UnidentifiedImageError

from functools import cache

from django.db.models.query import QuerySet

from inspect import getmembers, isfunction

import importlib



Image.MAX_IMAGE_PIXELS = None
@@ -23,6 +25,27 @@ XLSX_MULTIPLE_VALUES_SEPARATOR = "\n--\n" 
logger = logging.getLogger(__name__)





@cache

def _get_available_pipelines() -> dict[str, Callable]:

    from resources import pipelines



    pipelines_list = {}

    for fn_name, _ in getmembers(pipelines, isfunction):

        if fn_name[0:1] == "_":

            continue

        pipelines_list[fn_name] = getattr(pipelines, fn_name)

    for app_name in settings.AUTO_REGISTER_APPS:

        try:

            pipelines_module = importlib.import_module("{}.pipelines".format(app_name))

            for fn_name, _ in getmembers(pipelines_module, isfunction):

                if fn_name[0:1] == "_":

                    continue

                pipelines_list[fn_name] = getattr(pipelines_module, fn_name)

        except ModuleNotFoundError:

            continue

    return pipelines_list





def _flatten_resource(

    resource: "Resource", known_metadatas: dict, metadatas_labels: List

) -> dict:
@@ -105,6 +128,13 @@ class APIKey(models.Model): 
        unique_together = (("key", "user"),)





class Pipeline:

    def __init__(self, name: str, callable_function: Callable, params: Dict):

        self.name = name

        self.callable_function = callable_function

        self.params = params





class Project(models.Model):

    """

    Project is used for sharing collections
@@ -128,7 +158,86 @@ class Project(models.Model): 
    ark_redirect = models.TextField(null=True)

    # Exiftool returns a lot of data and you may not need it in your

    # project. When set to false, the exiftool task is bypassed.

    use_exiftool = models.BooleanField(default=False)

    use_exiftool = models.BooleanField(default=False)  # deprecated, use pipelines

    # This is a dict of pipeline name -> params.

    # Pipelines are simple functions taking resources or collections and params.

    # Pipelines functions can be added in Jama apps in module pipelines.py

    # Each pipeline is executed every time a resource or a collection is saved.

    # Each pipeline has to determine itelf if it's up to the task (example:

    # object is not a Resource, I return) and has to take care of the conditions of

    # execution (example: start an async task or not).

    # Order of execution is not guaranteed and no return is expected.

    # This is basically per-project pluggable signals for Resources and Collections.

    resources_pipelines = models.JSONField(null=True)

    collections_pipelines = models.JSONField(null=True)



    def add_resources_pipeline(self, name: str, params: dict = None):

        if name not in _get_available_pipelines().keys():

            logger.warning(

                f'Adding unavailable pipeline "{name}" to project({self.pk}) ({self.label})'

            )

        if not self.resources_pipelines:

            self.resources_pipelines = {}

        self.resources_pipelines[name] = params



    def remove_resources_pipeline(self, name: str):

        if not self.resources_pipelines:

            self.resources_pipelines = {}

        self.resources_pipelines.pop(name)



    def add_collections_pipeline(self, name: str, params: dict = None):

        if name not in _get_available_pipelines().keys():

            logger.warning(

                f'Adding unavailable pipeline "{name}" to project({self.pk}) ({self.label})'

            )

        if not self.collections_pipelines:

            self.collections_pipelines = {}

        self.collections_pipelines[name] = params



    def remove_collections_pipeline(self, name: str):

        if not self.collections_pipelines:

            self.collections_pipelines = {}

        self.collections_pipelines.pop(name)



    def available_collections_pipelines(self) -> List[Pipeline]:

        pipelines = []

        available_pipelines = _get_available_pipelines()

        available_pipelines_names = available_pipelines.keys()

        if self.collections_pipelines:

            for pipeline_name, pipeline_params in self.collections_pipelines.items():

                if pipeline_name in available_pipelines_names:

                    pipelines.append(

                        Pipeline(

                            pipeline_name,

                            available_pipelines[pipeline_name],

                            pipeline_params,

                        )

                    )

        return pipelines



    def available_resources_pipelines(self) -> List[Pipeline]:

        pipelines = []

        available_pipelines = _get_available_pipelines()

        available_pipelines_names = available_pipelines.keys()

        if self.resources_pipelines:

            for pipeline_name, pipeline_params in self.resources_pipelines.items():

                if pipeline_name in available_pipelines_names:

                    pipelines.append(

                        Pipeline(

                            pipeline_name,

                            available_pipelines[pipeline_name],

                            pipeline_params,

                        )

                    )

        return pipelines



    def process_pipelines_for_resource(self, resource: Union["Resource", "File"]):

        for pipeline in self.available_resources_pipelines():

            pipeline.callable_function(resource, pipeline.params)



    def process_pipelines_for_collection(self, collection: "Collection"):

        for pipeline in self.available_collections_pipelines():

            pipeline.callable_function(collection, pipeline.params)



    def __str__(self):

        return self.label
@@ -496,7 +605,7 @@ class File(Resource): 
            )

            boxes = pytesseract.image_to_boxes(im, output_type=pytesseract.Output.DICT)

            self.text_boxes = boxes

            self.save()

            File.objects.filter(pk=self.pk).update(text_boxes=boxes)

        except UnidentifiedImageError:

            pass
@@ -534,6 +643,18 @@ class File(Resource): 


    def save(self, *args, **kwargs):

        self.ptr_project = self.project

        if not self.denormalized_image_height and self.should_have_iiif:

            try:

                with Image.open(self.local_path()) as image:

                    width, height = image.size

                    self.denormalized_image_height = height

                    self.denormalized_image_width = width

            except FileNotFoundError:

                logger.warning(f"Could not open file {self.local_path()}")

            except UnidentifiedImageError:

                logger.warning(f"Could not identify file {self.local_path()}")

            except Exception:

                logger.warning(f"Could not get width and height from file({self.pk})")

        super(File, self).save(*args, **kwargs)



    @property
@@ -894,17 +1015,20 @@ class ProjectAccess(models.Model): 




@receiver(post_save, sender=Resource)

def resource_post_save(sender, instance: Resource, **kwargs):  # noqa

def resource_post_save(sender, instance: Resource, **kwargs):

    if not instance.ark:

        tasks.set_ark_to_resource(instance.pk)

    instance.ptr_project.process_pipelines_for_resource(instance)





@receiver(post_save, sender=File)

def file_post_save(sender, instance: File, **kwargs):  # noqa

    if not instance.ark:

        tasks.set_ark_to_resource(instance.pk)

    instance.project.process_pipelines_for_resource(instance)





@receiver(post_save, sender=Collection)

def collection_post_save(sender, instance: Collection, **kwargs):  # noqa

    tasks.set_ark_to_collection(instance.pk)

    instance.project.process_pipelines_for_collection(instance)

src/resources/pipelines.py

0 → 100644
+56 −0
Original line number Diff line number Diff line 
from typing import Dict, Union

from .models import Resource, Collection, File



import logging



logger = logging.getLogger(__name__)



# Pipelines are simple functions taking resources or collections and params.

# Pipelines functions can be added in Jama apps in module pipelines.py

# Each pipeline is executed every time a resource or a collection is saved.

# Each pipeline has to determine itelf if it's up to the task (example:

# object is not a Resource, I return) and has to take care of the conditions of

# execution (example: start an async task or not).

# Order of execution is not guaranteed and no return is expected.

# This is basically per-project pluggable signals for Resources and Collections.



# /!\ Since pipelines are executed upon Resource, File and Collection post-save event,

# pipelines and subsequent code should never call Resource, File or Collection save(), at

# the risk of creating a loop.

# Use QuerySet update() if you need to update a row, don't use the model's save() method.





def make_iiif(obj: Union[Resource, File, Collection], parameters: Dict = None):

    if type(obj) is Collection:

        return

    if obj.file and obj.file.should_have_iiif:

        from resources.tasks import iiif_task



        iiif_task(obj.file.pk)





def tesseract_ocr(obj: Union[Resource, File, Collection], parameters: Dict = None):

    if type(obj) is Collection:

        return

    if obj.file and obj.file.should_have_iiif:

        from resources.tasks import ocr_task



        ocr_task(obj.file.pk)





def exiftool(obj: Union[Resource, File, Collection], parameters: Dict = None):

    if type(obj) is Collection:

        return

    if obj.file:

        from resources.tasks import exif_task



        exif_task(obj.file.pk)





def arkify(obj: Union[Resource, File, Collection], parameters):

    from resources.tasks import set_ark_to_collection, set_ark_to_resource



    if type(obj) is Collection:

        set_ark_to_collection(obj.pk)

    if type(obj) in [Resource, File]:

        set_ark_to_resource(obj.pk)

src/resources/tasks.py

+2 −10
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ from django.urls import reverse 
from django.utils import timezone

from django.db import connection

from resources import models

from PIL import Image

import os

import time

import shutil
@@ -26,13 +25,6 @@ def exif_task(file_id: int): 
    try:

        f = File.objects.get(pk=file_id)

        set_exif_metas(f)

        if f.should_have_iiif:

            with Image.open(f.local_path()) as image:

                width, height = image.size

                f.denormalized_image_width = width

                f.denormalized_image_height = height

                f.save()



    except Exception as e:

        logger.warning("exit_task({}) failed".format(file_id))

        logger.info(repr(e))
@@ -95,7 +87,7 @@ def set_ark_to_resource(resource_id: int, location: str = None): 
        else:

            ark_name = client.create(location)

            resource.ark = ark_name

            resource.save()

            Resource.objects.filter(pk=resource.pk).update(ark=ark_name)





@db_task(retries=2, retry_delay=600)
@@ -131,7 +123,7 @@ def set_ark_to_collection(collection_id: int, location: str = None): 
        else:

            ark_name = client.create(location)

            collection.ark = ark_name

            collection.save()

            Collection.objects.filter(pk=collection.pk).update(ark=ark_name)





@db_task()