Commit 98b49387 authored by Mickaël Desfrênes's avatar Mickaël Desfrênes
Browse files

add xlsx upload and document limits

parent a4902c78
Loading
Loading
Loading
Loading
+15 −0
Original line number Diff line number Diff line
@@ -54,6 +54,17 @@ CORS. Après mise à jour, chaque installation doit vérifier ces variables :
| `JAMA_PARTIAL_UPLOADS_DIR` | `${JAMA_VAR_DIR}/partial_uploads`    | Répertoire des morceaux d'upload incomplets.                                                                                                                                                                   |
| `JAMA_UPLOAD_MAX_CHUNK_SIZE` | `134217728`                          | Taille maximale acceptée pour un morceau d'upload, en octets. La valeur par défaut correspond à 128 Mio.                                                                                                       |
| `JAMA_UPLOAD_MAX_TOTAL_CHUNKS` | `10000`                              | Nombre maximal de morceaux autorisés pour un upload.                                                                                                                                                           |
| `JAMA_XLSX_UPLOADS_DIR` | `${JAMA_VAR_DIR}/xlsx_uploads` | Répertoire privé conservant les imports XLSX jusqu'à leur traitement en arrière-plan. |
| `JAMA_XLSX_UPLOAD_MAX_SIZE` | `26214400` | Taille compressée maximale d'un import XLSX, en octets. |
| `JAMA_XLSX_MAX_UNCOMPRESSED_SIZE` | `104857600` | Taille décompressée cumulée maximale des entrées d'une archive XLSX. |
| `JAMA_XLSX_MAX_ZIP_ENTRIES` | `1000` | Nombre maximal d'entrées dans l'archive XLSX. |
| `JAMA_XLSX_MAX_COMPRESSION_RATIO` | `200` | Ratio de compression maximal accepté pour une entrée XLSX. |
| `JAMA_XLSX_MAX_ROWS` | `50000` | Nombre maximal de lignes dans la feuille importée. |
| `JAMA_XLSX_MAX_COLUMNS` | `256` | Nombre maximal de colonnes dans la feuille importée. |
| `JAMA_XLSX_MAX_CELLS` | `500000` | Nombre maximal de cellules parcourues pendant un import. |
| `JAMA_XLSX_MAX_CELL_LENGTH` | `32767` | Longueur maximale d'une valeur textuelle de cellule. |
| `JAMA_XLSX_MAX_AFFECTED_OBJECTS` | `100000` | Nombre maximal de ressources et collections mises à jour après expansion des cascades. |
| `JAMA_XLSX_MAX_METADATA_WRITES` | `500000` | Nombre maximal estimé de valeurs de métadonnées écrites après expansion des cascades. |
| `JAMA_TMP_THUMBNAILS_DIR` | répertoire temporaire système        | Répertoire de cache des miniatures temporaires générées par les vues `resources`.                                                                                                                              |
| `JAMA_TMP_THUMBNAILS_MAX_AGE_SECONDS` | `604800`                             | Âge maximal des miniatures temporaires avant suppression par la tâche de nettoyage. La valeur par défaut correspond à 7 jours. Mettre `0` pour désactiver le nettoyage.                                                   |
| `JAMA_THUMBNAIL_MAX_SIZE` | `2000`                               | Taille maximale acceptée pour les miniatures simples générées par `resources/simple_thumb`.                                                                                                                    |
@@ -62,6 +73,10 @@ CORS. Après mise à jour, chaque installation doit vérifier ces variables :
| `JAMA_MEDIA_SUBPROCESS_TIMEOUT_SECONDS` | `1800`                               | Timeout appliqué aux outils médias lancés en sous-processus (`ffmpeg`, `ffprobe`, `convert`, `vips`, `pdftoppm`, `pdftotext`, etc.). La valeur par défaut correspond à 30 minutes. Mettre `0` pour désactiver le timeout. |
| `JAMA_STATIC_ROOT` | `${JAMA_VAR_DIR}/static`             | Destination de `collectstatic` pour les fichiers statiques en production.                                                                                                                                      |

L'endpoint d'import de métadonnées XLSX exige l'en-tête `X-Project`. Le fichier
est validé puis conservé dans `JAMA_XLSX_UPLOADS_DIR` jusqu'à son traitement par
la tâche d'arrière-plan, qui le supprime après réussite ou échec.

## Base de données et cache

| Variable | Défaut | Description |
+14 −2
Original line number Diff line number Diff line
@@ -90,6 +90,11 @@ def _task_result_project_candidates(task_result: DBTaskResult) -> list[tuple[str
        if collection_id is not None:
            candidates.append((TASK_RESULT_COLLECTION_LOOKUP, collection_id))
    elif task_name == "update_data_from_xlsx_rows":
        xlsx_project_id = _object_id(_argument(args, kwargs, "project_id", 1))
        if xlsx_project_id is not None:
            candidates.append((TASK_RESULT_PROJECT_LOOKUP, xlsx_project_id))
        xlsx_user_task_id = _object_id(_argument(args, kwargs, "user_task_id", 3))
        if xlsx_user_task_id is None and xlsx_project_id is None:
            xlsx_user_task_id = _object_id(_argument(args, kwargs, "user_task_id", 2))
        if xlsx_user_task_id is not None:
            candidates.append((TASK_RESULT_USER_TASK_LOOKUP, xlsx_user_task_id))
@@ -177,7 +182,14 @@ def task_result_project(task_result: DBTaskResult) -> Project | None:
            _argument(args, kwargs, "collection_id", 1)
        )
    if task_name == "update_data_from_xlsx_rows":
        return _project_from_user_task(_argument(args, kwargs, "user_task_id", 2))
        project_id = _object_id(_argument(args, kwargs, "project_id", 1))
        user_task_id = _argument(args, kwargs, "user_task_id", 3)
        if user_task_id is None and project_id is None:
            user_task_id = _argument(args, kwargs, "user_task_id", 2)
        return (
            _project_from_user_task(user_task_id)
            or Project.objects.filter(pk=project_id).first()
        )
    return None


+61 −0
Original line number Diff line number Diff line
@@ -156,6 +156,49 @@ def _default_env_file_content(secret: str, var_dir: Path) -> str:
            "10000",
            "Maximum number of chunks accepted for one upload.",
        ),
        (
            "JAMA_XLSX_UPLOADS_DIR",
            str((var_dir / "xlsx_uploads").resolve()),
            "Directory used to hold XLSX imports while background tasks process them.",
        ),
        (
            "JAMA_XLSX_UPLOAD_MAX_SIZE",
            str(25 * 1024**2),
            "Maximum compressed XLSX upload size, in bytes.",
        ),
        (
            "JAMA_XLSX_MAX_UNCOMPRESSED_SIZE",
            str(100 * 1024**2),
            "Maximum total uncompressed size of an XLSX archive, in bytes.",
        ),
        (
            "JAMA_XLSX_MAX_ZIP_ENTRIES",
            "1000",
            "Maximum number of files in an XLSX archive.",
        ),
        (
            "JAMA_XLSX_MAX_COMPRESSION_RATIO",
            "200",
            "Maximum compression ratio accepted for an XLSX archive entry.",
        ),
        ("JAMA_XLSX_MAX_ROWS", "50000", "Maximum worksheet rows per import."),
        ("JAMA_XLSX_MAX_COLUMNS", "256", "Maximum worksheet columns per import."),
        ("JAMA_XLSX_MAX_CELLS", "500000", "Maximum worksheet cells per import."),
        (
            "JAMA_XLSX_MAX_CELL_LENGTH",
            "32767",
            "Maximum string length accepted in one XLSX cell.",
        ),
        (
            "JAMA_XLSX_MAX_AFFECTED_OBJECTS",
            "100000",
            "Maximum resource and collection updates after expanding cascades.",
        ),
        (
            "JAMA_XLSX_MAX_METADATA_WRITES",
            "500000",
            "Maximum estimated metadata values written after expanding cascades.",
        ),
        (
            "JAMA_TMP_THUMBNAILS_DIR",
            tmp_dir,
@@ -342,6 +385,20 @@ JAMA_RPC_MAX_BODY_SIZE = int(os.getenv("JAMA_RPC_MAX_BODY_SIZE", 1024 * 1024))
JAMA_RPC_MAX_BATCH_SIZE = int(os.getenv("JAMA_RPC_MAX_BATCH_SIZE", 100))
JAMA_UPLOAD_MAX_CHUNK_SIZE = int(os.getenv("JAMA_UPLOAD_MAX_CHUNK_SIZE", 128 * 1024**2))
JAMA_UPLOAD_MAX_TOTAL_CHUNKS = int(os.getenv("JAMA_UPLOAD_MAX_TOTAL_CHUNKS", 10000))
JAMA_XLSX_UPLOAD_MAX_SIZE = int(os.getenv("JAMA_XLSX_UPLOAD_MAX_SIZE", 25 * 1024**2))
JAMA_XLSX_MAX_UNCOMPRESSED_SIZE = int(
    os.getenv("JAMA_XLSX_MAX_UNCOMPRESSED_SIZE", 100 * 1024**2)
)
JAMA_XLSX_MAX_ZIP_ENTRIES = int(os.getenv("JAMA_XLSX_MAX_ZIP_ENTRIES", 1000))
JAMA_XLSX_MAX_COMPRESSION_RATIO = int(os.getenv("JAMA_XLSX_MAX_COMPRESSION_RATIO", 200))
JAMA_XLSX_MAX_ROWS = int(os.getenv("JAMA_XLSX_MAX_ROWS", 50000))
JAMA_XLSX_MAX_COLUMNS = int(os.getenv("JAMA_XLSX_MAX_COLUMNS", 256))
JAMA_XLSX_MAX_CELLS = int(os.getenv("JAMA_XLSX_MAX_CELLS", 500000))
JAMA_XLSX_MAX_CELL_LENGTH = int(os.getenv("JAMA_XLSX_MAX_CELL_LENGTH", 32767))
JAMA_XLSX_MAX_AFFECTED_OBJECTS = int(
    os.getenv("JAMA_XLSX_MAX_AFFECTED_OBJECTS", 100000)
)
JAMA_XLSX_MAX_METADATA_WRITES = int(os.getenv("JAMA_XLSX_MAX_METADATA_WRITES", 500000))
JAMA_THUMBNAIL_MAX_SIZE = int(os.getenv("JAMA_THUMBNAIL_MAX_SIZE", 2000))
JAMA_THUMBNAIL_MAX_CROP_PIXELS = int(
    os.getenv("JAMA_THUMBNAIL_MAX_CROP_PIXELS", 100000)
@@ -370,6 +427,10 @@ PARTIAL_UPLOADS_DIR = os.getenv("JAMA_PARTIAL_UPLOADS_DIR") or str(
    Path(VAR_DIR, "partial_uploads").resolve()
)
os.makedirs(PARTIAL_UPLOADS_DIR, exist_ok=True)
XLSX_UPLOADS_DIR = os.getenv("JAMA_XLSX_UPLOADS_DIR") or str(
    Path(VAR_DIR, "xlsx_uploads").resolve()
)
os.makedirs(XLSX_UPLOADS_DIR, exist_ok=True)

MEDIA_FILES_DIR = os.getenv("JAMA_FILES_DIR") or str(
    Path(VAR_DIR, "media_source_files").resolve()
+29 −0
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ from resources.models import (
    Resource,
    Role,
    ObjectPermission,
    UserTask,
)


@@ -83,6 +84,17 @@ class SettingsEnvTemplateTestCase(SimpleTestCase):
            "JAMA_PARTIAL_UPLOADS_DIR",
            "JAMA_UPLOAD_MAX_CHUNK_SIZE",
            "JAMA_UPLOAD_MAX_TOTAL_CHUNKS",
            "JAMA_XLSX_UPLOADS_DIR",
            "JAMA_XLSX_UPLOAD_MAX_SIZE",
            "JAMA_XLSX_MAX_UNCOMPRESSED_SIZE",
            "JAMA_XLSX_MAX_ZIP_ENTRIES",
            "JAMA_XLSX_MAX_COMPRESSION_RATIO",
            "JAMA_XLSX_MAX_ROWS",
            "JAMA_XLSX_MAX_COLUMNS",
            "JAMA_XLSX_MAX_CELLS",
            "JAMA_XLSX_MAX_CELL_LENGTH",
            "JAMA_XLSX_MAX_AFFECTED_OBJECTS",
            "JAMA_XLSX_MAX_METADATA_WRITES",
            "JAMA_TMP_THUMBNAILS_DIR",
            "JAMA_TMP_THUMBNAILS_MAX_AGE_SECONDS",
            "JAMA_THUMBNAIL_MAX_SIZE",
@@ -276,6 +288,23 @@ class AdminTaskServicesTestCase(TestCase):
        self.assertEqual(project_labels, ["batch project"] * 3)
        self.assertLessEqual(len(queries), 3)

    def test_task_results_infer_xlsx_project_from_new_task_signature(self):
        project = Project.objects.create(label="XLSX task project")
        user = User.objects.create(username="xlsx-task-user")
        user_task = UserTask.objects.create(
            owner=user, project=project, description="XLSX import"
        )
        task_result = self.create_task_result(
            status="FAILED",
            task_path="resources.tasks.update_data_from_xlsx_rows",
            args_kwargs={
                "args": [user.pk, project.pk, "/private/import.xlsx", user_task.pk],
                "kwargs": {},
            },
        )

        self.assertEqual(task_services.task_result_project(task_result), project)

    def test_task_results_queryset_defers_detail_fields(self):
        task_result = self.create_task_result(
            status="FAILED",
+172 −42
Original line number Diff line number Diff line
from __future__ import annotations

import logging
from collections import Counter
from typing import List
from django.conf import settings
from django.urls import reverse
@@ -13,6 +16,12 @@ from pathlib import Path
from django.core.mail import mail_admins
from django.tasks import task
from crontask import cron
from resources.xlsx import (
    XLSXLimitExceeded,
    XLSXValidationError,
    iter_xlsx_rows,
    resolve_xlsx_upload_path,
)

logger = logging.getLogger(__name__)

@@ -241,61 +250,182 @@ def recursive_set_metas_to_collection(
    return True


def _xlsx_object_id(value, field_name: str) -> int | None:
    if value in (None, ""):
        return None
    if isinstance(value, bool):
        raise XLSXValidationError(f"invalid {field_name}")
    if isinstance(value, str):
        if not value.strip().isdigit():
            raise XLSXValidationError(f"invalid {field_name}")
        object_id = int(value.strip())
    else:
        try:
            object_id = int(value)
        except (TypeError, ValueError) as error:
            raise XLSXValidationError(f"invalid {field_name}") from error
        if object_id != value:
            raise XLSXValidationError(f"invalid {field_name}")
    if object_id < 1:
        raise XLSXValidationError(f"invalid {field_name}")
    return object_id


def _xlsx_row_references(
    xlsx_path: Path,
) -> list[tuple[int | None, int | None, bool, int]]:
    references = []
    for row in iter_xlsx_rows(xlsx_path):
        resource_id = _xlsx_object_id(row.get("resource_pk"), "resource_pk")
        collection_id = _xlsx_object_id(row.get("collection_pk"), "collection_pk")
        cascade = str(row.get("cascade") or "").strip().lower() == "y"
        metadata_value_count = sum(
            str(value).count(models.XLSX_MULTIPLE_VALUES_SEPARATOR) + 1
            for column_name, value in row.items()
            if ":" in column_name and value is not None
        )
        references.append((resource_id, collection_id, cascade, metadata_value_count))
    return references


def _validate_xlsx_import_scope(project: models.Project, xlsx_path: Path) -> None:
    references = _xlsx_row_references(xlsx_path)
    resource_ids = {
        resource_id for resource_id, _, _, _ in references if resource_id is not None
    }
    collection_ids = {
        collection_id
        for _, collection_id, _, _ in references
        if collection_id is not None
    }
    resources_by_id = (
        models.Resource.objects.filter(pk__in=resource_ids, deleted_at__isnull=True)
        .only("id", "ptr_project_id")
        .in_bulk()
    )
    collections_by_id = (
        models.Collection.objects.filter(pk__in=collection_ids, deleted_at__isnull=True)
        .only("id", "project_id")
        .in_bulk()
    )
    if set(resources_by_id) != resource_ids or any(
        resource.ptr_project_id != project.pk for resource in resources_by_id.values()
    ):
        raise XLSXValidationError("resource does not belong to the import project")
    if set(collections_by_id) != collection_ids or any(
        collection.project_id != project.pk for collection in collections_by_id.values()
    ):
        raise XLSXValidationError("collection does not belong to the import project")

    affected_objects = sum(
        resource_id is not None for resource_id, _, _, _ in references
    ) + sum(collection_id is not None for _, collection_id, _, _ in references)
    metadata_writes = sum(
        metadata_value_count * ((resource_id is not None) + (collection_id is not None))
        for resource_id, collection_id, _, metadata_value_count in references
    )
    cascade_counts = Counter(
        (collection_id, metadata_value_count)
        for _, collection_id, cascade, metadata_value_count in references
        if cascade and collection_id is not None
    )
    maximum_affected_objects = settings.JAMA_XLSX_MAX_AFFECTED_OBJECTS
    maximum_metadata_writes = settings.JAMA_XLSX_MAX_METADATA_WRITES
    cascade_sizes = {}
    for (
        collection_id,
        metadata_value_count,
    ), occurrence_count in cascade_counts.items():
        collection = collections_by_id[collection_id]
        if collection_id not in cascade_sizes:
            cascade_sizes[collection_id] = (
                collection.descendants_count() + collection.descendant_resources_count()
            )
        cascade_size = cascade_sizes[collection_id]
        affected_objects += cascade_size * occurrence_count
        metadata_writes += cascade_size * metadata_value_count * occurrence_count
        if maximum_affected_objects > 0 and affected_objects > maximum_affected_objects:
            raise XLSXLimitExceeded(
                "XLSX cascade exceeds the configured affected-object limit"
            )
        if maximum_metadata_writes > 0 and metadata_writes > maximum_metadata_writes:
            raise XLSXLimitExceeded(
                "XLSX cascade exceeds the configured metadata-write limit"
            )

    if maximum_affected_objects > 0 and affected_objects > maximum_affected_objects:
        raise XLSXLimitExceeded(
            "XLSX import exceeds the configured affected-object limit"
        )
    if maximum_metadata_writes > 0 and metadata_writes > maximum_metadata_writes:
        raise XLSXLimitExceeded(
            "XLSX import exceeds the configured metadata-write limit"
        )


@task
def update_data_from_xlsx_rows(
    user_id: int, xlsx_rows: List[dict], user_task_id: int = None
):
    user_id: int,
    project_id: int,
    xlsx_path: str,
    user_task_id: int = None,
) -> bool:
    from django.contrib.auth.models import User
    from resources.models import UserTask
    from resources.acl import UserAccess
    from rpc.methods import (
        ServiceException,
        update_collection_from_xlsx_row,
        update_resource_from_xlsx_row,
    )

    project = None

    # Try to find the project
    for r in xlsx_rows:
        if r.get("resource_pk"):
            first_resource = models.Resource.objects.filter(
                pk=r.get("resource_pk")
    user_task = UserTask.objects.filter(
        pk=user_task_id,
        owner_id=user_id,
        project_id=project_id,
    ).first()
            if first_resource:
                project = first_resource.ptr_project
                break
        if r.get("collection_pk"):
            first_collection = models.Collection.objects.filter(
                pk=r.get("collection_pk")
            ).first()
            if first_collection:
                project = first_collection.project
                break
    user_task = None
    user = User.objects.filter(pk=user_id).first()
    if user:
        if user_task_id is not None:
            user_task = UserTask.objects.filter(pk=user_task_id).first()
    resolved_xlsx_path = None
    try:
        resolved_xlsx_path = resolve_xlsx_upload_path(xlsx_path)
        user = User.objects.filter(pk=user_id, is_active=True).first()
        project = models.Project.objects.filter(pk=project_id).first()
        if not user or not project:
            raise XLSXValidationError("invalid XLSX import owner or project")
        if user_task_id is not None and not user_task:
            raise XLSXValidationError("invalid XLSX import task")
        UserAccess(user, project).check_project_access()
        if user_task:
                user_task.project = project
            user_task.started_at = timezone.now()
                user_task.save()
        from rpc.methods import (
            update_resource_from_xlsx_row,
            update_collection_from_xlsx_row,
            ServiceException,
        )
            user_task.save(update_fields=["started_at"])

        for xlsx_row in xlsx_rows:
        _validate_xlsx_import_scope(project, resolved_xlsx_path)
        for xlsx_row in iter_xlsx_rows(resolved_xlsx_path):
            try:
                if xlsx_row.get("resource_pk"):
                    update_resource_from_xlsx_row(user, xlsx_row)
                if xlsx_row.get("collection_pk"):
                    update_collection_from_xlsx_row(user, xlsx_row)
            except ServiceException as e:
                logger.warning(e)
            except ServiceException as error:
                logger.warning(error)
    except (ServiceException, XLSXValidationError) as error:
        logger.warning("XLSX import rejected: %s", error)
        if user_task:
            user_task.finished_at = timezone.now()
            user_task.save()
            user_task.failed_at = timezone.now()
            user_task.save(update_fields=["failed_at"])
        return False
    except Exception:
        if user_task:
            user_task.failed_at = timezone.now()
            user_task.save(update_fields=["failed_at"])
        raise
    else:
        logger.warning(
            f"No such user({user_id}) for update_data_from_xlsx_rows task, canceling."
        )
        if user_task:
            user_task.finished_at = timezone.now()
            user_task.save(update_fields=["finished_at"])
        return True
    finally:
        if resolved_xlsx_path:
            resolved_xlsx_path.unlink(missing_ok=True)


@cron("0 * * * *")
Loading