better file integrity check + collections xls first pass (eb64254e) · Commits · certic / Jama

src/resources/migrations/0045_file_integrity_check_failed_at.py

0 → 100644

+17 −0

Original line number	Diff line number	Diff line
		# Generated by Django 6.0 on 2026-01-21 12:51

		from django.db import migrations, models


		class Migration(migrations.Migration):
		dependencies = [
		("resources", "0044_alter_metadatacollectionvalue_options"),
		]

		operations = [
		migrations.AddField(
		model_name="file",
		name="integrity_check_failed_at",
		field=models.DateTimeField(blank=True, null=True),
		),
		]

src/resources/models.py

+37 −8

Original line number	Diff line number	Diff line
		@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
		def _flatten_resource(
		resource: "Resource", known_metadatas: dict, metadatas_labels: List
		) -> dict:
		data = {"pk": resource.pk, "title": resource.title}
		data = {"resource_pk": resource.pk, "collection_pk": None, "title": resource.title}
		metadata_ids = known_metadatas.keys()
		for known_metadata_label in metadatas_labels:
		data[known_metadata_label] = None
		@@ -144,6 +144,11 @@ class Project(models.Model):
		return self._cached_root_collection

		def metadatas(self, exclude_automatic_metas=True) -> List["Metadata"]:
		"""
		Fetches all metadatas available in the project, excluding OCR and ExifTool.

		This is the way to define available columns in a XLSX export.
		"""
		metadatas = []
		query = MetadataSet.objects.filter(project=self).order_by("title")
		if exclude_automatic_metas:
		@@ -316,6 +321,7 @@ class File(Resource):
		)
		text_boxes = models.JSONField("text boxes de tesseract", blank=True, null=True)
		tiled = models.BooleanField(default=False)
		integrity_check_failed_at = models.DateTimeField(null=True, blank=True)

		def __str__(self):
		return "{}".format(self.title)
		@@ -705,6 +711,30 @@ class Collection(models.Model):
		self.deleted_at = now()
		self.save()

		def metadatas_for_xlsx(
		self, known_metadatas: dict, metadatas_labels: List[str]
		) -> dict:
		data = {"resource_pk": None, "collection_pk": self.pk, "title": self.title}
		metadata_ids = known_metadatas.keys()
		for known_metadata_label in metadatas_labels:
		data[known_metadata_label] = None
		for mr in (
		self.metadatacollectionvalue_set.exclude(metadata__title="ExifTool")
		.exclude(metadata__title="OCR")
		.exclude(metadata__title="scd_cms")
		.select_related("metadata")
		.iterator()
		):
		if mr.metadata.pk in metadata_ids:
		meta_key = f"{str(mr.metadata)}"
		if data[meta_key]:
		data[meta_key] = (
		data[meta_key] + XLSX_MULTIPLE_VALUES_SEPARATOR + mr.value
		)
		else:
		data[meta_key] = mr.value
		return data

		def dublin_core_metas(self) -> List["MetadataCollectionValue"]:
		metas = []
		try:
		@@ -739,13 +769,12 @@ class Collection(models.Model):
		for metadata in self.project.metadatas():
		known_metadatas[metadata.pk] = metadata
		known_metadatas_labels.append(f"{str(metadata)}")

		# for res in (
		# self.resources.filter(deleted_at__isnull=True)
		# .order_by("title")
		# .prefetch_related("metadataresourcevalue_set")
		# ):
		# yield _flatten_resource(res, known_metadatas, known_metadatas_labels)
		# First line is for the current category
		yield self.metadatas_for_xlsx(known_metadatas, known_metadatas_labels)
		for sub_collection in self.descendants():
		yield sub_collection.metadatas_for_xlsx(
		known_metadatas, known_metadatas_labels
		)
		for res in self.descendants_resources():
		yield _flatten_resource(res, known_metadatas, known_metadatas_labels)

src/resources/tasks.py

+14 −4

Original line number	Diff line number	Diff line
		@@ -217,7 +217,7 @@ def recursive_set_metas_to_collection(


		@db_task()
		def update_resources_from_xlsx_rows(
		def update_data_from_xlsx_rows(
		user_id: int, xlsx_rows: List[dict], user_task_id: int = None
		):
		from django.contrib.auth.models import User
		@@ -239,11 +239,18 @@ def update_resources_from_xlsx_rows(
		user_task.project = project
		user_task.started_at = timezone.now()
		user_task.save()
		from rpc.methods import update_resource_from_xlsx_row, ServiceException
		from rpc.methods import (
		update_resource_from_xlsx_row,
		update_collection_from_xlsx_row,
		ServiceException,
		)

		for xlsx_row in xlsx_rows:
		try:
		if xlsx_row.get("resource_pk"):
		update_resource_from_xlsx_row(user, xlsx_row)
		if xlsx_row.get("collection_pk"):
		update_collection_from_xlsx_row(user, xlsx_row)
		except ServiceException as e:
		logger.warning(e)
		if user_task:
		@@ -332,10 +339,13 @@ def check_files_integrity():

		cursor = connection.cursor()
		cursor.execute(
		"select hash, resource_ptr_id from resources_file where resources_file.tiled = false order by resource_ptr_id"
		"select distinct(hash) from resources_file where resources_file.integrity_check_failed_at is null"
		)
		for row in cursor.fetchall():
		new_hash = file_hash256(models.hash_to_local_path(row[0]))
		if new_hash != row[0]:
		logger.warning(f"FILE INTEGRITY CHECK FAILED. File id: {row[1]}")
		models.File.objects.filter(hash=row[0]).update(
		integrity_check_failed_at=timezone.now()
		)
		connection.close()

src/rpc/methods.py

+10 −2

Original line number	Diff line number	Diff line
		@@ -3199,13 +3199,21 @@ def auto_find_rotate_angle(user: User, resource_id: int) -> float:
		return _deskew(resource_instance.file.local_path())


		@_rpc_groups(["Collections"])
		def update_collection_from_xlsx_row(user: User, collection_data: dict) -> bool:
		"""
		Compound method to update collection and collection metadatas from a xlsx file row.
		"""
		return True


		@_rpc_groups(["Resources"])
		def update_resource_from_xlsx_row(user: User, resource_data: dict) -> bool:
		"""
		Compound method to update resource and resources metadata from a xlsx file row.
		Compound method to update resource and resource metadatas from a xlsx file row.
		"""
		resource_instance = Resource.objects.filter(
		pk=resource_data.get("pk"), deleted_at__isnull=True
		pk=resource_data.get("resource_pk"), deleted_at__isnull=True
		).first()
		if not resource_instance:
		raise ServiceException(NO_SUCH_RESOURCE)

src/rpc/views.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -35,7 +35,7 @@ from resources.tasks import (
		exif_task,
		iiif_task,
		ocr_task,
		update_resources_from_xlsx_rows,
		update_data_from_xlsx_rows,
		)
		from openpyxl import Workbook, load_workbook
		from openpyxl.utils.exceptions import IllegalCharacterError
		@@ -446,7 +446,7 @@ def upload_metas_xls(request: HttpRequest) -> HttpResponse:
		user_task = models.UserTask.objects.create(
		owner=user, description="Traitement de fichier XLS de métadonnées"
		)
		update_resources_from_xlsx_rows(user.pk, data_rows, user_task.pk)
		update_data_from_xlsx_rows(user.pk, data_rows, user_task.pk)
		return HttpResponse("OK", status=200)
		return HttpResponse("Bad Request", status=400)