TUI query optimizations (fef5a03c) · Commits · certic / Jama

src/jama/admin_services/tasks.py

+141 −6

Original line number	Diff line number	Diff line
		@@ -11,6 +11,11 @@ from resources.models import Collection, File, Project, Resource, UserTask

		TASK_RESULT_STATUSES = ["SUCCESSFUL", "FAILED", "RUNNING", "READY"]
		TASK_RESULT_LIST_FIELDS = ["task_path", "enqueued_at", "status"]
		TASK_RESULT_PROJECT_LOOKUP = "project"
		TASK_RESULT_USER_TASK_LOOKUP = "user_task"
		TASK_RESULT_FILE_LOOKUP = "file"
		TASK_RESULT_RESOURCE_LOOKUP = "resource"
		TASK_RESULT_COLLECTION_LOOKUP = "collection"


		@dataclass
		@@ -38,6 +43,60 @@ def _argument(args: list, kwargs: dict, name: str, index: int):
		return None


		def _object_id(value) -> int \| None:
		if value in (None, ""):
		return None
		try:
		return int(value)
		except (TypeError, ValueError):
		return None


		def _task_result_project_candidates(task_result: DBTaskResult) -> list[tuple[str, int]]:
		args, kwargs = _task_args_kwargs(task_result)
		candidates = []

		explicit_project = _object_id(kwargs.get("project_id"))
		if explicit_project is not None:
		candidates.append((TASK_RESULT_PROJECT_LOOKUP, explicit_project))

		user_task_id = _object_id(kwargs.get("user_task_id"))
		if user_task_id is not None:
		candidates.append((TASK_RESULT_USER_TASK_LOOKUP, user_task_id))

		task_name = task_result.task_path.rsplit(".", 1)[-1]
		if task_name in {
		"iiif_task",
		"exif_task",
		"ocr_task",
		"hls_task",
		}:
		file_id = _object_id(_argument(args, kwargs, "file_id", 0))
		if file_id is not None:
		candidates.append((TASK_RESULT_FILE_LOOKUP, file_id))
		elif task_name == "set_ark_to_resource":
		resource_id = _object_id(_argument(args, kwargs, "resource_id", 0))
		if resource_id is not None:
		candidates.append((TASK_RESULT_RESOURCE_LOOKUP, resource_id))
		elif task_name == "set_ark_to_collection":
		collection_id = _object_id(_argument(args, kwargs, "collection_id", 0))
		if collection_id is not None:
		candidates.append((TASK_RESULT_COLLECTION_LOOKUP, collection_id))
		elif task_name == "recursive_set_metas_to_collection":
		recursive_user_task_id = _object_id(_argument(args, kwargs, "user_task_id", 3))
		if recursive_user_task_id is not None:
		candidates.append((TASK_RESULT_USER_TASK_LOOKUP, recursive_user_task_id))
		collection_id = _object_id(_argument(args, kwargs, "collection_id", 1))
		if collection_id is not None:
		candidates.append((TASK_RESULT_COLLECTION_LOOKUP, collection_id))
		elif task_name == "update_data_from_xlsx_rows":
		xlsx_user_task_id = _object_id(_argument(args, kwargs, "user_task_id", 2))
		if xlsx_user_task_id is not None:
		candidates.append((TASK_RESULT_USER_TASK_LOOKUP, xlsx_user_task_id))

		return candidates


		def _project_from_user_task(user_task_id) -> Project \| None:
		if not user_task_id:
		return None
		@@ -122,6 +181,79 @@ def task_result_project(task_result: DBTaskResult) -> Project \| None:
		return None


		def task_result_projects_by_id(
		task_results: list[DBTaskResult],
		) -> dict[object, Project \| None]:
		project_ids: set[int] = set()
		user_task_ids: set[int] = set()
		file_ids: set[int] = set()
		resource_ids: set[int] = set()
		collection_ids: set[int] = set()
		candidates_by_task_id: dict[object, list[tuple[str, int]]] = {}

		for task_result in task_results:
		candidates = _task_result_project_candidates(task_result)
		candidates_by_task_id[task_result.pk] = candidates
		for lookup_type, object_id in candidates:
		if lookup_type == TASK_RESULT_PROJECT_LOOKUP:
		project_ids.add(object_id)
		elif lookup_type == TASK_RESULT_USER_TASK_LOOKUP:
		user_task_ids.add(object_id)
		elif lookup_type == TASK_RESULT_FILE_LOOKUP:
		file_ids.add(object_id)
		elif lookup_type == TASK_RESULT_RESOURCE_LOOKUP:
		resource_ids.add(object_id)
		elif lookup_type == TASK_RESULT_COLLECTION_LOOKUP:
		collection_ids.add(object_id)

		projects = Project.objects.only("id", "label").in_bulk(project_ids)
		user_task_projects = {
		user_task.pk: user_task.project
		for user_task in UserTask.objects.filter(pk__in=user_task_ids)
		.select_related("project")
		.only("id", "project_id", "project__id", "project__label")
		if user_task.project_id
		}
		file_projects = {
		file_instance.pk: file_instance.project
		for file_instance in File.objects.filter(pk__in=file_ids)
		.select_related("project")
		.only("id", "project_id", "project__id", "project__label")
		}
		resource_projects = {
		resource.pk: resource.ptr_project
		for resource in Resource.objects.filter(pk__in=resource_ids)
		.select_related("ptr_project")
		.only("id", "ptr_project_id", "ptr_project__id", "ptr_project__label")
		if resource.ptr_project_id
		}
		collection_projects = {
		collection.pk: collection.project
		for collection in Collection.objects.filter(pk__in=collection_ids)
		.select_related("project")
		.only("id", "project_id", "project__id", "project__label")
		if collection.project_id
		}
		project_maps = {
		TASK_RESULT_PROJECT_LOOKUP: projects,
		TASK_RESULT_USER_TASK_LOOKUP: user_task_projects,
		TASK_RESULT_FILE_LOOKUP: file_projects,
		TASK_RESULT_RESOURCE_LOOKUP: resource_projects,
		TASK_RESULT_COLLECTION_LOOKUP: collection_projects,
		}

		projects_by_task_id = {}
		for task_result in task_results:
		for lookup_type, object_id in candidates_by_task_id[task_result.pk]:
		project = project_maps[lookup_type].get(object_id)
		if project:
		projects_by_task_id[task_result.pk] = project
		break
		else:
		projects_by_task_id[task_result.pk] = None
		return projects_by_task_id


		def task_path_filter_values() -> list[str]:
		return list(
		DBTaskResult.objects.order_by("task_path")
		@@ -149,7 +281,7 @@ def task_results_queryset(
		field_names = [field.name for field in fields]
		queryset = DBTaskResult.objects.all()

		if filters.enqueued_at_date in set(enqueued_at_date_filter_values()):
		if filters.enqueued_at_date:
		selected_date = parse_date(filters.enqueued_at_date)
		if selected_date:
		queryset = queryset.filter(enqueued_at__date=selected_date)
		@@ -158,7 +290,6 @@ def task_results_queryset(
		if filters.status in TASK_RESULT_STATUSES:
		queryset = queryset.filter(status=filters.status)
		if filters.task_path:
		if filters.task_path in set(task_path_filter_values()):
		queryset = queryset.filter(task_path=filters.task_path)
		if filters.search:
		search_query = None
		@@ -179,7 +310,9 @@ def task_results_queryset(
		sort_field = sort[1:] if sort.startswith("-") else sort
		if sort_field not in field_names:
		sort = "-enqueued_at" if "enqueued_at" in field_names else field_names[0]
		return queryset.order_by(sort)
		return queryset.order_by(sort).only(
		"id", "status", "enqueued_at", "task_path", "args_kwargs"
		)


		def task_results(filters: TaskResultFilters \| None = None, limit: int = 200):
		@@ -187,8 +320,10 @@ def task_results(filters: TaskResultFilters \| None = None, limit: int = 200):
		project_id = str(filters.project_id or "")
		results = []
		# Project filtering is inferred per row, so read ahead without making the TUI unbounded.
		for task_result in task_results_queryset(filters)[: max(limit * 5, limit)]:
		project = task_result_project(task_result)
		task_results_slice = list(task_results_queryset(filters)[: max(limit * 5, limit)])
		projects_by_task_id = task_result_projects_by_id(task_results_slice)
		for task_result in task_results_slice:
		project = projects_by_task_id.get(task_result.pk)
		if project_id and (not project or str(project.pk) != project_id):
		continue
		task_result.inferred_project = project

src/jama/adminui/app.py

+67 −19

Original line number	Diff line number	Diff line
		@@ -820,7 +820,20 @@ class JamaAdminApp(App):
		selected = self.selected_object()
		return selected if isinstance(selected, Project) else None
		if self.context.get("project_id"):
		return Project.objects.filter(pk=self.context["project_id"]).first()
		return (
		Project.objects.only(
		"id",
		"label",
		"description",
		"admin_mail",
		"ark_redirect",
		"use_exiftool",
		"resources_pipelines",
		"collections_pipelines",
		)
		.filter(pk=self.context["project_id"])
		.first()
		)
		return None

		def refresh_table(self) -> None:
		@@ -841,7 +854,7 @@ class JamaAdminApp(App):
		def project_choices(self) -> list[tuple[str, str]]:
		return [
		(f"{project.pk}: {project.label}", str(project.pk))
		for project in Project.objects.order_by("label", "id")
		for project in Project.objects.only("id", "label").order_by("label", "id")
		]

		def role_choices(self, project: Project \| None = None) -> list[tuple[str, str]]:
		@@ -850,6 +863,7 @@ class JamaAdminApp(App):
		)
		if project:
		roles = roles.filter(project=project)
		roles = roles.only("id", "label", "project_id", "project__id", "project__label")
		return [
		(f"{role.pk}: {role.project.label} / {role.label}", str(role.pk))
		for role in roles
		@@ -858,7 +872,7 @@ class JamaAdminApp(App):
		def user_choices(self) -> list[tuple[str, str]]:
		return [
		(f"{user.pk}: {user.username}", str(user.pk))
		for user in User.objects.order_by("username", "id")
		for user in User.objects.only("id", "username").order_by("username", "id")
		]

		def form(
		@@ -1004,7 +1018,9 @@ class JamaAdminApp(App):
		self.open_mode("project_stats", {"project_id": project.pk})

		def load_project_stats(self) -> None:
		project = get_object_or_404(Project, pk=self.context["project_id"])
		project = get_object_or_404(
		Project.objects.only("id", "label"), pk=self.context["project_id"]
		)
		self.set_title(f"{self.translate('Project stats')}: {project.label}")
		self.reset_table(["metric", "value"])
		stats = project_services.project_statistics(project)
		@@ -1013,19 +1029,23 @@ class JamaAdminApp(App):
		self.set_status("Esc back. Ctrl+R roles, Ctrl+M metadata, Ctrl+U members.")

		def load_project_roles(self) -> None:
		project = get_object_or_404(Project, pk=self.context["project_id"])
		project = get_object_or_404(
		Project.objects.only("id", "label"), pk=self.context["project_id"]
		)
		self.set_title(f"{self.translate('Roles')}: {project.label}")
		self.reset_table(["id", "label", "permissions"])
		for role in access_services.list_roles(project):
		self.add_row(
		f"role:{role.pk}",
		role,
		[role.pk, role.label, role.objectpermission_set.count()],
		[role.pk, role.label, role.permissions_count],
		)
		self.set_status("Enter edits permission matrix. A add, E rename, D delete.")

		def add_role(self) -> None:
		project = get_object_or_404(Project, pk=self.context["project_id"])
		project = get_object_or_404(
		Project.objects.only("id", "label"), pk=self.context["project_id"]
		)
		self.form(
		"Add role",
		[Field("label", "Label")],
		@@ -1063,7 +1083,12 @@ class JamaAdminApp(App):
		)

		def load_role_matrix(self) -> None:
		role = get_object_or_404(Role, pk=self.context["role_id"])
		role = get_object_or_404(
		Role.objects.select_related("project").only(
		"id", "label", "project_id", "project__id", "project__label"
		),
		pk=self.context["role_id"],
		)
		self.set_title(
		f"{self.translate('Permissions')}: {role.project.label} / {role.label}"
		)
		@@ -1105,7 +1130,12 @@ class JamaAdminApp(App):
		if not isinstance(object_class, str) or crud is None:
		self.set_status("Select a create/read/update/delete cell to toggle.")
		return
		role = get_object_or_404(Role, pk=self.context["role_id"])
		role = get_object_or_404(
		Role.objects.select_related("project").only(
		"id", "label", "project_id", "project__id", "project__label"
		),
		pk=self.context["role_id"],
		)
		matrix = access_services.role_permission_matrix(role)
		matrix.setdefault(object_class, {verb: False for verb in acl.CRUD_VERBS})
		matrix[object_class][crud] = not matrix[object_class][crud]
		@@ -1123,7 +1153,12 @@ class JamaAdminApp(App):
		object_class = self.selected_object()
		if not isinstance(object_class, str):
		return
		role = get_object_or_404(Role, pk=self.context["role_id"])
		role = get_object_or_404(
		Role.objects.select_related("project").only(
		"id", "label", "project_id", "project__id", "project__label"
		),
		pk=self.context["role_id"],
		)
		matrix = access_services.role_permission_matrix(role)
		permissions = matrix[object_class]
		self.form(
		@@ -1150,7 +1185,9 @@ class JamaAdminApp(App):
		)

		def load_project_members(self) -> None:
		project = get_object_or_404(Project, pk=self.context["project_id"])
		project = get_object_or_404(
		Project.objects.only("id", "label"), pk=self.context["project_id"]
		)
		self.set_title(f"{self.translate('Project members')}: {project.label}")
		self.reset_table(["access id", "user id", "username", "role id", "role"])
		for access in access_services.list_project_accesses(project):
		@@ -1168,7 +1205,9 @@ class JamaAdminApp(App):
		self.set_status("A add user by ids. D removes selected role from user.")

		def add_project_member(self) -> None:
		project = get_object_or_404(Project, pk=self.context["project_id"])
		project = get_object_or_404(
		Project.objects.only("id", "label"), pk=self.context["project_id"]
		)
		user_choices = self.user_choices()
		role_choices = self.role_choices(project)
		if not user_choices:
		@@ -1201,7 +1240,9 @@ class JamaAdminApp(App):
		)

		def load_metadata_sets(self) -> None:
		project = get_object_or_404(Project, pk=self.context["project_id"])
		project = get_object_or_404(
		Project.objects.only("id", "label"), pk=self.context["project_id"]
		)
		self.set_title(f"{self.translate('Metadata sets')}: {project.label}")
		self.reset_table(["id", "title", "metadata"])
		for metadata_set in metadata_services.list_metadata_sets(project):
		@@ -1213,7 +1254,9 @@ class JamaAdminApp(App):
		self.set_status("Enter opens metadata. A add set, E rename, D delete.")

		def add_metadata_set(self) -> None:
		project = get_object_or_404(Project, pk=self.context["project_id"])
		project = get_object_or_404(
		Project.objects.only("id", "label"), pk=self.context["project_id"]
		)
		self.form(
		"Add metadata set",
		[Field("title", "Title")],
		@@ -1257,7 +1300,10 @@ class JamaAdminApp(App):

		def load_metadata(self) -> None:
		metadata_set = get_object_or_404(
		MetadataSet, pk=self.context["metadata_set_id"]
		MetadataSet.objects.select_related("project").only(
		"id", "title", "project_id", "project__id", "project__label"
		),
		pk=self.context["metadata_set_id"],
		)
		self.set_title(
		f"{self.translate('Metadata')}: "
		@@ -1267,7 +1313,6 @@ class JamaAdminApp(App):
		["id", "rank", "title", "expose", "resource values", "collection values"]
		)
		for metadata in metadata_services.list_metadata(metadata_set):
		usage = metadata_services.metadata_usage(metadata)
		self.add_row(
		f"metadata:{metadata.pk}",
		metadata,
		@@ -1276,15 +1321,18 @@ class JamaAdminApp(App):
		metadata.rank,
		metadata.title,
		metadata.expose,
		usage["resources"],
		usage["collections"],
		metadata.resource_values_count,
		metadata.collection_values_count,
		],
		)
		self.set_status("A add metadata. E edit. D delete.")

		def add_metadata(self) -> None:
		metadata_set = get_object_or_404(
		MetadataSet, pk=self.context["metadata_set_id"]
		MetadataSet.objects.select_related("project").only(
		"id", "title", "project_id", "project__id", "project__label"
		),
		pk=self.context["metadata_set_id"],
		)
		self.form(
		"Add metadata",

src/jama/management/commands/adminui.py

+21 −8

Original line number	Diff line number	Diff line
		import importlib.util
		import os

		from django.core.management.base import BaseCommand, CommandError

		from jama.adminui.i18n import DEFAULT_LANGUAGE, SUPPORTED_LANGUAGES


		class Command(BaseCommand):
		help = "Start the Jama terminal administration interface."
		@@ -9,21 +12,31 @@ class Command(BaseCommand):
		def add_arguments(self, parser):
		parser.add_argument(
		"--language",
		choices=["fr", "en"],
		default="fr",
		choices=SUPPORTED_LANGUAGES,
		default=DEFAULT_LANGUAGE,
		help="Language used by the admin TUI. Defaults to French.",
		)

		def handle(self, args, *options):
		if importlib.util.find_spec("textual") is None:
		raise CommandError(
		"The admin TUI requires Textual. Install project dependencies with uv sync."
		)

		# Textual runs inside an asyncio event loop. This command is an
		# interactive admin process, so direct synchronous Django ORM calls are
		# intentional here.
		os.environ.setdefault("DJANGO_ALLOW_ASYNC_UNSAFE", "true")
		async_unsafe_env_name = "DJANGO_ALLOW_ASYNC_UNSAFE"
		previous_async_unsafe_value = os.environ.get(async_unsafe_env_name)
		had_async_unsafe_value = async_unsafe_env_name in os.environ
		os.environ[async_unsafe_env_name] = "true"
		try:
		# Import lazily so non-TUI commands do not require Textual at import time.
		from jama.adminui.app import JamaAdminApp
		except ImportError as error:
		raise CommandError(
		"The admin TUI requires Textual. Install project dependencies with uv sync."
		) from error
		JamaAdminApp(language=options.get("language", "fr")).run()

		JamaAdminApp(language=options.get("language", DEFAULT_LANGUAGE)).run()
		finally:
		if had_async_unsafe_value:
		os.environ[async_unsafe_env_name] = previous_async_unsafe_value
		else:
		os.environ.pop(async_unsafe_env_name, None)

src/jama/tests.py

+98 −2

Original line number	Diff line number	Diff line
		@@ -213,6 +213,49 @@ class AdminTaskServicesTestCase(TestCase):
		task_services.task_result_project(matching_task), matching_project
		)

		def test_task_results_batch_project_inference_for_file_tasks(self):
		project = Project.objects.create(label="batch project", description="")
		file_type = FileType.objects.create(title="Text", mime="text/plain")
		files = [
		File.objects.create(
		title=f"file {index}",
		original_name=f"file-{index}.txt",
		project=project,
		hash=str(index) * 64,
		file_type=file_type,
		size=1,
		)
		for index in range(1, 4)
		]
		for file_instance in files:
		self.create_task_result(
		status="FAILED",
		task_path="resources.tasks.iiif_task",
		args_kwargs={"args": [file_instance.pk], "kwargs": {}},
		)

		with CaptureQueriesContext(connection) as queries:
		results = task_services.task_results()
		project_labels = [
		result.inferred_project.label
		for result in results
		if result.inferred_project
		]

		self.assertEqual(project_labels, ["batch project"] * 3)
		self.assertLessEqual(len(queries), 3)

		def test_task_results_queryset_defers_detail_fields(self):
		task_result = self.create_task_result(
		status="FAILED",
		task_path="resources.tasks.iiif_task",
		)

		result = task_services.task_results_queryset().get(pk=task_result.pk)

		self.assertIn("traceback", result.get_deferred_fields())
		self.assertIn("return_value", result.get_deferred_fields())


		class AdminCacheServicesTestCase(SimpleTestCase):
		def test_clear_cache_aliases_rejects_unknown_alias(self):
		@@ -230,17 +273,70 @@ class AdminUiCommandTestCase(SimpleTestCase):
		def test_adminui_allows_sync_orm_inside_textual_event_loop(self):
		from jama.management.commands.adminui import Command

		def assert_async_unsafe_enabled():
		self.assertEqual(os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"], "true")

		with (
		patch.dict(os.environ, {}, clear=False),
		patch("jama.adminui.app.JamaAdminApp.run") as run,
		patch(
		"jama.adminui.app.JamaAdminApp.run",
		side_effect=assert_async_unsafe_enabled,
		) as run,
		):
		os.environ.pop("DJANGO_ALLOW_ASYNC_UNSAFE", None)

		Command().handle()

		self.assertEqual(os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"], "true")
		self.assertNotIn("DJANGO_ALLOW_ASYNC_UNSAFE", os.environ)
		run.assert_called_once()

		def test_adminui_restores_existing_async_unsafe_value(self):
		from jama.management.commands.adminui import Command

		def assert_async_unsafe_enabled():
		self.assertEqual(os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"], "true")

		with (
		patch.dict(os.environ, {"DJANGO_ALLOW_ASYNC_UNSAFE": "existing"}),
		patch(
		"jama.adminui.app.JamaAdminApp.run",
		side_effect=assert_async_unsafe_enabled,
		),
		):
		Command().handle()

		self.assertEqual(os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"], "existing")

		def test_adminui_missing_textual_reports_command_error(self):
		from django.core.management.base import CommandError

		from jama.management.commands.adminui import Command

		with (
		patch("importlib.util.find_spec", return_value=None),
		self.assertRaisesMessage(CommandError, "The admin TUI requires Textual."),
		):
		Command().handle()

		def test_adminui_does_not_mask_unrelated_import_error(self):
		import builtins

		from jama.management.commands.adminui import Command

		original_import = builtins.__import__

		def raise_for_admin_app(name, globals=None, locals=None, fromlist=(), level=0):
		if name == "jama.adminui.app":
		raise ImportError("broken app import")
		return original_import(name, globals, locals, fromlist, level)

		with (
		patch("importlib.util.find_spec", return_value=object()),
		patch("builtins.__import__", side_effect=raise_for_admin_app),
		self.assertRaisesRegex(ImportError, "broken app import"),
		):
		Command().handle()

		def test_adminui_command_passes_language_option(self):
		from jama.management.commands.adminui import Command

src/resources/admin_services/access.py

+6 −1

Original line number	Diff line number	Diff line
		from django.contrib.auth.models import User
		from django.db import IntegrityError, transaction
		from django.db.models import Count

		from resources import acl
		from resources.models import ObjectPermission, Project, ProjectAccess, Role


		def list_roles(project: Project):
		return Role.objects.filter(project=project).order_by("label", "id")
		return (
		Role.objects.filter(project=project)
		.annotate(permissions_count=Count("objectpermission"))
		.order_by("label", "id")
		)


		def list_project_accesses(project: Project):