small query optimizations (277d7422) · Commits · certic / Jama

src/rpc/methods.py

+262 −51

Original line number	Diff line number	Diff line
		@@ -88,6 +88,68 @@ from django.db.models import BinaryField, Count, Q
		logger = logging.getLogger(__name__)


		RESOURCE_SERIALIZATION_FIELDS = (
		"id",
		"title",
		"created_at",
		"updated_at",
		"deleted_at",
		"ptr_project_id",
		"ark",
		)
		FILE_SERIALIZATION_FIELDS = (
		"file__id",
		"file__title",
		"file__created_at",
		"file__updated_at",
		"file__deleted_at",
		"file__ptr_project_id",
		"file__ark",
		"file__original_name",
		"file__project_id",
		"file__project__id",
		"file__project__use_exiftool",
		"file__hash",
		"file__file_type_id",
		"file__file_type__id",
		"file__file_type__title",
		"file__file_type__mime",
		"file__file_type__serve_with_iiif",
		"file__file_type__serve_with_hls",
		"file__denormalized_image_width",
		"file__denormalized_image_height",
		)
		COLLECTION_SERIALIZATION_FIELDS = (
		"id",
		"title",
		"parent_id",
		"project_id",
		"public_access",
		"published_at",
		"created_at",
		"updated_at",
		"deleted_at",
		"representative_id",
		"ark",
		)
		COLLECTION_PARENT_SERIALIZATION_FIELDS = (
		"parent__id",
		"parent__title",
		"parent__parent_id",
		"parent__project_id",
		"parent__public_access",
		"parent__published_at",
		"parent__created_at",
		"parent__updated_at",
		"parent__deleted_at",
		"parent__representative_id",
		"parent__ark",
		)
		COLLECTION_REPRESENTATIVE_SERIALIZATION_FIELDS = tuple(
		f"representative__{field}" for field in RESOURCE_SERIALIZATION_FIELDS
		) + tuple(f"representative__{field}" for field in FILE_SERIALIZATION_FIELDS)


		def _deskew(
		image_path: str, max_skew: int = 10, fast: bool = True, scale_max: int = 1000
		) -> float:
		@@ -191,6 +253,11 @@ def _collections_with_serialization_data(query_set: QuerySet) -> QuerySet:
		# lookups once per collection.
		return (
		query_set.select_related("parent", "representative", "representative__file")
		.only(
		*COLLECTION_SERIALIZATION_FIELDS,
		*COLLECTION_PARENT_SERIALIZATION_FIELDS,
		*COLLECTION_REPRESENTATIVE_SERIALIZATION_FIELDS,
		)
		.prefetch_related(
		"tags",
		"metadatacollectionvalue_set",
		@@ -226,11 +293,18 @@ def _resources_with_serialization_data(
		) -> QuerySet:
		# File serialization needs the subtype row, file type, project, and tags.
		# Metadata is only prefetched for callers that include it in the response.
		query_set = query_set.select_related(
		query_set = (
		query_set.select_related(
		"file",
		"file__project",
		"file__file_type",
		).prefetch_related("file__tags", "tags")
		)
		.only(
		*RESOURCE_SERIALIZATION_FIELDS,
		*FILE_SERIALIZATION_FIELDS,
		)
		.prefetch_related("file__tags", "tags")
		)
		if include_metas:
		query_set = query_set.prefetch_related(
		"file__metadataresourcevalue_set",
		@@ -2744,9 +2818,13 @@ def recycle_bin(user: User, project_id: int) -> List[Dict]:
		date_limit = timezone.now() - timedelta(days=15)

		results = []
		for collection_instance in Collection.objects.filter(
		for collection_instance in (
		Collection.objects.filter(
		deleted_at__isnull=False, project_id=project_id, deleted_at__gt=date_limit
		).order_by("-deleted_at"):
		)
		.only("id", "title", "deleted_at")
		.order_by("-deleted_at")
		):
		results.append(
		{
		"object_type": "collection",
		@@ -2755,9 +2833,15 @@ def recycle_bin(user: User, project_id: int) -> List[Dict]:
		"deleted_at": collection_instance.deleted_at.isoformat(),
		}
		)
		for resource_instance in Resource.objects.filter(
		deleted_at__isnull=False, ptr_project_id=project_id, deleted_at__gt=date_limit
		).order_by("-deleted_at"):
		for resource_instance in (
		Resource.objects.filter(
		deleted_at__isnull=False,
		ptr_project_id=project_id,
		deleted_at__gt=date_limit,
		)
		.only("id", "title", "deleted_at")
		.order_by("-deleted_at")
		):
		results.append(
		{
		"object_type": "resource",
		@@ -3097,7 +3181,7 @@ def _find_items_sets_from_selection_dict(
		id__in=selection["exclude"].get("resources_ids")
		)

		return resources_set, children_collections_set
		return resources_set.only("id"), children_collections_set.only("id")


		@_rpc_groups(["Resources", "Collections"])
		@@ -3285,9 +3369,11 @@ def update_collection_from_xlsx_row(user: User, collection_data: dict) -> bool:
		if new_title and new_title.strip():
		collection_instance.title = new_title.strip()

		for k in collection_data.keys():
		if k and ":" in k: # example: Dublin Core: creator
		meta_set_name, meta_name = [x.strip() for x in k.split(":", 1)]
		for column_name in collection_data.keys():
		if column_name and ":" in column_name: # example: Dublin Core: creator
		meta_set_name, meta_name = [
		column_part.strip() for column_part in column_name.split(":", 1)
		]
		if (
		meta_set_name
		and meta_name
		@@ -3302,12 +3388,12 @@ def update_collection_from_xlsx_row(user: User, collection_data: dict) -> bool:
		MetadataCollectionValue.objects.filter(
		metadata=metadata_instance, collection=collection_instance
		).delete()
		if collection_data.get(k) is not None:
		if collection_data.get(column_name) is not None:
		for value in [
		x.strip()
		for x in str(collection_data.get(k, "")).split(
		XLSX_MULTIPLE_VALUES_SEPARATOR
		)
		metadata_value_part.strip()
		for metadata_value_part in str(
		collection_data.get(column_name, "")
		).split(XLSX_MULTIPLE_VALUES_SEPARATOR)
		]:
		meta_collection_value = MetadataCollectionValue()
		meta_collection_value.collection = collection_instance
		@@ -3350,9 +3436,11 @@ def update_resource_from_xlsx_row(user: User, resource_data: dict) -> bool:
		if new_title and new_title.strip():
		resource_instance.title = new_title.strip()

		for k in resource_data.keys():
		if k and ":" in k: # example: Dublin Core: creator
		meta_set_name, meta_name = [x.strip() for x in k.split(":", 1)]
		for column_name in resource_data.keys():
		if column_name and ":" in column_name: # example: Dublin Core: creator
		meta_set_name, meta_name = [
		column_part.strip() for column_part in column_name.split(":", 1)
		]
		if (
		meta_set_name
		and meta_name
		@@ -3367,12 +3455,12 @@ def update_resource_from_xlsx_row(user: User, resource_data: dict) -> bool:
		MetadataResourceValue.objects.filter(
		metadata=metadata_instance, resource=resource_instance
		).delete()
		if resource_data.get(k) is not None:
		if resource_data.get(column_name) is not None:
		for value in [
		x.strip()
		for x in str(resource_data.get(k, "")).split(
		XLSX_MULTIPLE_VALUES_SEPARATOR
		)
		metadata_value_part.strip()
		for metadata_value_part in str(
		resource_data.get(column_name, "")
		).split(XLSX_MULTIPLE_VALUES_SEPARATOR)
		]:
		meta_resource_value = MetadataResourceValue()
		meta_resource_value.resource = resource_instance
		@@ -3402,10 +3490,10 @@ def user_tasks_status(user: User, project_id: int = None) -> List[dict]:
		```
		"""
		out = []
		q = UserTask.objects.filter(owner=user)
		user_tasks_query = UserTask.objects.filter(owner=user)
		if project_id is not None:
		q = q.filter(project_id=project_id)
		for user_task in q:
		user_tasks_query = user_tasks_query.filter(project_id=project_id)
		for user_task in user_tasks_query:
		out.append(serializers.user_task(user_task))
		return out

		@@ -3423,9 +3511,12 @@ def _serialize_annotation(annotation: Annotation) -> dict:


		def _fetch_resource(resource_id: int) -> Resource:
		resource_instance: Union[Resource, None] = Resource.objects.filter(
		pk=resource_id, deleted_at__isnull=True
		).first()
		resource_instance: Union[Resource, None] = (
		Resource.objects.filter(pk=resource_id, deleted_at__isnull=True)
		.select_related("ptr_project")
		.only("id", "title", "ptr_project_id", "ptr_project__id")
		.first()
		)
		if not resource_instance:
		raise ServiceException(NO_SUCH_RESOURCE)
		return resource_instance
		@@ -3439,8 +3530,21 @@ def list_annotations(user: User, resource_id: int) -> List[dict]:
		resource_instance = _fetch_resource(resource_id)
		UserAccess(user, resource_instance.ptr_project).check_read(resource_instance)
		annotations_list = []
		for annotation in Annotation.objects.filter(resource=resource_instance).order_by(
		"created_at"
		for annotation in (
		Annotation.objects.filter(resource=resource_instance)
		.select_related("owner")
		.only(
		"id",
		"owner_id",
		"owner__id",
		"owner__username",
		"data",
		"created_at",
		"updated_at",
		"resource_id",
		"public",
		)
		.order_by("created_at")
		):
		annotations_list.append(_serialize_annotation(annotation))
		return annotations_list
		@@ -3451,19 +3555,42 @@ def list_collection_annotations(user: User, collection_id: int) -> List[dict]:
		"""
		List all annotations for a given collection
		"""
		collection_instance = Collection.objects.filter(pk=collection_id).first()
		collection_instance = (
		Collection.objects.filter(pk=collection_id)
		.select_related("project")
		.only(
		*COLLECTION_SERIALIZATION_FIELDS,
		"project__id",
		)
		.first()
		)
		if not collection_instance:
		raise ServiceException("no such collection")
		UserAccess(user, collection_instance.project).check_read(collection_instance)
		resources_list = []
		for resource_instance in collection_instance.available_resources():
		for resource_instance in _resources_with_serialization_data(
		collection_instance.available_resources(), include_metas=False
		):
		serialized_resource = serializers.resource(
		resource_instance, include_metas=False
		)
		serialized_resource["annotations"] = []
		for annotation in Annotation.objects.filter(
		resource=resource_instance
		).order_by("created_at"):
		for annotation in (
		Annotation.objects.filter(resource=resource_instance)
		.select_related("owner")
		.only(
		"id",
		"owner_id",
		"owner__id",
		"owner__username",
		"data",
		"created_at",
		"updated_at",
		"resource_id",
		"public",
		)
		.order_by("created_at")
		):
		serialized_resource["annotations"].append(_serialize_annotation(annotation))
		resources_list.append(serialized_resource)
		return resources_list
		@@ -3503,7 +3630,26 @@ def update_annotation(
		"""
		Updates an annotation, returning the serialized annotation
		"""
		annotation_instance = Annotation.objects.filter(pk=annotation_id).first()
		annotation_instance = (
		Annotation.objects.filter(pk=annotation_id)
		.select_related("owner", "resource", "resource__ptr_project")
		.only(
		"id",
		"owner_id",
		"owner__id",
		"owner__username",
		"data",
		"created_at",
		"updated_at",
		"resource_id",
		"resource__id",
		"resource__title",
		"resource__ptr_project_id",
		"resource__ptr_project__id",
		"public",
		)
		.first()
		)
		if not annotation_instance:
		raise ServiceException("No such annotation")

		@@ -3521,7 +3667,19 @@ def delete_annotation(user: User, annotation_id: int) -> bool:
		"""
		Deletes an annotation
		"""
		annotation_instance = Annotation.objects.filter(pk=annotation_id).first()
		annotation_instance = (
		Annotation.objects.filter(pk=annotation_id)
		.select_related("resource", "resource__ptr_project")
		.only(
		"id",
		"resource_id",
		"resource__id",
		"resource__title",
		"resource__ptr_project_id",
		"resource__ptr_project__id",
		)
		.first()
		)
		if not annotation_instance:
		raise ServiceException("No such annotation")
		UserAccess(user, annotation_instance.resource.ptr_project).check_update(
		@@ -3533,7 +3691,21 @@ def delete_annotation(user: User, annotation_id: int) -> bool:

		@_rpc_groups(["Annotations"])
		def publish_annotation(user: User, annotation_id: int) -> bool:
		annotation_instance = Annotation.objects.filter(pk=annotation_id).first()
		annotation_instance = (
		Annotation.objects.filter(pk=annotation_id)
		.select_related("resource", "resource__ptr_project")
		.only(
		"id",
		"public",
		"updated_at",
		"resource_id",
		"resource__id",
		"resource__title",
		"resource__ptr_project_id",
		"resource__ptr_project__id",
		)
		.first()
		)
		if not annotation_instance:
		raise ServiceException("No such annotation")
		UserAccess(user, annotation_instance.resource.ptr_project).check_update(
		@@ -3546,7 +3718,21 @@ def publish_annotation(user: User, annotation_id: int) -> bool:

		@_rpc_groups(["Annotations"])
		def unpublish_annotation(user: User, annotation_id: int) -> bool:
		annotation_instance = Annotation.objects.filter(pk=annotation_id).first()
		annotation_instance = (
		Annotation.objects.filter(pk=annotation_id)
		.select_related("resource", "resource__ptr_project")
		.only(
		"id",
		"public",
		"updated_at",
		"resource_id",
		"resource__id",
		"resource__title",
		"resource__ptr_project_id",
		"resource__ptr_project__id",
		)
		.first()
		)
		if not annotation_instance:
		raise ServiceException("No such annotation")
		UserAccess(user, annotation_instance.resource.ptr_project).check_update(
		@@ -3608,15 +3794,29 @@ def resource_access_for_role(user: User, resource_id: int, role_id: int) -> dict
		@_require_superuser
		def list_projects(user: User) -> List[dict]:
		return [
		serializers.project(p)
		for p in Project.objects.filter(restore_state=Project.RESTORE_STATE_READY)
		serializers.project(project_instance)
		for project_instance in Project.objects.filter(
		restore_state=Project.RESTORE_STATE_READY
		).only("id", "label", "description")
		]


		@_rpc_groups(["Access"])
		@_require_superuser
		def list_users(user: User) -> List[dict]:
		return [serializers.user(u) for u in User.objects.all()]
		return [
		serializers.user(user_instance)
		for user_instance in User.objects.only(
		"id",
		"username",
		"last_name",
		"first_name",
		"is_staff",
		"is_superuser",
		"is_active",
		"email",
		)
		]


		@_rpc_groups(["Collections"])
		@@ -3661,17 +3861,28 @@ def collections_resources_pair(
		if not project_instance:
		raise ServiceException(NO_SUCH_PROJECT)
		UserAccess(user, project_instance).check_read(project_instance.root_collection)
		q = (
		memberships_query = (
		CollectionMembership.objects.filter(
		collection__project_id=project_id,
		collection__deleted_at__isnull=True,
		resource__deleted_at__isnull=True,
		)
		.select_related("collection", "resource")
		.only(
		"id",
		"rank",
		"collection_id",
		"resource_id",
		"collection__id",
		"collection__title",
		"collection__parent_id",
		"resource__id",
		"resource__title",
		)
		.order_by("collection_id", "rank")
		)
		if collection_id:
		q = q.filter(collection_id=collection_id)
		memberships_query = memberships_query.filter(collection_id=collection_id)

		return [
		{
		@@ -3685,5 +3896,5 @@ def collections_resources_pair(
		},
		"rank": item.rank,
		}
		for item in q
		for item in memberships_query
		]