performance tweaks (d8d2cfdb) · Commits · certic / Jama

jama/settings.py

+2 −0

Original line number	Diff line number	Diff line
		@@ -60,6 +60,8 @@ INSTALLED_APPS = [
		if DEBUG:
		INSTALLED_APPS.append("django.contrib.admin")

		SHOW_SQL = True if os.getenv("JAMA_SHOW_SQL", False) == "1" else False

		AUTO_REGISTER_APPS = []
		for app in os.getenv("JAMA_APPS", "").split(","):
		if app:

rpc/cache.py

0 → 100644

+27 −0

Original line number	Diff line number	Diff line
		from resources import models
		from django.contrib.auth.models import User


		class SerializerCache:
		def __init__(self):
		self._cache_dict = {}

		def fetch_obj(self, obj_type: str, obj_id: int):
		cache_key = "{}{}".format(obj_type, obj_id)
		if cache_key not in self._cache_dict.keys():
		if obj_type == "metadata":
		self._cache_dict[cache_key] = models.Metadata.objects.get(pk=obj_id)
		elif obj_type == "metadataset":
		self._cache_dict[cache_key] = models.MetadataSet.objects.get(pk=obj_id)
		elif obj_type == "user":
		self._cache_dict[cache_key] = User.objects.get(pk=obj_id)
		elif obj_type == "project":
		self._cache_dict[cache_key] = models.Project.objects.get(pk=obj_id)
		elif obj_type == "role":
		self._cache_dict[cache_key] = models.Role.objects.get(pk=obj_id)
		elif obj_type == "file_type":
		self._cache_dict[cache_key] = models.FileType.objects.get(pk=obj_id)
		else:
		raise ValueError("unsupported object type: {}".format(obj_type))
		print("cache size: {}".format(len(self._cache_dict)))
		return self._cache_dict.get(cache_key)

rpc/methods.py

+73 −27

Original line number	Diff line number	Diff line
		@@ -24,7 +24,7 @@ import os
		import re
		from glob import glob as _glob
		import unidecode

		from rpc.cache import SerializerCache

		from functools import wraps as _wraps
		from rpc.const import *
		@@ -59,6 +59,7 @@ def _log_call(func):

		@_wraps(func)
		def wrapper(args, *kwargs):
		print(func)
		print(args)
		print(kwargs)
		return func(args, *kwargs)
		@@ -183,8 +184,9 @@ def metadatasets(user: User, project_id: int) -> List[Dict]:
		project__projectaccess__user=user,
		project__projectaccess__role__permissions__label=PERM_METADATASET_READ,
		)
		serializer_cache = SerializerCache()
		for item in query_set:
		data.append(serializers.metadataset(item))
		data.append(serializers.metadataset(item, cache=serializer_cache))
		return data
		except Project.DoesNotExist:
		raise ServiceException(NO_SUCH_PROJECT)
		@@ -229,8 +231,9 @@ def metadatas(user: User, metadata_set_id: int) -> List[Dict]:
		)
		metadata_set: MetadataSet = query_set.first()
		if metadata_set:
		serializer_cache = SerializerCache()
		for metadata_instance in Metadata.objects.filter(set=metadata_set, expose=True):
		data.append(serializers.metadata(metadata_instance))
		data.append(serializers.metadata(metadata_instance, cache=serializer_cache))
		return data


		@@ -261,7 +264,7 @@ def metadata(user: User, metadata_id: int) -> Dict:
		metadata_instance = query_set.first()
		if not metadata_instance:
		raise ServiceException(NO_SUCH_METADATA)
		return serializers.metadata(metadata_instance)
		return serializers.metadata(metadata_instance, cache=SerializerCache())


		@_log_call
		@@ -356,10 +359,14 @@ def collections(
		if only_published:
		query_set = query_set.filter(public_access=True)
		query_set = query_set[limit_from:limit_to]
		serializer_cache = SerializerCache()
		for item in query_set:
		data.append(
		serializers.collection(
		item, recursive=recursive, only_published=only_published
		item,
		recursive=recursive,
		only_published=only_published,
		cache=serializer_cache,
		)
		)
		return data
		@@ -399,7 +406,9 @@ def collection(user: User, collection_id: int) -> Dict:
		# Special case: user has limiting permission
		if public_only and not collection_instance.public_access:
		raise ServiceException(CAN_ONLY_SHOW_PUBLIC_COLLECTIONS_CONTENT)
		return serializers.collection(collection_instance, only_published=public_only)
		return serializers.collection(
		collection_instance, only_published=public_only, cache=SerializerCache()
		)


		@_log_call
		@@ -449,7 +458,7 @@ def add_collection(user: User, title: str, parent_id: int) -> Dict:
		if collection_instance.deleted_at:
		collection_instance.deleted_at = None
		collection_instance.save()
		return serializers.collection(collection_instance)
		return serializers.collection(collection_instance, cache=SerializerCache())


		@_log_call
		@@ -522,6 +531,7 @@ def add_collection_from_path(user: User, path: str, project_id: int) -> List[Dic
		_check_project_permission(user, project, PERM_COLLECTION_CREATE)
		hierarchy = []
		previous_dir = project.root_collection
		serializer_cache = SerializerCache()
		for dir_name in path.split("/"):
		# force ascii representation of unicode strings
		dir_name = unidecode.unidecode(dir_name.strip())
		@@ -533,7 +543,9 @@ def add_collection_from_path(user: User, path: str, project_id: int) -> List[Dic
		if previous_dir.deleted_at:
		previous_dir.deleted_at = None
		previous_dir.save()
		hierarchy.append(serializers.collection(previous_dir))
		hierarchy.append(
		serializers.collection(previous_dir, cache=serializer_cache)
		)
		return hierarchy


		@@ -770,8 +782,13 @@ def resources(

		query_set = query_set.order_by(order_by)

		serializer_cache = SerializerCache()
		for item in query_set[limit_from:limit_to]:
		data.append(serializers.resource(item, include_metas=bool(include_metas)))
		data.append(
		serializers.resource(
		item, include_metas=bool(include_metas), cache=serializer_cache
		)
		)
		return data


		@@ -810,7 +827,7 @@ def resource(user: User, resource_id: int) -> Dict:
		for _ in resource_instance.collections.filter(public_access=False):
		raise ServiceException(CAN_ONLY_SHOW_PUBLIC_COLLECTIONS_CONTENT)
		_check_project_permission(user, resource_instance.ptr_project, PERM_RESOURCE_READ)
		return serializers.resource(resource_instance)
		return serializers.resource(resource_instance, cache=SerializerCache())


		@_log_call
		@@ -969,10 +986,13 @@ def simple_search(
		)
		if public_only:
		collections_set = collections_set.filter(public_access=True)
		serializer_cache = SerializerCache()
		for collection_instance in collections_set.distinct().order_by(order_by)[
		limit_from:limit_to
		]:
		results["collections"].append(serializers.collection(collection_instance))
		results["collections"].append(
		serializers.collection(collection_instance, cache=serializer_cache)
		)
		resources_set: Union[Iterator[Resource], QuerySet] = Resource.objects.filter(
		title__icontains=query, deleted_at__isnull=True, ptr_project=project
		)
		@@ -982,7 +1002,9 @@ def simple_search(
		limit_from:limit_to
		]:
		results["resources"].append(
		serializers.resource(resource_instance, include_metas=False)
		serializers.resource(
		resource_instance, include_metas=False, cache=serializer_cache
		)
		)

		return results
		@@ -1139,11 +1161,14 @@ def advanced_search(
		resources_set = resources_set.exclude(
		tags__uid__in=search_term["exclude_tags"]
		)
		serializer_cache = SerializerCache()
		for resource_instance in resources_set.distinct().order_by(order_by)[
		limit_from:limit_to
		]:
		results["resources"].append(
		serializers.resource(resource_instance, include_metas=include_metas)
		serializers.resource(
		resource_instance, include_metas=include_metas, cache=serializer_cache
		)
		)

		# Search collections second
		@@ -1200,10 +1225,13 @@ def advanced_search(
		collections_set = collections_set.exclude(
		tags__uid__in=search_term["exclude_tags"]
		)
		serializer_cache = SerializerCache()
		for collection_instance in collections_set.distinct().order_by(order_by)[
		limit_from:limit_to
		]:
		results["collections"].append(serializers.collection(collection_instance))
		results["collections"].append(
		serializers.collection(collection_instance, cache=serializer_cache)
		)

		return results

		@@ -1294,8 +1322,9 @@ def supported_file_types(user: User) -> List[Dict]:
		```
		"""
		ftypes = []
		serializer_cache = SerializerCache()
		for ftype in FileType.objects.all():
		ftypes.append(serializers.file_type(ftype))
		ftypes.append(serializers.file_type(ftype, cache=serializer_cache))
		return ftypes


		@@ -1695,6 +1724,7 @@ def ancestors_from_collection(
		if not collection_instance:
		return breadcrumb
		current_collection = collection_instance
		serializer_cache = SerializerCache()
		while True:
		if current_collection.parent:
		if (
		@@ -1702,14 +1732,22 @@ def ancestors_from_collection(
		and not current_collection.deleted_at
		):
		breadcrumb.append(
		serializers.collection(current_collection.parent, recursive=False)
		serializers.collection(
		current_collection.parent,
		recursive=False,
		cache=serializer_cache,
		)
		)
		current_collection = current_collection.parent
		else:
		break
		breadcrumb.reverse()
		if include_self:
		breadcrumb.append(serializers.collection(collection_instance, recursive=False))
		breadcrumb.append(
		serializers.collection(
		collection_instance, recursive=False, cache=serializer_cache
		)
		)
		return breadcrumb


		@@ -1800,8 +1838,9 @@ def metadatatypes(user: User) -> List[dict]:
		```
		"""
		types = []
		serializer_cache = SerializerCache()
		for mt in MetadataType.objects.all():
		types.append(serializers.metadatatype(mt))
		types.append(serializers.metadatatype(mt, cache=serializer_cache))
		return types


		@@ -1856,7 +1895,7 @@ def move_collection(
		- parent collection is a descendant of child collection
		"""
		if child_collection_id == parent_collection_id: # no loop !
		return False
		raise ServiceException(NO_FOOTGUNS)
		query_set: Union[Iterator[Collection], QuerySet] = Collection.objects.filter(
		pk=child_collection_id, deleted_at__isnull=True
		)
		@@ -1909,7 +1948,7 @@ def set_tag(
		tag_instance.ark = ark
		tag_instance.label = label
		tag_instance.save()
		return serializers.tag(tag_instance)
		return serializers.tag(tag_instance, cache=SerializerCache())


		@_log_call
		@@ -1957,8 +1996,9 @@ def tags(user: User, project_id: int) -> List[dict]:
		project = Project.objects.filter(pk=project_id).first()
		_check_project_permission(user, project, "tag.read")
		tags_list = []
		serializer_cache = SerializerCache()
		for tag_instance in Tag.objects.filter(project=project):
		tags_list.append(serializers.tag(tag_instance))
		tags_list.append(serializers.tag(tag_instance, cache=serializer_cache))
		return tags_list


		@@ -2236,7 +2276,7 @@ def create_project(user: User, project_label: str, project_description: str) ->
		project = Project.objects.create(
		label=project_label, description=project_description
		)
		return serializers.project(project)
		return serializers.project(project, cache=SerializerCache())


		@_log_call
		@@ -2276,8 +2316,9 @@ def list_permissions(user: User) -> List[Dict]:
		```
		"""
		data = []
		serializer_cache = SerializerCache()
		for perm in Permission.objects.all():
		data.append(serializers.permission(perm))
		data.append(serializers.permission(perm, cache=serializer_cache))
		return data


		@@ -2300,8 +2341,11 @@ def projects_user_permissions(user: User) -> List[Dict]:
		```
		"""
		access_list = []
		serializer_cache = SerializerCache()
		for project_access in ProjectAccess.objects.filter(user=user):
		access_list.append(serializers.project_access(project_access))
		access_list.append(
		serializers.project_access(project_access, cache=serializer_cache)
		)
		return access_list


		@@ -2327,8 +2371,9 @@ def list_roles(user: User, project_id: int) -> List[Dict]:
		"""
		_check_project_permission(user, project_id, PERM_ROLE_READ)
		data = []
		serializer_cache = SerializerCache()
		for role in Role.objects.filter(project_id=project_id):
		data.append(serializers.role(role))
		data.append(serializers.role(role, cache=serializer_cache))
		return data


		@@ -2354,7 +2399,7 @@ def set_role(
		role.permissions.add(perm)
		except Permission.DoesNotExist:
		pass
		return serializers.role(role)
		return serializers.role(role, cache=SerializerCache())


		@_log_call
		@@ -2570,6 +2615,7 @@ def public_collections(user: User, project_id: int) -> List[dict]:
		"""
		_check_project_permission(user, project_id, PERM_COLLECTION_READ)
		data = []
		serializer_cache = SerializerCache()
		for col in Collection.objects.filter(public_access=True, project_id=project_id):
		data.append(serializers.collection(col))
		data.append(serializers.collection(col, cache=serializer_cache))
		return data

rpc/serializers.py

+69 −27

Original line number	Diff line number	Diff line
		from resources import models
		from jama import settings
		import os
		from rpc.cache import SerializerCache


		def get_cache(cache: SerializerCache = None):
		if not cache:
		return SerializerCache()
		return cache


		def iiif_file_exists(file_hash) -> bool:
		@@ -10,7 +17,7 @@ def iiif_file_exists(file_hash) -> bool:
		return os.path.isfile(path_to_test)


		def file_type(ftype: models.FileType) -> dict:
		def file_type(ftype: models.FileType, cache: SerializerCache = None) -> dict:
		extensions = []
		for ext in ftype.extensions.all():
		extensions.append(ext.label)
		@@ -21,7 +28,9 @@ def file_type(ftype: models.FileType) -> dict:
		}


		def metadataset(metadataset_instance: models.MetadataSet) -> dict:
		def metadataset(
		metadataset_instance: models.MetadataSet, cache: SerializerCache = None
		) -> dict:
		return {
		"object_type": "metadataset",
		"id": metadataset_instance.id,
		@@ -31,13 +40,14 @@ def metadataset(metadataset_instance: models.MetadataSet) -> dict:
		}


		def metadata(metadata_instance: models.Metadata) -> dict:
		def metadata(metadata_instance: models.Metadata, cache: SerializerCache = None) -> dict:
		set_instance = get_cache(cache).fetch_obj("metadataset", metadata_instance.set_id)
		return {
		"object_type": "metadata",
		"id": metadata_instance.id,
		"title": metadata_instance.title,
		"set_id": metadata_instance.set_id,
		"set_title": metadata_instance.set.title,
		"set_title": set_instance.title,
		"rank": metadata_instance.rank,
		"project_id": metadata_instance.project_id,
		}
		@@ -45,22 +55,30 @@ def metadata(metadata_instance: models.Metadata) -> dict:

		def metadata_collection_value(
		metadata_collection_value_instance: models.MetadataCollectionValue,
		cache: SerializerCache = None,
		) -> dict:
		metadata_instance = get_cache(cache).fetch_obj(
		"metadata", metadata_collection_value_instance.metadata_id
		)
		return {
		"object_type": "metadata_collection_value",
		"id": metadata_collection_value_instance.id,
		"meta": metadata(metadata_collection_value_instance.metadata),
		"meta": metadata(metadata_instance, cache=cache),
		"value": metadata_collection_value_instance.value,
		}


		def metadata_resource_value(
		metadata_resource_value_instance: models.MetadataResourceValue,
		cache: SerializerCache = None,
		) -> dict:
		metadata_instance = get_cache(cache).fetch_obj(
		"metadata", metadata_resource_value_instance.metadata_id
		)
		return {
		"object_type": "metadata_resource_value",
		"id": metadata_resource_value_instance.id,
		"meta": metadata(metadata_resource_value_instance.metadata),
		"meta": metadata(metadata_instance, cache=cache),
		"value": metadata_resource_value_instance.value,
		}

		@@ -69,10 +87,11 @@ def collection(
		collection_instance: models.Collection,
		recursive: bool = False,
		only_published: bool = False,
		cache: SerializerCache = None,
		) -> dict:
		tags = []
		for tag_instance in collection_instance.tags.all():
		tags.append(tag(tag_instance))
		tags.append(tag(tag_instance, cache=cache))
		payload = {
		"object_type": "collection",
		"id": collection_instance.id,
		@@ -89,7 +108,9 @@ def collection(
		"metas": [],
		"public_access": collection_instance.public_access,
		"tags": tags,
		"represented_by": resource(collection_instance.representative, False)
		"represented_by": resource(
		collection_instance.representative, False, cache=cache
		)
		if collection_instance.representative
		and collection_instance.representative.deleted_at is None
		else None,
		@@ -106,23 +127,27 @@ def collection(
		# filter private content
		if only_published and child.public_access == False:
		continue
		payload["children"].append(collection(child, recursive))
		payload["children"].append(collection(child, recursive, cache=cache))

		for prop in collection_instance.metadatacollectionvalue_set.all():
		payload["metas"].append(metadata_collection_value(prop))
		payload["metas"].append(metadata_collection_value(prop, cache=cache))

		return payload


		def resource(resource_instance: models.Resource, include_metas: bool = True) -> dict:
		def resource(
		resource_instance: models.Resource,
		include_metas: bool = True,
		cache: SerializerCache = None,
		) -> dict:
		try:
		if resource_instance.file:
		return file(resource_instance.file, include_metas)
		return file(resource_instance.file, include_metas, cache=cache)
		except models.File.DoesNotExist:
		pass
		tags = []
		for tag_instance in resource_instance.tags.all():
		tags.append(tag(tag_instance))
		tags.append(tag(tag_instance, cache=cache))
		return {
		"object_type": "resource",
		"id": resource_instance.id,
		@@ -133,14 +158,18 @@ def resource(resource_instance: models.Resource, include_metas: bool = True) ->
		}


		def file(file_instance: models.File, include_metas: bool = True) -> dict:
		def file(
		file_instance: models.File,
		include_metas: bool = True,
		cache: SerializerCache = None,
		) -> dict:
		props = []
		urls = {
		"download": "{}rpc/download/{}".format(settings.JAMA_SITE, file_instance.pk)
		}
		tags = []
		for tag_instance in file_instance.tags.all():
		tags.append(tag(tag_instance))
		tags.append(tag(tag_instance, cache=cache))
		if (
		file_instance.should_have_iiif()
		): # and iiif_file_exists(file_instance.hash): # FS is slow
		@@ -159,13 +188,16 @@ def file(file_instance: models.File, include_metas: bool = True) -> dict:
		for prop in file_instance.metadataresourcevalue_set.filter(
		metadata__expose=True
		).order_by("metadata__title"):
		props.append(metadata_resource_value(prop))
		props.append(metadata_resource_value(prop, cache=cache))
		file_type_instance = get_cache(cache).fetch_obj(
		"file_type", file_instance.file_type_id
		)
		return {
		"object_type": "resource",
		"id": file_instance.id,
		"title": file_instance.title,
		"original_name": file_instance.original_name,
		"type": str(file_instance.file_type),
		"type": str(file_type_instance),
		"hash": file_instance.hash,
		"metas": props or None,
		"urls": urls,
		@@ -175,7 +207,9 @@ def file(file_instance: models.File, include_metas: bool = True) -> dict:
		}


		def metadatatype(metada_type: models.MetadataType) -> dict:
		def metadatatype(
		metada_type: models.MetadataType, cache: SerializerCache = None
		) -> dict:
		return {
		"object_type": "metada_type",
		"id": metada_type.id,
		@@ -183,7 +217,7 @@ def metadatatype(metada_type: models.MetadataType) -> dict:
		}


		def tag(tag_instance: models.Tag) -> dict:
		def tag(tag_instance: models.Tag, cache: SerializerCache = None) -> dict:
		return {
		"object_type": "tag",
		"id": tag_instance.id,
		@@ -193,7 +227,7 @@ def tag(tag_instance: models.Tag) -> dict:
		}


		def project(project_instance: models.Project) -> dict:
		def project(project_instance: models.Project, cache: SerializerCache = None) -> dict:
		return {
		"object_type": "project",
		"id": project_instance.pk,
		@@ -202,7 +236,9 @@ def project(project_instance: models.Project) -> dict:
		}


		def permission(permission_instance: models.Permission) -> dict:
		def permission(
		permission_instance: models.Permission, cache: SerializerCache = None
		) -> dict:
		return {
		"object_type": "permission",
		"id": permission_instance.pk,
		@@ -210,10 +246,10 @@ def permission(permission_instance: models.Permission) -> dict:
		}


		def role(role_instance: models.Role) -> dict:
		def role(role_instance: models.Role, cache: SerializerCache = None) -> dict:
		permissions = []
		for permission_instance in role_instance.permissions.all():
		permissions.append(permission(permission_instance))
		permissions.append(permission(permission_instance, cache=cache))
		return {
		"object_type": "role",
		"id": role_instance.id,
		@@ -222,10 +258,16 @@ def role(role_instance: models.Role) -> dict:
		}


		def project_access(project_access_instance: models.ProjectAccess) -> dict:
		def project_access(
		project_access_instance: models.ProjectAccess, cache: SerializerCache = None
		) -> dict:
		cache = get_cache(cache)
		role_instance = cache.fetch_obj("role", project_access_instance.role_id)
		project_instance = cache.fetch_obj("project", project_access_instance.project_id)
		user_instance = cache.fetch_obj("user", project_access_instance.user_id)
		return {
		"object_type": "project_access",
		"project": project(project_access_instance.project),
		"role": role(project_access_instance.role),
		"user": project_access_instance.user.username,
		"project": project(project_instance, cache=cache),
		"role": role(role_instance, cache=cache),
		"user": user_instance.username,
		}

rpc/views.py

+34 −19

Original line number	Diff line number	Diff line
		@@ -42,18 +42,33 @@ def _silent_remove(file_path: str):
		pass


		def debug_sql():
		if settings.DEBUG:
		def _debug_sql():
		if not settings.SHOW_SQL:
		return
		print("")
		print("#--- Start PostgreSQL Queries ---#")
		print("")
		req_count = 0
		req_mult_count = {}
		has_doubled_queries = False
		for query in connection.queries:
		if query["sql"] in req_mult_count.keys():
		req_mult_count[query["sql"]] = req_mult_count[query["sql"]] + 1
		has_doubled_queries = True
		else:
		req_mult_count[query["sql"]] = 1
		print("")
		print(" ", query["sql"])
		print("")
		req_count = req_count + 1
		print("#--- End PostgreSQL Queries ({} queries) ---#".format(req_count))
		if has_doubled_queries:
		print("")
		print("#--- Doubles ---#")
		print("")
		for k, v in req_mult_count.items():
		if v > 1:
		print("({} times): {}".format(v, k))


		def _file_hash256(file_path: str) -> str:
		@@ -166,16 +181,16 @@ def rpc(request: HttpRequest) -> HttpResponse:
		params.insert(0, user) # always add user to method call
		req_id = json_data["id"]
		result = methods[method](*params)
		debug_sql()
		_debug_sql()
		return JsonResponse({"result": result, "error": None, "id": req_id})
		except json.JSONDecodeError:
		return HttpResponse("Bad Request", status=400)
		except KeyError:
		return HttpResponse("Bad Request", status=400)
		except TypeError:
		return HttpResponse("Bad Request", status=400)
		except ValueError:
		return HttpResponse("Bad Request", status=400)
		# except json.JSONDecodeError:
		# return HttpResponse("Bad Request", status=400)
		# except KeyError:
		# return HttpResponse("Bad Request", status=400)
		# except TypeError:
		# return HttpResponse("Bad Request", status=400)
		# except ValueError:
		# return HttpResponse("Bad Request", status=400)
		except ServiceException as e:
		return JsonResponse(
		{