expose meilisearch through rpc api (d8870e93) · Commits · certic / Jama

jama/settings.py

+0 −1

Original line number	Diff line number	Diff line
		@@ -31,7 +31,6 @@ TMP_THUMBNAILS_DIR = os.getenv("JAMA_TMP_THUMBNAILS_DIR") or tempfile.gettempdir
		JAMA_SITE = os.getenv("JAMA_SITE") or "http://localhost:8000/"

		JAMA_IIIF_UPSCALING_PREFIX = os.getenv("JAMA_IIIF_UPSCALING_PREFIX") or ""

		IIIF_DIR = os.getenv("JAMA_IIIF_DIR") or "{}/{}".format(BASE_DIR, "iiif")
		IIIF_PATH_SEPARATOR = os.getenv("JAMA_IIIF_PATH_SEPARATOR") or os.path.sep

resources/management/commands/meilisearchindex.py

0 → 100644

+41 −0

Original line number	Diff line number	Diff line
		from django.core.management.base import BaseCommand
		from meilisearch.index import Index
		from meilisearch import Client
		from django.conf import settings
		from typing import Tuple
		from resources.models import Resource
		from django.forms.models import model_to_dict


		MAX_TOTAL_HITS = 1000000


		def meili_client_and_index() -> Tuple[Client, Index]:
		if settings.MEILISEARCH_KEY:
		client = Client(settings.MEILISEARCH_URL, settings.MEILISEARCH_KEY)
		else:
		client = Client(settings.MEILISEARCH_URL)
		return client, client.index("jama_resources")


		def resource_to_dict(resource: Resource) -> dict:
		as_dict = model_to_dict(resource)
		return as_dict


		def index_docs():
		client, index = meili_client_and_index()
		index.update_settings({"pagination": {"maxTotalHits": MAX_TOTAL_HITS}})
		index.update_typo_tolerance({"enabled": True})
		index.update_filterable_attributes(Resource.facets())
		index.update_searchable_attributes(["title", "metas.OCR / tesseract output"])
		index.update_sortable_attributes(Resource.facets())

		for resource in Resource.objects.filter(deleted_at__isnull=True):
		resource.send_to_meilisearch(client)
		print(resource.title)


		class Command(BaseCommand):
		def handle(self, args, *options):
		index_docs()

resources/models.py

+18 −1

Original line number	Diff line number	Diff line
		@@ -241,7 +241,24 @@ class Resource(models.Model):
		as_dict["collections"].append(" / ".join(col.to_path()[1:]))
		if not client:
		client = meili_client()
		client.index("jama").add_documents([as_dict])
		client.index("jama_resources").add_documents([as_dict])

		@staticmethod
		def facets() -> List[str]:
		return [
		"project",
		"metas.Dublin Core / title",
		"metas.Dublin Core / contributor",
		"metas.Dublin Core / publisher",
		"metas.Dublin Core / date",
		"metas.Dublin Core / format",
		"metas.Dublin Core / description",
		"metas.Dublin Core / subject",
		"metas.Dublin Core / language",
		"metas.Dublin Core / source",
		"metas.scd / cote",
		"metas.scd / library",
		]


		class File(Resource):

resources/tasks.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)

		def _meili_client_and_index() -> Tuple[MeilisearchClient, MeilisearchIndex]:
		client = MeilisearchClient(settings.MEILISEARCH_URL, settings.MEILISEARCH_KEY)
		return client, client.index("jama")
		return client, client.index("jama_resources")


		@db_task()

rpc/methods.py

+77 −0

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@ from resources.models import (
		APIKey,
		ProjectProperty,
		)
		from resources.models import meili_client as _meilisearch_client
		from django.contrib.auth.models import User
		from django.db.models import QuerySet
		from typing import List, Dict, Iterator, Union, Tuple
		@@ -3138,3 +3139,79 @@ def auto_find_rotate_angle(user: User, resource_id: int) -> float:
		if not resource_instance.file.should_have_iiif():
		raise ServiceException(NOT_AN_IMAGE)
		return _deskew(resource_instance.file.local_path())


		@_rpc_groups(["Search"])
		def meilisearch_resources_facets(user: User) -> List[str]:
		"""
		List Resource model facets for Meilisearch:

		```
		[
		"project",
		"metas.Dublin Core / title",
		"metas.Dublin Core / contributor",
		"metas.Dublin Core / publisher",
		"metas.Dublin Core / date",
		"metas.Dublin Core / format",
		"metas.Dublin Core / description",
		"metas.Dublin Core / subject",
		"metas.Dublin Core / language",
		"metas.Dublin Core / source",
		"metas.scd / cote",
		"metas.scd / library",
		]
		```
		"""
		return Resource.facets()


		@_rpc_groups(["Search"])
		def meilisearch_resources(
		user: User, project_id: int, query: str = None, opt_params: dict = None
		) -> dict:
		"""
		Performs a query on the Meilisearch index.

		_query_ searches in resource _title_, and _meta_ "metas.OCR / tesseract output" if available. Set to None or empty string if not plain-text search is required.

		_opt\_params_ is the optional parameters for Meilisearch. See [Meilisearch documentation](https://www.meilisearch.com/docs/reference/api/search) .

		Example opt_params:

		```
		{
		"facets": [
		"title",
		"metas.Dublin Core / date"
		],
		"filter": "\\"metas.Dublin Core / date\\"=1999",
		"hitsPerPage": 50,
		"page": 1,
		"sort": [
		"title:asc"
		]
		}
		```

		_project_ filter is automatically appended, according to given _project\_id_
		"""
		project = Project.objects.filter(pk=project_id).first()
		if not query:
		query = ""
		if not project:
		raise ServiceException(NO_SUCH_PROJECT)
		_check_project_permission(user, project, PERM_RESOURCE_READ)
		if not opt_params:
		opt_params = {}
		if not opt_params.get("filter"):
		opt_params["filter"] = f'project="{project.label}"'
		else:
		opt_params["filter"] = opt_params["filter"] + f' AND project="{project.label}"'
		meilisearch_client = _meilisearch_client()
		meilisearch_index = meilisearch_client.index("jama_resources")
		search_result = meilisearch_index.search(
		query,
		opt_params,
		)
		return search_result