add support for sqlite (d9daa055) · Commits · certic / Jama

jama/settings.py

+23 −0

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@ JAMA_USE_CACHALOT = True if os.getenv("JAMA_USE_CACHALOT", "0") == "1" else Fals
		JAMA_USE_WEBP = True if os.getenv("JAMA_USE_WEBP", "1") == "1" else False
		JAMA_USE_OAI = True if os.getenv("JAMA_USE_OAI", "0") == "1" else False
		JAMA_USE_ARK = True if os.getenv("JAMA_USE_ARK", "0") == "1" else False
		JAMA_SQLITE_DB_PATH = os.getenv("JAMA_SQLITE_DB_PATH", "")
		JAMA_IIIF_PROCESSING_DIR = os.getenv(
		"JAMA_IIIF_PROCESSING_DIR", f"{BASE_DIR}/processing"
		)
		@@ -163,6 +164,28 @@ WSGI_APPLICATION = "jama.wsgi.application"
		# Database
		# https://docs.djangoproject.com/en/3.1/ref/settings/#databases


		if JAMA_SQLITE_DB_PATH:
		DATABASES = {
		"default": {
		"ENGINE": "django.db.backends.sqlite3",
		"NAME": JAMA_SQLITE_DB_PATH,
		"OPTIONS": {
		"init_command": (
		"PRAGMA foreign_keys=ON;"
		"PRAGMA journal_mode = WAL;"
		"PRAGMA synchronous = NORMAL;"
		"PRAGMA busy_timeout = 5000;"
		"PRAGMA temp_store = MEMORY;"
		"PRAGMA mmap_size = 134217728;"
		"PRAGMA journal_size_limit = 67108864;"
		"PRAGMA cache_size = 2000;"
		),
		"transaction_mode": "IMMEDIATE",
		},
		}
		}
		else:
		DATABASES = {
		"default": {
		"ENGINE": "django.db.backends.postgresql",

resources/migrations/0005_descendants_count.py

+5 −1

Original line number	Diff line number	Diff line
		# Generated by Django 3.1.1 on 2021-02-10 05:59

		from django.conf import settings
		from django.db import migrations

		if settings.JAMA_SQLITE_DB_PATH:
		CREATE_SQL = ""
		DROP_SQL = ""
		else:
		CREATE_SQL = """
		CREATE OR REPLACE FUNCTION get_all_collection_descendants_array(use_parent INT4) RETURNS INT4[] as $$
		DECLARE

resources/models.py

+38 −3

Original line number	Diff line number	Diff line
		@@ -122,11 +122,14 @@ class Project(models.Model):

		@property
		def root_collection(self) -> "Collection":
		try:
		col, created = Collection.objects.get_or_create(project=self, parent=None)
		if created:
		col.title = "root {}".format(self.label)
		col.save()
		return col
		except Collection.MultipleObjectsReturned:
		return Collection.objects.filter(project=self, parent=None).first()

		def metadatas(self, exclude_automatic_metas=True) -> List["Metadata"]:
		metadatas = []
		@@ -461,7 +464,11 @@ class File(Resource):
		if self.has_extension("pdf") or self.has_extension("ai"):
		# extract PDF text layer if available
		pdftotext_return_code = call(
		["pdftotext", self.local_path(), self.local_path() + ".pdftotext.txt"]
		[
		"pdftotext",
		self.local_path(),
		self.local_path() + ".pdftotext.txt",
		]
		)
		if pdftotext_return_code == 0:
		with open(self.local_path() + ".pdftotext.txt", "r") as text_layer:
		@@ -552,13 +559,23 @@ class Collection(models.Model):
		def children(self):
		return Collection.objects.filter(parent=self, deleted_at__isnull=True)

		def descendants(self) -> RawQuerySet:
		def descendants(self) -> Union[RawQuerySet, Iterator["Collection"]]:
		if settings.JAMA_SQLITE_DB_PATH:
		yield from _recurse_collection(self)
		else:
		return Collection.objects.raw(
		"select * from resources_collection where id = any(get_all_collection_descendants_array(%s)) and deleted_at is null",
		[self.id],
		)

		def descendants_resources(self) -> RawQuerySet:
		def descendants_resources(self) -> Union[RawQuerySet, Iterator[Resource]]:
		if settings.JAMA_SQLITE_DB_PATH:
		for res in self.resources.filter(deleted_at__isnull=True):
		yield res
		for col in _recurse_collection(self):
		for res in col.resources.filter(deleted_at__isnull=True):
		yield res
		else:
		sql = """
		select * from resources_resource
		inner join resources_collectionmembership rc
		@@ -577,6 +594,9 @@ class Collection(models.Model):
		return ids

		def descendants_count(self) -> int:
		if settings.JAMA_SQLITE_DB_PATH:
		return sum(1 for _ in self.descendants())
		else:
		with connection.cursor() as cursor:
		cursor.execute(
		"""select count(*)
		@@ -593,6 +613,12 @@ class Collection(models.Model):
		This counts resources from all the descendant collections,
		EXCLUDING the current collection's direct resources.
		"""
		if settings.JAMA_SQLITE_DB_PATH:
		total = 0
		for col in self.descendants():
		total = total + col.resources.filter(deleted_at__isnull=True).count()
		return total
		else:
		with connection.cursor() as cursor:
		# For some reason, Postgresql will perform a fast index scan when given the list of collection ids as a
		# string literal in an ANY clause.
		@@ -748,6 +774,15 @@ class Collection(models.Model):
		yield _flatten_resource(res, known_metadatas, known_metadatas_labels)


		def _recurse_collection(collection: Collection) -> Iterator[Collection]:
		try:
		for child in collection.children():
		yield child
		yield from _recurse_collection(child)
		except RecursionError:
		pass


		class MetadataCollectionValue(models.Model):
		metadata = models.ForeignKey(Metadata, on_delete=models.CASCADE)
		collection = models.ForeignKey(Collection, on_delete=models.CASCADE)