Commit 033a51bb authored by Mickaël Desfrênes's avatar Mickaël Desfrênes
Browse files

remove owner references from OAI

parent 3ca5be00

oai/urls.py

+1 −1
Original line number Diff line number Diff line
@@ -3,5 +3,5 @@ from django.urls import path 
from . import views



urlpatterns = [

    path("owner/<int:owner_id>/", views.oai),

    path("project/<int:project_id>/", views.oai),

]

oai/views.py

+39 −37
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ from django.http import ( 
from django.contrib.auth.models import User

from django.template.loader import render_to_string

from mediatheque import settings

from resources.models import Collection

from resources.models import Collection, Project

from datetime import datetime

from django.views.decorators.csrf import csrf_exempt

from django.core.paginator import Paginator
@@ -45,34 +45,36 @@ OAI_VERBS = [ 
]





def _oai_error(owner: User, request: HttpRequest, errors: dict) -> HttpResponse:

def _oai_error(project: Project, request: HttpRequest, errors: dict) -> HttpResponse:

    return HttpResponse(

        render_to_string(

            "oai/errors.xml",

            {"owner": owner, "base_url": settings.JAMA_SITE, "errors": errors},

            {"project": project, "base_url": settings.JAMA_SITE, "errors": errors},

        ),

        content_type="text/xml",

    )





def _get_record(owner: User, request: HttpRequest) -> HttpResponse:

def _get_record(project: Project, request: HttpRequest) -> HttpResponse:

    try:

        identifier = int(request.GET.get("identifier")) or request.POST.get(

            "identifier"

        )

        if not identifier:

            return _oai_error(owner, request, {"badArgument": "identifier is missing"})

            return _oai_error(

                project, request, {"badArgument": "identifier is missing"}

            )

    except ValueError:

        return _oai_error(

            owner, request, {"idDoesNotExist": "identifier does not exist"}

            project, request, {"idDoesNotExist": "identifier does not exist"}

        )

    try:

        collection_instance = Collection.objects.get(pk=identifier, owner=owner)

        collection_instance = Collection.objects.get(pk=identifier, project=project)

        return HttpResponse(

            render_to_string(

                "oai/get_record.xml",

                {

                    "owner": owner,

                    "project": project,

                    "base_url": settings.JAMA_SITE,

                    "collection": collection_instance,

                },
@@ -81,13 +83,13 @@ def _get_record(owner: User, request: HttpRequest) -> HttpResponse: 
        )

    except Collection.DoesNotExist:

        return _oai_error(

            owner, request, {"idDoesNotExist": "identifier does not exist"}

            project, request, {"idDoesNotExist": "identifier does not exist"}

        )





def _identify(owner: User, request: HttpRequest) -> HttpResponse:

def _identify(project: Project, request: HttpRequest) -> HttpResponse:

    earliest_record = (

        Collection.objects.filter(is_oai_record=True, owner=owner)

        Collection.objects.filter(is_oai_record=True, project=project)

        .order_by("created_at")

        .first()

    )
@@ -95,7 +97,7 @@ def _identify(owner: User, request: HttpRequest) -> HttpResponse: 
        render_to_string(

            "oai/identify.xml",

            {

                "owner": owner,

                "project": project,

                "base_url": settings.JAMA_SITE,

                "earliest_record": earliest_record,

            },
@@ -104,16 +106,16 @@ def _identify(owner: User, request: HttpRequest) -> HttpResponse: 
    )





def _list_identifiers(owner: User, request: HttpRequest) -> HttpResponse:

    return _list_records(owner, request, only_headers=True)

def _list_identifiers(project: Project, request: HttpRequest) -> HttpResponse:

    return _list_records(project, request, only_headers=True)





def _list_metadata_formats(owner: User, request: HttpRequest) -> HttpResponse:

def _list_metadata_formats(project: Project, request: HttpRequest) -> HttpResponse:

    return HttpResponse(

        render_to_string(

            "oai/list_metadata_formats.xml",

            {

                "owner": owner,

                "project": project,

                "base_url": settings.JAMA_SITE,

            },

        ),
@@ -122,7 +124,7 @@ def _list_metadata_formats(owner: User, request: HttpRequest) -> HttpResponse: 




def _list_records(

    owner: User, request: HttpRequest, only_headers=False

    project: Project, request: HttpRequest, only_headers=False

) -> HttpResponse:

    page_number = 1

    resumption_token = request.GET.get("resumptionToken") or request.POST.get(
@@ -143,14 +145,14 @@ def _list_records( 
    #

    from_date = request.GET.get("from") or request.POST.get("until")

    if from_date and not _validate_date(from_date):

        return _oai_error(owner, request, {"badArgument": "bad from date format"})

        return _oai_error(project, request, {"badArgument": "bad from date format"})



    #

    # Get until date

    #

    until_date = request.GET.get("until") or request.POST.get("until")

    if until_date and not _validate_date(until_date):

        return _oai_error(owner, request, {"badArgument": "bad until date format"})

        return _oai_error(project, request, {"badArgument": "bad until date format"})



    #

    # Get setSpec collection
@@ -164,7 +166,7 @@ def _list_records( 
    # Build query

    #

    all_records = Collection.objects.filter(

        is_oai_record=True, owner=owner, public_access=True

        is_oai_record=True, project=project, public_access=True

    ).order_by("created_at")

    if from_date:

        all_records = all_records.filter(created_at__gte=from_date)
@@ -180,7 +182,7 @@ def _list_records( 
    #

    total_count = all_records.count()

    if total_count == 0:

        return _oai_error(owner, request, {"noRecordsMatch": "no records match"})

        return _oai_error(project, request, {"noRecordsMatch": "no records match"})

    paginator = Paginator(all_records, PAGINATION_SIZE)

    page = paginator.page(page_number)

    if only_headers:
@@ -191,7 +193,7 @@ def _list_records( 
        render_to_string(

            xml_template,

            {

                "owner": owner,

                "project": project,

                "base_url": settings.JAMA_SITE,

                "page": page,

                "from": from_date,
@@ -207,15 +209,15 @@ def _list_records( 
    )





def _list_sets(owner: User, request: HttpRequest) -> HttpResponse:

def _list_sets(project: Project, request: HttpRequest) -> HttpResponse:

    collections = Collection.objects.filter(

        owner=owner, is_oai_record=False, public_access=True

        project=project, is_oai_record=False, public_access=True

    )

    return HttpResponse(

        render_to_string(

            "oai/list_sets.xml",

            {

                "owner": owner,

                "project": project,

                "base_url": settings.JAMA_SITE,

                "collections": collections,

            },
@@ -225,31 +227,31 @@ def _list_sets(owner: User, request: HttpRequest) -> HttpResponse: 




@csrf_exempt

def oai(request: HttpRequest, owner_id: int) -> HttpResponse:

def oai(request: HttpRequest, project_id: int) -> HttpResponse:

    try:

        owner = User.objects.get(pk=owner_id)

        project = Project.objects.get(pk=project_id)

        oai_verb = request.GET.get("verb") or request.POST.get("verb")

        if oai_verb not in OAI_VERBS:

            return _oai_error(owner, request, {"badVerb": "Illegal OAI verb"})

            return _oai_error(project, request, {"badVerb": "Illegal OAI verb"})

        metadata_prefix = request.GET.get("metadataPrefix") or request.POST.get(

            "metadataPrefix"

        )

        if metadata_prefix and metadata_prefix != "oai_dc":

            return _oai_error(

                owner, request, {"cannotDisseminateFormat": "oai_dc only"}

                project, request, {"cannotDisseminateFormat": "oai_dc only"}

            )

        if oai_verb == "GetRecord":

            return _get_record(owner, request)

            return _get_record(project, request)

        if oai_verb == "Identify":

            return _identify(owner, request)

            return _identify(project, request)

        if oai_verb == "ListIdentifiers":

            return _list_identifiers(owner, request)

            return _list_identifiers(project, request)

        if oai_verb == "ListMetadataFormats":

            return _list_metadata_formats(owner, request)

            return _list_metadata_formats(project, request)

        if oai_verb == "ListRecords":

            return _list_records(owner, request)

            return _list_records(project, request)

        if oai_verb == "ListSets":

            return _list_sets(owner, request)

    # each user/owner has his own endpoint

    except User.DoesNotExist:

            return _list_sets(project, request)

    # each project has its own endpoint

    except Project.DoesNotExist:

        raise Http404()

resources/helpers.py

+20 −19
Original line number Diff line number Diff line
@@ -181,10 +181,9 @@ def set_exif_metas(f: models.File) -> int: 
        # first delete ExifTool metas

        delete_exif_metas(f)

        # then insert new metas from extracted data

        exiftools_metas_set = models.MetadataSet.objects.filter(

        exiftools_metas_set, created = models.MetadataSet.objects.get_or_create(

            title="ExifTool", project=f.project

        ).first()

        if exiftools_metas_set:

        )

        for key in exiftools_metas:

            metadata, created = models.Metadata.objects.get_or_create(

                title=key, set=exiftools_metas_set, project=f.project
@@ -217,7 +216,7 @@ def _copy_uploaded_file_to_destination( 
    return False





def handle_local_file(file_path: str, owner: User) -> int:

def handle_local_file(file_path: str, project: models.Project) -> int:

    file_hash = _file_hash256(file_path)

    destination_file = models.hash_to_local_path(file_hash)

    os.makedirs(os.path.dirname(destination_file), exist_ok=True)
@@ -226,14 +225,14 @@ def handle_local_file(file_path: str, owner: User) -> int: 
    title, extension = os.path.splitext(os.path.basename(file_path))



    # create db file record if not already here

    file = models.File.objects.filter(hash=file_hash, owner=owner).first()

    file = models.File.objects.filter(hash=file_hash, project=project).first()

    if not file:

        file = models.File.objects.create(

            hash=file_hash,

            title=unidecode(title),

            original_name=unidecode(os.path.basename(file_path)),

            size=os.path.getsize(destination_file),

            owner=owner,

            project=project,

            file_type_id=get_file_type_from_extension(extension).id,

            post_treated=False,

        )
@@ -257,14 +256,16 @@ def handle_local_file(file_path: str, owner: User) -> int: 
    return file.id





def handle_uploaded_file(uploaded_file: File, user: User, force_file_name=None) -> int:

def handle_uploaded_file(

    uploaded_file: File, project: models.Project, force_file_name=None

) -> int:

    upload_hash = hash_upload(uploaded_file)

    destination_file = models.hash_to_local_path(upload_hash)

    _copy_uploaded_file_to_destination(uploaded_file, destination_file)

    title, extension = os.path.splitext(uploaded_file.name)



    # create db file record if not already here

    file = models.File.objects.filter(hash=upload_hash, owner=user).first()

    file = models.File.objects.filter(hash=upload_hash, project=project).first()

    if not file:

        try:

            file = models.File.objects.create(
@@ -272,7 +273,7 @@ def handle_uploaded_file(uploaded_file: File, user: User, force_file_name=None) 
                title=force_file_name or os.path.basename(title),

                original_name=force_file_name or os.path.basename(uploaded_file.name),

                size=uploaded_file.size,

                owner=user,

                project=project,

                file_type_id=get_file_type_from_extension(extension).id,

                post_treated=False,

            )

resources/management/commands/addlocalfiles.py

+13 −6
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ from typing import Iterator 
from rpc.methods import add_collection_from_path, add_resource_to_collection

from functools import lru_cache

from resources.helpers import handle_local_file

from resources.models import Project





@lru_cache(maxsize=None)
@@ -29,6 +30,7 @@ class Command(BaseCommand): 
    def add_arguments(self, parser):

        parser.add_argument("start_dir", type=str)

        parser.add_argument("user", type=str)

        parser.add_argument("project", type=int)

        parser.add_argument("--extensions", nargs="+", type=str)



    def handle(self, *args, **options):
@@ -40,19 +42,24 @@ class Command(BaseCommand): 
        if not os.path.isdir(start_dir):

            raise CommandError("{} is not a directory".format(start_dir))

        try:

            owner = User.objects.get(username=options.get("user"))

            project = Project.objects.get(pk=options.get("project"))

        except Project.DoesNotExist:

            raise CommandError(

                "({}) is not a known project".format(options.get("project"))

            )

        try:

            current_user = User.objects.get(username=options.get("user"))

        except User.DoesNotExist:

            raise CommandError("{} is not a known user".format(options.get("user")))



            raise CommandError("({}) is not a known user".format(options.get("user")))

        for file_path in scan_dir(start_dir):

            _, extension = os.path.splitext(file_path)

            if extension.lower() not in allowed_extensions:

                continue

            resource_id = handle_local_file(file_path, owner)

            resource_id = handle_local_file(file_path, project)

            collection_path = os.path.dirname(file_path)[len(start_dir) :]

            if collection_path:

                hierarchy_of_collections = cached_add_collection_from_path(

                    owner, collection_path

                    current_user, collection_path

                )

                collection_id = hierarchy_of_collections[-1]["id"]

                add_resource_to_collection(owner, resource_id, collection_id)
 
                add_resource_to_collection(current_user, resource_id, collection_id)

rpc/views.py

+21 −9
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ from django.contrib.auth import authenticate 
from django.shortcuts import render

import json

import rpc.methods as rpc_methods

from rpc.methods import ServiceException

from rpc.methods import ServiceException, _check_project_permission

from inspect import getmembers, isfunction

from resources.helpers import (

    handle_uploaded_file,
@@ -68,7 +68,7 @@ def _secure_filename(filename: str) -> str: 




def _collection_from_origin_dir(

    origin_dir: str, user: User

    origin_dir: str, project: models.Project

) -> Union[models.Collection, None]:

    # root dir always first

    previous_dir = models.Collection.objects.get(title="root", parent=None)
@@ -77,7 +77,7 @@ def _collection_from_origin_dir( 
        dir_name = dir_name.strip()

        if dir_name:

            previous_dir, created = models.Collection.objects.get_or_create(

                owner=user, title=dir_name, parent=previous_dir

                project=project, title=dir_name, parent=previous_dir

            )

    return previous_dir
@@ -120,6 +120,12 @@ def _get_user_from_request(request: HttpRequest) -> Union[User, None]: 
            return None





def _get_project_from_request(request: HttpRequest) -> Union[models.Project, None]:

    if "X-Project" in request.headers:

        return models.Project.objects.filter(pk=request.headers["X-Project"]).first()

    return None





def _get_rpc_methods() -> dict:

    methods = {}

    for fn_name, _ in getmembers(rpc_methods, isfunction):
@@ -174,7 +180,9 @@ def upload_partial(request: HttpRequest) -> HttpResponse: 
    user = _get_user_from_request(request)

    if not user:

        return HttpResponse("Forbidden", status=403)



    project = _get_project_from_request()

    if not project:

        return HttpResponse("Bad Request", status=400)

    # optional header, used to create collections

    origin_dir = request.headers.get("X-origin-dir", None)
@@ -184,14 +192,14 @@ def upload_partial(request: HttpRequest) -> HttpResponse: 
            raise ValueError

        file_name = base64.b64decode(request.headers["X-file-name"]).decode("utf-8")

        chunk_number, total_chunks = request.headers["X-file-chunk"].split("/")

        file_instance = models.File.objects.get(hash=file_hash, owner=user)

        file_instance = models.File.objects.get(hash=file_hash, project=project)

        # Rise from your grave !

        if file_instance.deleted_at:

            file_instance.deleted_at = None

            file_instance.original_name = file_name

            file_instance.save()

        if origin_dir:

            collection = _collection_from_origin_dir(origin_dir, user)

            collection = _collection_from_origin_dir(origin_dir, project)

            collection.resources.add(file_instance)

        return HttpResponse(file_instance.id, status=200)

    except (KeyError, ValueError):
@@ -229,12 +237,12 @@ def upload_partial(request: HttpRequest) -> HttpResponse: 
            try:

                with transaction.atomic():

                    file_id = handle_uploaded_file(

                        File(f), user, force_file_name=file_name

                        File(f), project, force_file_name=file_name

                    )

                    f.close()

                    _silent_rmdir(partials_dir)

                    if origin_dir:

                        collection = _collection_from_origin_dir(origin_dir, user)

                        collection = _collection_from_origin_dir(origin_dir, project)

                        collection.resources.add(models.File.objects.get(id=file_id))

                return HttpResponse(file_id, status=200)

            except UnknownFileType:
@@ -256,7 +264,11 @@ def force_download( 
    user = _get_user_from_request(request)

    if not user:

        return HttpResponse("Forbidden", status=403)

    file = models.File.objects.filter(id=file_id, owner=user).first()

    file = models.File.objects.filter(id=file_id).first()

    if not file:

        return HttpResponse("Not Found", 404)

    try:

        _check_project_permission(user, file.project, "file.download")

    except ServiceException:

        return HttpResponse("Forbidden", status=403)

    return FileResponse(open(file.local_path(), "rb"), filename=file.original_name)