Commit cab63878 authored by Mickaël Desfrênes's avatar Mickaël Desfrênes
Browse files

Merge branch 'md/upgrade-sanic' into 'master' 

Md/upgrade sanic

See merge request !1
parents 846c1982 cf4a58bd

circe/__init__.py

+58 −31
Original line number Diff line number Diff line 
import os

import sys

from time import sleep



sys.path.insert(0, os.getcwd())
@@ -164,20 +165,20 @@ STATIC_AVAILABLE_TRANSFORMATIONS_JSON = json.dumps( 
###############################

#      WEB SERVER ROUTES      #

###############################

server = Sanic(name="circe")

server = Sanic(name="circe", strict_slashes=True)

server.config.REQUEST_TIMEOUT = 60 * 30

server.config.RESPONSE_TIMEOUT = 60 * 30

server.config.KEEP_ALIVE = False





@server.route("/")

@server.get("/")

async def index(request: sanic.request):

    if CONFIG["CIRCE_ENABLE_WEB_UI"]:

        return sanic.response.redirect("/ui/")

    return sanic.response.html(STATIC_HOMEPAGE_HTML)





@server.route("/transformations/", methods=["GET"])

@server.get("/transformations/")

async def transformations(request: sanic.request):

    return sanic.response.HTTPResponse(

        STATIC_AVAILABLE_TRANSFORMATIONS_JSON,
@@ -187,7 +188,20 @@ async def transformations(request: sanic.request): 
    )





@server.route("/job/", methods=["POST"])

@server.get("/job/<job_id:str>")

async def job_get(request: sanic.request, job_id: str):

    _check_request_auth(request, job_id)

    if os.path.isfile("{}/queue/{}.tar.gz".format(CONFIG["CIRCE_WORKING_DIR"], job_id)):

        return sanic.response.HTTPResponse("Accepted", status=202)

    result_file_path = "{}/done/{}.tar.gz".format(CONFIG["CIRCE_WORKING_DIR"], job_id)

    if os.path.isfile(result_file_path):

        return await sanic.response.file(result_file_path)

        # file_stream is slow ? bad chunk size ?

        # return await sanic.response.file_stream(result_file_path, chunked=False)

    return sanic.response.HTTPResponse("Not Found", status=404)





@server.post("/job/")

async def job_post(request: sanic.request):

    _check_request_auth(request)

    uuid = uuid4()
@@ -207,22 +221,8 @@ async def job_post(request: sanic.request): 
    return sanic.response.text(uuid.hex)





@server.route("/job/<job_id>", methods=["GET"])

async def job_get(request: sanic.request, job_id: str):

    _check_request_auth(request, job_id)

    if os.path.isfile("{}/queue/{}.tar.gz".format(CONFIG["CIRCE_WORKING_DIR"], job_id)):

        return sanic.response.HTTPResponse("Accepted", status=202)

    result_file_path = "{}/done/{}.tar.gz".format(CONFIG["CIRCE_WORKING_DIR"], job_id)

    if os.path.isfile(result_file_path):

        return await sanic.response.file(result_file_path)

        # file_stream is slow ? bad chunk size ?

        # return await sanic.response.file_stream(result_file_path, chunked=False)

    return sanic.response.HTTPResponse("Not Found", status=404)





if CONFIG["CIRCE_ENABLE_WEB_UI"]:

    cookie_signer = Signer(CONFIG["CIRCE_WEB_UI_CRYPT_KEY"])

    server.static("/static/", os.path.dirname(os.path.abspath(__file__)) + "/static/")



    def _check_request_session(request: sanic.request) -> str:

        try:
@@ -233,7 +233,7 @@ if CONFIG["CIRCE_ENABLE_WEB_UI"]: 
        except (BadSignature, TypeError):

            raise Forbidden("Bad session")



    @server.route("/ui/", methods=["GET"])

    @server.get("/ui/")

    async def index(request: sanic.request):

        with open(

            os.path.dirname(os.path.abspath(__file__)) + "/static/index.html", "r"
@@ -276,7 +276,7 @@ if CONFIG["CIRCE_ENABLE_WEB_UI"]: 
        await _write_file(os.path.join(dir_to_create, dest_name), uploaded.body)

        return sanic.response.HTTPResponse(request.files.get("file").name, status=200)



    @server.route("/webui/setjob/", methods=["POST"])

    @server.post("/webui/setjob/")

    async def set_job(request: sanic.request):

        session_id = _check_request_session(request)

        session_dir = "{}web_ui_sessions/{}".format(
@@ -313,7 +313,7 @@ if CONFIG["CIRCE_ENABLE_WEB_UI"]: 
            return sanic.response.HTTPResponse("ok", status=200)

        return sanic.response.HTTPResponse("Bad Request", status=400)



    @server.route("/webui/fetchjob/", methods=["GET"])

    @server.get("/webui/fetchjob/")

    async def fetch_job(request: sanic.request):

        session_id = _check_request_session(request)

        job_id = session_id
@@ -351,11 +351,18 @@ def serve( 
    workers=CONFIG["CIRCE_WORKERS"],

    debug=CONFIG["CIRCE_DEBUG"],

    access_log=CONFIG["CIRCE_ACCESS_LOG"],

    static_dir=None,

):

    """

    Start Circe HTTP server

    """

    _check_port(host, port)

    if static_dir:

        server.static("/static", static_dir, name="static_files")

    else:

        server.static(

            "/static", os.path.dirname(os.path.abspath(__file__)) + "/static/"

        )

    try:

        server.run(

            host=host,
@@ -378,9 +385,19 @@ def run( 
    """

    Start both HTTP server and job workers

    """

    try:

        _check_port(host, port)

        static_dir = os.path.dirname(os.path.abspath(__file__)) + "/static/"

        http_process = Process(

        target=serve, args=(host, int(port), int(workers), bool(debug))

            target=serve,

            args=(

                host,

                int(port),

                int(workers),

                bool(debug),

                CONFIG["CIRCE_ACCESS_LOG"],

                static_dir,

            ),

        )

        http_process.start()

        if not CONFIG["CIRCE_IMMEDIATE_MODE"]:
@@ -388,6 +405,16 @@ def run( 
            transfo_process.start()

            transfo_process.join()

        http_process.join()

    except KeyboardInterrupt:

        try:

            transfo_process.terminate()

        except NameError:

            pass

        sleep(1)

        try:

            http_process.terminate()

        except NameError:

            pass





def remove_api_access(app_uuid: str):

circe/__main__.py

+4 −0
Original line number Diff line number Diff line
@@ -8,6 +8,10 @@ from circe import ( 
    list_transformations,

    run,

)

import sys

import os



sys.path.insert(0, os.getcwd())





def run_cli():

requirements.txt

+1 −1
Original line number Diff line number Diff line 
sanic==20.12.3

sanic

argh

requests

huey

setup.py

+2 −2
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh: 


setuptools.setup(

    name="circe-CERTIC",

    version="0.0.22",

    version="0.0.24",

    author="Mickaël Desfrênes",

    author_email="mickael.desfrenes@unicaen.fr",

    description="Circe server",
@@ -21,7 +21,7 @@ setuptools.setup( 
    install_requires=[

        "requests",

        "argh",

        "sanic==20.12.3",

        "sanic",

        "huey",

        "asyncio",

        "itsdangerous",

test.py

+5 −5
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ def test_blocking_call_to_post_job(): 
    job = client.new_job()

    job.add_file("test_assets/index.html")

    job.add_file("test_assets/style.css")

    job.add_transformation("html2pdf")

    job.add_transformation("sleep_well", {"time": 1})

    client.send(job, wait=True, destination_file="test_assets/result.tar.gz")

    assert job.result_file_path is not None
@@ -47,7 +47,7 @@ def test_non_blocking_call_to_post_job(): 
    global client

    job = client.new_job()

    job.add_file("test_assets/doc.docx")

    job.add_transformation("docx2markdown")

    job.add_transformation("sleep_well", {"time": 1})

    client.send(job)

    assert job.uuid is not None
@@ -56,7 +56,7 @@ def test_polling(): 
    global client

    job = client.new_job()

    job.add_file("test_assets/doc.docx")

    job.add_transformation("donothing")

    job.add_transformation("sleep_well", {"time": 5})

    client.send(job)

    client.poll(job, "test_assets/result.tar.gz")

    assert os.path.isfile("test_assets/result.tar.gz")
@@ -66,7 +66,7 @@ def test_log_presence(): 
    global client

    job = client.new_job()

    job.add_file("test_assets/doc.docx")

    job.add_transformation("donothing")

    job.add_transformation("sleep_well", {"time": 1})

    client.send(job, wait=True)

    file_names = []

    for file_name, _ in job.result.files:
@@ -85,6 +85,6 @@ def test_bad_auth(): 
        )

        job = bad_client.new_job()

        job.add_file("test_assets/doc.docx")

        job.add_transformation("donothing")

        job.add_transformation("sleep_well", {"time": 5})

        with pytest.raises(circe_client.AccessDenied) as e:

            bad_client.send(job)