Commit 793840ce authored by Francois Ledoyen's avatar Francois Ledoyen
Browse files

Add abstract pipeline block

parent 3975e7bc

tagthunder/pipeline/blocks/_abstract.py

0 → 100644
+8 −0
Original line number Diff line number Diff line 
import abc





class AbstractPipelineBlock(abc.ABC):



    @abc.abstractmethod

    def config(self, **kwargs):

        ...
 
 No newline at end of file

tagthunder/pipeline/blocks/augmentation/_abstract.py

+12 −3
Original line number Diff line number Diff line 
from abc import ABC, abstractmethod

from typing import Optional

from typing import Optional, List



from pydantic import HttpUrl



from pipeline.models.responses import HTML, HTMLP

from pipeline.blocks._abstract import AbstractPipelineBlock





class AbstractAugmentationBlock(ABC):

class AbstractAugmentationBlock(AbstractPipelineBlock):



    def __init__(self):

        self.styles = None



    def config(self, styles: Optional[List[str]] = None):

        if styles is None:

            styles = []

        self.styles = styles



    @abstractmethod

    def __call__(self, *, html: HTML, url: HttpUrl, **kwargs) -> HTMLP:

    def __call__(self, html: HTML, url: HttpUrl, **kwargs) -> HTMLP:

        ...

tagthunder/pipeline/blocks/augmentation/puppeteer.py

+12 −9
Original line number Diff line number Diff line 
from typing import Optional, List

from typing import Optional, List, Tuple



import pydantic

import requests
@@ -10,14 +10,17 @@ from pipeline.models.responses import HTMLP 


class Puppeteer(AbstractAugmentationBlock):

    def __init__(self, crawler_address):

        self.crawler_address: HttpUrl = crawler_address

        self.requester = PuppeteerCrawlerWrapper(crawler_address=self.crawler_address)



    def __call__(self, url: HttpUrl, page_width: int = 1200, page_height: int = 1200,

                 styles: Optional[List[str]] = None) -> HTMLP:

        if styles is None:

            styles = []

        response = self.requester(url=url, width=page_width, height=page_height, styles=styles)

        super(Puppeteer, self).__init__()

        self.page_height = None

        self.page_width = None

        self.requester = PuppeteerCrawlerWrapper(crawler_address=crawler_address)



    def config(self, page_width: int = 1200, page_height: int = 1200, styles: Optional[List[str]] = None):

        super(Puppeteer, self).config(styles)

        self.page_width, self.page_height = page_width, page_height



    def __call__(self, url: HttpUrl, **kwargs) -> HTMLP:

        response = self.requester(url=url, width=self.page_width, height=self.page_height, styles=self.styles)

        return HTMLP(markup=response.content)

tagthunder/pipeline/blocks/cleaning/_abstract.py

+3 −1
Original line number Diff line number Diff line
@@ -5,8 +5,10 @@ import bs4 
from pipeline.models.responses import HTMLPP, HTMLP

from pipeline.models.web_elements import HTMLPTag, HTMLPPTag



from pipeline.blocks._abstract import AbstractPipelineBlock



class AbstractCleaningBlock(ABC):



class AbstractCleaningBlock(AbstractPipelineBlock):

    NON_CONTAINER_TAGS = ["img"]



    @abstractmethod

tagthunder/pipeline/blocks/cleaning/vision_based.py

+3 −0
Original line number Diff line number Diff line
@@ -9,6 +9,9 @@ class VisionBased(AbstractCleaningBlock): 
    USELESS_TAGS = ["title", "link", "script", "noscript", "style", "doctype", "head", "base", "command",

                    "meta", "br"]



    def config(self, **kwargs):

        pass



    def __call__(self, htmlp: HTMLP) -> HTMLPP:

        self.remove_comments(htmlp)

        for tag in list(htmlp())[::-1]: