Commit 75429828 authored by David Guo's avatar David Guo
Browse files

feat : not using cleaning module (unused useless_tag in the vision_based file)

parent 394ea115

tagthunder/api/routers/services.py

+1 −0
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ class AlgorithmServices: 
    def augmentation(cls, url: [HttpUrl], algorithm_name: str, parameters) -> Optional[schemas.HTMLP]:

        algorithm = cls.get_algorithm("augmentation", algorithm_name, **parameters)

        htmlp = algorithm(url)

        print(htmlp)

        return factories.Responses.HTMLP(htmlp)



    @classmethod

tagthunder/pipeline/blocks/cleaning/vision_based.py

+3 −2
Original line number Diff line number Diff line
@@ -6,8 +6,9 @@ from pipeline.models.web_elements import HTMLPTag 




class VisionBased(AbstractCleaningBlock):

    USELESS_TAGS = ["title", "link", "script", "noscript", "style", "doctype", "head", "base", "command",

                    "meta", "br"]

    #USELESS_TAGS = ["title", "link", "script", "noscript", "style", "doctype", "head", "base", "command", "meta", "br"]



    USELESS_TAGS = []



    def config(self, **kwargs):

        pass