Commit 394ea115 authored by David Guo's avatar David Guo
Browse files

feat : extract paths from htmlpp

parent a85746f0

package-lock.json

0 → 100644
+6 −0
Original line number Diff line number Diff line 
{

  "name": "tagthunder",

  "lockfileVersion": 2,

  "requires": true,

  "packages": {}

}

tagthunder/api/models/factories.py

+12 −3
Original line number Diff line number Diff line 
import pipeline.models.responses as algos_responses

import api.models.schemas as schemas

import re





class AlgorithmInput:
@@ -24,7 +25,8 @@ class AlgorithmInput: 
        return algos_responses.Zone(

            id=zone.id,

            htmlpp=cls.HTMLPP(zone.htmlpp),

            keywords=[cls.Keyword(kw) for kw in zone.keywords]

            keywords=[cls.Keyword(kw) for kw in zone.keywords],

            xpath = []

        )



    @classmethod
@@ -64,12 +66,19 @@ class Responses: 


    @classmethod

    def Zone(cls, zone: algos_responses.Zone) -> schemas.Zone:

        extracted_xpaths = cls.extract_xpath_from_htmlpp(str(cls.HTMLPP(zone.htmlpp)))



        return schemas.Zone(

            id=zone.id,

            htmlpp= cls.HTMLPP(zone.htmlpp),

            keywords=cls.Keywords(zone.keywords)

            keywords=cls.Keywords(zone.keywords),

            xpath = extracted_xpaths

        )



    def extract_xpath_from_htmlpp(htmlpp :str) :

        xpath_pattern = r'xpath="([^"]+)"'

        return re.findall(xpath_pattern, htmlpp)



    @classmethod

    def Segmentation(cls, segmentation: algos_responses.Segmentation) -> schemas.Segmentation:

        return schemas.Segmentation(

tagthunder/api/models/schemas.py

+1 −0
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ class Zone(BaseModel): 
    id: int

    htmlpp: Union[List[HTMLPP], HTMLPP]

    keywords: Optional[Keywords] = []

    xpath : List[str] = []



    class Config:

        arbitrary_types_allowed = True

tagthunder/pipeline/models/responses.py

+6 −0
Original line number Diff line number Diff line
@@ -25,6 +25,10 @@ class HTMLP(HTML): 
    def bbox(self):

        return self.body.bbox



    @property

    def xpath(self):

        return self.body.xpath



    def get_leafs(self):

        return [node for node in self.find_all() if node.find() is None]
@@ -70,6 +74,8 @@ class Zone(BaseModel): 
    id: int

    htmlpp: Union[List[HTMLPP], HTMLPP]

    keywords: Optional[Keywords] = []

    #xpath : str





    class Config:

        arbitrary_types_allowed = True