Progressing on lemmatisation script documentation.

%% Cell type:markdown id: tags:

# Adding semi-automatic @xml:id to structured TEI-XML files

Script constructing identifiers for TEI-XML divs, according to the ConDÉ project schema, all values separated by `-`, all body div numbers formatted with three digits and all front|back div numbers formatted with two digits.

For `//tei:text/tei:front` and `//tei:text/tei:back` divs, the construction is as follows:

* source id,

* type of edition (base/txm/simplified),

* current version (alpha/beta)

* frontMatter or backMatter

* number of current front/div or back/div

* subtype of current front/div if any,

* number of current front/div/div if subject div is inside a div itself (max 2 levels of div in front and back).

Example: `basnage-base-beta-frontMatter-01-titlePage`.

For `//tei:text/tei:body` divs, construction is as follows:

* source id,

* type of edition (base/txm/simplified),

* current version (alpha/beta)

* current part div number,

* current chapter div number (if subject div is a chapter or section)

* current section div number (if subject div is a section).

Example: `basnage-base-beta-002-005-036`.

All body divs need to be typed (`part`/`chapter`/`section`) for this script to function.

### Imports and declarations

%% Cell type:code id: tags:

``` python

import xml.etree.ElementTree as ET

ET.register_namespace("", "http://www.tei-c.org/ns/1.0")

ET.register_namespace('xml','http://www.w3.org/XML/1998/namespace')

```

%% Cell type:markdown id: tags:

### FUNCTION: contains all the actual code of this file

%% Cell type:code id: tags:

``` python

def add_ids(xml_in, xml_out, fileID):

    """

    Function taking one TEI-XML file and adding @xml:id to <tei:div> elements.

    :param xml_in: The local path to the TEI-XML file needing <tei:div> identification, as a string.

    :param xml_out: The local path to the new TEI-XML file with identified <tei:div>, as a string.

    :param fileID: A string to prefix all identifiers, at best with only small letters inside.

                    Meant to use the corpus identifier of the current source.

    """

    # Starting a new file, we make counters for each type of div.

    section_counter = 0

    chapter_counter = 0

    part_counter = 0

    front_counter = 0

    back_counter = 0

    # Open and parse current TEI-XML file.

    tree = ET.parse(xml_in)

    root = tree.getroot()

    # Add the specified source ID to the <tei:text> element.

    textElement = root.find('.//{http://www.tei-c.org/ns/1.0}text')

    textElement.set('{http://www.w3.org/XML/1998/namespace}id', fileID)

    # START WITH THE FRONT MATTER DIVs.

    for item in root.findall(".//{http://www.tei-c.org/ns/1.0}front/*"):

    #for item in root.findall(".//front/*"):

        # Found one more div: add 1 to counter.

        front_counter += 1

        # If current element is a <tei:titlePage> element,

        # this will be included in its identifier.

        if item.tag == "{http://www.tei-c.org/ns/1.0}titlePage":

        #if item.tag == "titlePage":

            frontID = fileID + "-frontMatter-" + str("{:02}".format(front_counter)) + "-titlepage"

            item.set("{http://www.w3.org/XML/1998/namespace}id", frontID)

        # If current element is not a <tei:titlePage> element,

        # and it has an @type, this will be included in its identifier.

        elif item.get("type"):

            frontID = fileID + "-frontMatter-" + str("{:02}".format(front_counter)) + "-" + item.get("type")

            item.set("{http://www.w3.org/XML/1998/namespace}id", frontID)

        # If current element is not a <tei:titlePage> element,

        # and it has no @type, its identifier will only have its number.

        else:

            frontID = fileID + "-frontMatter-" + str("{:02}".format(front_counter))

            item.set("{http://www.w3.org/XML/1998/namespace}id", frontID)

    # THEN DO THE BACK MATTER DIVs.

    for item in root.findall(".//{http://www.tei-c.org/ns/1.0}back/*"):

    #for item in root.findall(".//back/*"):

        # Found one more div: add 1 to counter.

        back_counter += 1

        # If current element has an @type, this will be included in its identifier.

        if item.get("subtype"):

            backID = fileID + "-backMatter-" + str("{:02}".format(back_counter)) + "-" + item.get("subtype")

            item.set("{http://www.w3.org/XML/1998/namespace}id", backID)

        # Otherwise, we will use its number only.

        else:

            backID = fileID + "-backMatter-" + str("{:02}".format(back_counter))

            item.set("{http://www.w3.org/XML/1998/namespace}id", backID)

    # FINALLY, DO THE MAIN CONTENT DIVs.

    #for part in root.findall(".//{http://www.tei-c.org/ns/1.0}div[@type='part']"):

    for part in root.findall(".//{http://www.tei-c.org/ns/1.0}body/{http://www.tei-c.org/ns/1.0}div[@type='part']"):

        # When entering a new part div, start chapter numbers anew, add 1

        # to part counter and make the div id.

        chapter_counter = 0

        part_counter += 1

        part_identifier = fileID + "-" + str("{:03}".format(part_counter))

        # If there is no @xml:id, make it. Otherwise, replace it.

        if part.get("{http://www.w3.org/XML/1998/namespace}id"):

            del part.attrib['{http://www.w3.org/XML/1998/namespace}id']

        part.set("{http://www.w3.org/XML/1998/namespace}id", part_identifier)

        #for chapter in part.findall(".//{http://www.tei-c.org/ns/1.0}div[@type='chapter']"):

        for chapter in part.findall(".//{http://www.tei-c.org/ns/1.0}div[@type='chapter']"):

            # When entering a new chapter div, start section numbers anew, add 1

            # to chapter counter and make the div id.

            section_counter = 0

            chapter_counter += 1

            chapter_identifier = fileID + "-" + str("{:03}".format(part_counter)) + "-" + str("{:03}".format(chapter_counter))

            # If there is no @xml:id, make it. Otherwise, replace it.

            if chapter.get("{http://www.w3.org/XML/1998/namespace}id"):

                del chapter.attrib["{http://www.w3.org/XML/1998/namespace}id"]

            chapter.set("{http://www.w3.org/XML/1998/namespace}id", chapter_identifier)

            #for section in chapter.findall(".//{http://www.tei-c.org/ns/1.0}div[@type='section']"):

            for section in chapter.findall(".//{http://www.tei-c.org/ns/1.0}div[@type='section']"):

                section_counter += 1

                section_identifier = fileID + "-" + str("{:03}".format(part_counter)) + "-" + str("{:03}".format(chapter_counter)) + "-" + str("{:03}".format(section_counter))

                if section.get("{http://www.w3.org/XML/1998/namespace}id"):

                    del section.attrib["{http://www.w3.org/XML/1998/namespace}id"]

                section.set("{http://www.w3.org/XML/1998/namespace}id", section_identifier)

    # Write final TEI-XML file into specified output path.

    tree.write(xml_out, encoding="unicode")

```

%% Cell type:markdown id: tags:

### Apply the function to desired files.

Here is where you put Python to work and change settings with the function parameters:

* a string containing the path of the input file,

* a string containing the path for an output file,

* a prefix for all `<tei:div>` identifiers in the file.

%% Cell type:code id: tags:

``` python

add_ids(

    "/home/erminea/Documents/CONDE/nov-21_renum/terrien_base.xml",

    "/home/erminea/Documents/CONDE/nov-21_divID/terrien_base.xml",

    "terrien-base-beta"

)

```

%% Cell type:markdown id: tags:

# Extract ambiguous tokens into CSV file

From one TEI-XML file, this script extracts all tokens whose linguistic enrichment is unsure, whether because the lemmatiser gave several possible options, or because it could not give one.

This script has possible issues which need to be examined and dealt with eventually. They include these facts:

* the XML information is parsed *without* namespace information (so the TEI namespace declaration needs to be removed from the XML file for the script to work),

* the `get_text()` function may be obsolete,

* the `get_text()` function has two versions, one is commented,

* documentation is currently in French and old.

### IMPORTS and declarations

%% Cell type:code id: tags:

``` python

import xml.etree.ElementTree as ET

import csv

import datetime

```

%% Cell type:markdown id: tags:

### FUNCTION: extract the text from a given token

%% Cell type:code id: tags:

``` python

def get_text(token):

    texte = ""

    choice = ["corr", "expan", "reg"]

    w_token = ET.fromstring(token)

    if w_token.text:

        texte += w_token.text

    """for item in w_token.findall("./*"):

        # if item.tag == '{http://tei-c.org/ns/1.0}height' or item.tag == '{http://tei-c.org/ns/1.0}supplied':

        if item.tag == 'c' or item.tag == 'supplied':

            texte += str(item.text)

                        # S'il y a du texte après la balise fermante et avant

                        # le prochain enfant ou la balise fermante du <w>,

                        # on l'ajoute.

            if item.tail:

                texte += str(item.tail)

                    # elif item.tag == '{http://tei-c.org/ns/1.0}lb':

        elif item.tag == 'lb':

            if item.tail:

                texte += str(item.tail)

                    # Si l'enfant est un <choice>, on récupère le texte de son

                    # second enfant et on vérifie s'il y a du texte après le <choice>.

                    # elif item.tag == '{http://tei-c.org/ns/1.0}choice':

        elif item.tag == 'choice':

            for subitem in item:

                if subitem.tag in choice:

                    texte += str(subitem.text)

            if item.tail:

                texte += str(item.tail)

        elif item.tag == 'add':

                        # On refait tous les tests.

            if item.find('.') == None :

                texte = str(item.text)

            else:

                if item.text:

                    texte += str(item.text)

                for subitem in item:

                    if subitem.tag == 'lb':

                        if subitem.tail:

                            texte += str(subitem.tail)

                    elif subitem.tag == 'choice':

                        texte += str(subitem[1].text)

                        if subitem.tail:

                            texte += str(subitem.tail)

                            """

    for item in w_token:

        # Si l'enfant est un <height>, on récupère son texte.

        if item.tag == 'height' or item.tag == 'supplied':

            texte += str(item.text)

            # S'il y a du texte après la balise fermante et avant

            # le prochain enfant ou la balise fermante du <w>,

            # on l'ajoute.

            if item.tail:

                texte += str(item.tail)

        elif item.tag == 'lb':

            if item.tail:

                texte += str(item.tail)

                    # Si l'enfant est un <choice>, on récupère le texte de son

                    # second enfant et on vérifie s'il y a du texte après le <choice>.

        elif item.tag == 'choice':

            texte += str(item[1].text)

            if item.tail:

                texte += str(item.tail)

        elif item.tag == 'c':

            texte += item.text

            if item.tail:

                texte += str(item.tail)

        elif item.tag == 'add':

            # On refait tous les tests.

            if item.find('.') == None :

                texte = str(item.text)

            else:

                if item.text:

                    texte += str(item.text)

                for subitem in item:

                    if subitem.tag == 'lb':

                        if subitem.tail:

                            texte += str(subitem.tail)

                    elif subitem.tag == 'choice':

                        texte += str(subitem[1].text)

                        if subitem.tail:

                            texte += str(subitem.tail)

    return texte

```

%% Cell type:markdown id: tags:

### FUNCTION: Extract wanted data.

The main body of the script.

%% Cell type:code id: tags:

``` python

def extraction(xml_entree, csv_simple, csv_concordancier, txt_stats):

    dico_tokens={}

    # colonnes des CSV:

    simple_cols = ["ID", "TOKEN", "LEMMES", "POS"]

    concord_cols = ["ID", "POS", "GAUCHE", "TOKEN", "DROIT"]

    # compteurs

    nb_total_tokens = 0

    pos_ambigus = 0

    pos_uniques = 0

    pos_inc = 0

    lemmes_ambigus = 0

    lemmes_uniques = 0

    lemmes_inc = 0

    # Pour que Python comprenne les éléments dont on parlera,

    # il faut lui donner la déclaration TEI, mais comme c'est

    # la seule qu'on utilisera, pas besoin de lui donner un préfixe.

    # ET.register_namespace('', "http://tei-c.org/ns/1.0")

    # On va chercher le fichier XML-TEI et on le lit.

    tree = ET.parse(xml_entree)

    root = tree.getroot()

    for word in root.findall('.//w'):

        dico_tokens[int(word.get('n'))] = get_text(ET.tostring(word))

    # On ouvre le CSV de sortie en mode "écriture", on y écrit le nom des colonnes.

    with open(csv_simple, 'w') as csv_file:

        csv_contenu = csv.DictWriter(csv_file, fieldnames = simple_cols, delimiter=";")

        csv_contenu.writeheader()

        # On boucle sur les éléments <w> du XML, dans l'ordre du fichier.

        # for word in root.findall('.//{http://tei-c.org/ns/1.0}w'):

        for word in root.findall('.//w'):

            nb_total_tokens += 1

            # On récupère les @n, @lemma et @pos dans les variables

            # "numero", "lemmes" et "pos"

            # et on crée la chaîne "texte", pour l'instant vide.

            numero = str(word.get('n'))

            lemmes = str(word.get('lemma'))

            pos = str(word.get('pos'))

            texte = get_text(ET.tostring(word))

            if '|' in lemmes:

                lemmes_ambigus += 1

                if '|' in pos:

                    pos_ambigus += 1

                elif pos=="Inconnu":

                    pos_inc += 1

                csv_contenu.writerow(

                    {

                        "ID":numero,

                        "TOKEN":texte,

                        "LEMMES":lemmes,

                        "POS":pos

                    }

                )

            elif lemmes=="INC":

                lemmes_inc += 1

                if '|' in pos:

                    pos_ambigus += 1

                elif pos=="Inconnu":

                    pos_inc += 1

                csv_contenu.writerow(

                    {

                        "ID":numero,

                        "TOKEN":texte,

                        "LEMMES":lemmes,

                        "POS":pos

                    }

                )

            else:

                lemmes_uniques += 1

    """with open(csv_concordancier, 'w') as csv_file:

        csv_contenu = csv.DictWriter(csv_file, fieldnames = concord_cols, delimiter=";")

        csv_contenu.writeheader()

        # On boucle sur les éléments <w> du XML, dans l'ordre du fichier.

        # for word in root.findall('.//{http://tei-c.org/ns/1.0}w'):

        for word in root.findall('.//w'):

            # On récupère les @n, @lemma et @pos dans les variables

            # "numero", "lemmes" et "pos"

            # et on crée la chaîne "texte", pour l'instant vide.

            numero = str(word.get('n'))

            lemmes = str(word.get('lemma'))

            pos = str(word.get('pos'))

            texte = get_text(ET.tostring(word))

            ["ID", "POSG", "GAUCHE", "TOKEN", "DROIT", "POSD"]

            if '|' in lemmes or '|' in pos or lemmes=="INC" or pos=="Inconnu":

                gauche = [dico_tokens[int(numero)-3], dico_tokens[int(numero)-2], dico_tokens[int(numero)-1]]

                droit = [dico_tokens[int(numero)+1], dico_tokens[int(numero)+2], dico_tokens[int(numero)+3]]

                csv_contenu.writerow(

                    {

                        "ID":numero,

                        "POS":pos,

                        "GAUCHE": " ".join(gauche),

                        "TOKEN": texte,

                        "DROITE":" ".join(droit)

                    }

                )"""

    pourcentage_lemmes = lemmes_uniques * 100 / nb_total_tokens

    pourcentage_pos = pos_uniques * 100 / nb_total_tokens

    pourcentage_lemmes_inc = lemmes_inc * 100 / nb_total_tokens

    pourcentage_extraits = pos_ambigus * 100 / nb_total_tokens

    with open(txt_stats, "w") as file:

        file.write(str(datetime.datetime.now()))

        file.write(round(pourcentage_lemmes,2), "% de lemmes uniques.")

        file.write(round(pourcentage_pos,2), "% de POS uniques.")

        file.write(lemmes_inc, "lemmes inconnus, soit", round(pourcentage_lemmes_inc,2), "%.")

        file.write(str(round(pourcentage_lemmes,2)) + "% de lemmes uniques.")

        file.write(str(round(pourcentage_pos,2)) + "% de POS uniques.")

        file.write(str(lemmes_inc) + "lemmes inconnus, soit" + str(round(pourcentage_lemmes_inc,2)) + "%.")

```

%% Cell type:markdown id: tags:

### Treating one file with the script

The `extraction()` function requires four parameters:

* the local path to one lemmatized TEI-XML file,

* the local output path for the CSV table containing the problematic tokens,

* the local output path for the CSV table containing the problematic tokens *as a concordance table*,

* the local output path for the TXT file containing the number and percentage of tokens extracted into the previous tables.

%% Cell type:code id: tags:

``` python

extraction('/home/erminea/Documents/CONDE/Rouille-TS/Rouille_19-lemmatise_div-ided.xml',

          '/home/erminea/Documents/CONDE/Rouille-TS/rouille_ambig_tableau.csv',

          '/home/erminea/Documents/CONDE/Rouille-TS/rouille_ambig_concord.csv',

          '/home/erminea/Documents/CONDE/Rouille-TS/stats.txt')

extraction('/local/path/to/Rouille_19-lemmatise_div-ided.xml',

          '/local/path/to/rouille_ambig_tableau.csv',

          '/local/path/to/rouille_ambig_concord.csv',

          '/local/path/to/stats.txt')

```

%% Output

    ---------------------------------------------------------------------------

    TypeError                                 Traceback (most recent call last)

    <ipython-input-27-2eec5b523fc6> in <module>

    ----> 1 extraction('/home/erminea/Documents/CONDE/Rouille-TS/Rouille_19-lemmatise_div-ided.xml',

          2           '/home/erminea/Documents/CONDE/Rouille-TS/rouille_ambig_tableau.csv',

          3           '/home/erminea/Documents/CONDE/Rouille-TS/rouille_ambig_concord.csv',

          4           '/home/erminea/Documents/CONDE/Rouille-TS/stats.txt')

    <ipython-input-26-92e1d0861357> in extraction(xml_entree, csv_simple, csv_concordancier, txt_stats)

        125     with open(txt_stats, "w") as file:

        126         file.write(str(datetime.datetime.now()))

    --> 127         file.write(round(pourcentage_lemmes,2), "% de lemmes uniques.")

        128         file.write(round(pourcentage_pos,2), "% de POS uniques.")

        129         file.write(lemmes_inc, "lemmes inconnus, soit", round(pourcentage_lemmes_inc,2), "%.")

    TypeError: write() takes exactly one argument (2 given)

%% Cell type:markdown id: tags:

# Re-numbering tokens after revisions

* __Note__: This script is similar to [this one](../lemmatize-new-witness/NV_1_numerotation_tokens.ipynb), which is rather meant for a file whose `<w>` still have no `att.n` attributes.

* __Note__: due to the fact that on GitHub the `@` sign is used to tag users, it is replaced by `att.` in XPath expressions.

This script takes a valid TEI-XML file, tokenised with `//tei:w[att.n]` elements. Its only function will remove the current `att.n` when they are there, and give a new unique number to each `<w>` element, within an `att.n` attribute, in the reading order.

### FUNCTION: give each `<w>` element a (new?) number

%% Cell type:code id: tags:

``` python

def id_tokens_in_tei(chemin_entree, chemin_sortie):

    """

    Fonction permettant de lire un fichier XML-TEI pour cibler les

    éléments <w> et leur ajouter un @n unique, numéroté à partir de 1.

    Si on souhaite utiliser des entités, elles sont résolues dans le

    fichier de sortie, mieux vaut donc les installer ensuite.

    :param chemin_entree: Le chemin local du fichier XML-TEI tokenisé

        aux éléments <w> duquel on souhaite ajouter des numéros.

    :param chemin_sortie: Le chemin local auquel on souhaite écrire le

        fichier XML-TEI de sortie avec ses @n ajoutés.

    This function takes a valid TEI-XML file as input.

    It targets all <w> elements and gives them a unique

    @n attribute, numbered from 1, removing the previous

    one if needed. The result is a valid TEI-XML file.

    :param chemin_entree: The local path to the tokenized

        TEI-XML file whose <w> elements need to be numbered.

    :param chemin_sortie: The local path for the output file.

    """

    import xml.etree.ElementTree as ET

    ET.register_namespace('', 'http://www.tei-c.org/ns/1.0')

    # On crée un compteur pour les numéros des tokens.

    # Create a counter.

    counter = 1

    # On donne au module XML le namespace de la TEI, sans préfixe car ce sera le seul.

    # ET.register_namespace('', "http://tei-c.org/ns/1.0")

    # Declare the TEI namespace, without a prefix since it is the only one.

    ET.register_namespace('', "http://tei-c.org/ns/1.0")

    # On importe le XML-TEI d'entrée et on le lit.

    # Import and parse the input XML file.

    tree = ET.parse(chemin_entree)

    root = tree.getroot()

    # On boucle sur les éléments <w> dans l'ordre du fichier.

    # Loop on <w> elements in reading order.

    for word in root.findall('.//{http://www.tei-c.org/ns/1.0}w'):

    # for word in root.findall('.//w'):

        # Si l'élément <w> a déjà un numéro, on l'enlève pour le remplacer.

        # If the <w> element already has an @n attribute, remove it

        # so we can replace it.

        if word.get('n'):

            del word.attrib['n']

        # On crée un attribut "n" avec, comme valeur, l'état actuel du compteur.

        # Make an @n attribute with the current state of the counter as value.

        word.set('n', str(counter))

        # On ajoute 1 au compteur pour le prochain <w>.

        # Add 1 to the counter for the next <w> element.

        counter += 1

    # On écrit le TEI obtenu dans le fichier spécifié en second paramètre.

    # Write the output file at the path specified as second argument.

    tree.write(chemin_sortie, xml_declaration=True, encoding="unicode")

```

%% Cell type:code id: tags:

``` python

# Pour exécuter la fonction, on remplace les deux chemins par ceux souhaités.

id_tokens_in_tei(

    '/home/erminea/Documents/CONDE/editions/base-version/gc_base.xml',

    '/home/erminea/Documents/CONDE/nov-21_renum/gc_base.xml'

    )