Finalisation du fichier trainer_carla.ipynb

%% Cell type:markdown id: tags:

# PROJET ANNUEL Véhicule autonome et Deep Learning

## PRONOST Sacha, NEVEU Thomas, VASSE Thomas, BERNEAUD Noah

%% Cell type:code id: tags:

``` python

import flash

from flash.core.data.utils import download_data

from flash.image import SemanticSegmentation, SemanticSegmentationData

import matplotlib.pyplot as plt

import numpy as np

import torch

import PIL

import os

import torch

import torch.utils.data.dataset

import torchvision.transforms as transforms

import torchvision.datasets as datasets

from sklearn.model_selection import train_test_split

import torchmetrics

```

%% Cell type:markdown id: tags:

# Préparation des données

%% Cell type:markdown id: tags:

### Les données récupérées sont sur le [lien](https://npm3d.fr/kitti-carla) suivant:

 https://npm3d.fr/kitti-carla

 #### (Il faut les traiter à l'aide du fichier 'convert_image_ss' au préalable)

### Les anciennes données sont sur le [lien](https://github.com/ongchinkiat/LyftPerceptionChallenge/releases/download/v0.1/carla-capture-20180513A.zip) suivant:

 https://github.com/ongchinkiat/LyftPerceptionChallenge/releases/download/v0.1/carla-capture-20180513A.zip

%% Cell type:code id: tags:

``` python

# Séparation de la donnée

X = []

for filename in os.listdir('data/Town01/generated/images_rgb/'):

    X.append('data/Town01/generated/images_rgb/'+filename)

y = []

for filename in os.listdir('data/Town01/generated/image_ss_new/'):

    y.append('data/Town01/generated/image_ss_new/'+filename)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

```

%% Cell type:code id:2556a99b-1d2c-416c-b376-1a8d23aba5e6 tags:

``` python

# 1. Create the DataModule

datamodule = SemanticSegmentationData.from_files(

    train_files=X_train,

    train_targets=y_train,

    val_split=0.1,

    transform_kwargs=dict(image_size=(256, 256)),

    num_classes=23,

    batch_size=4,

)

```

%% Cell type:markdown id:28d5f668-eee1-48ee-9bd0-cae25b82dc3d tags:

# Vérification des images d'apprentissage

%% Cell type:code id:2d5c74b6-87ad-4805-9814-c86cecefae44 tags:

``` python

data = next(iter(datamodule.train_dataloader()))

```

%% Cell type:code id:fb2559a7-f7d1-4a9f-9d8d-e6b5e2d9f7c9 tags:

``` python

print(data.keys())

```

%% Cell type:code id:b5217d12-2fcc-45d9-87e6-48fbef9b0b94 tags:

``` python

im = data['input'][0]

y = data['target'][0]

```

%% Cell type:code id:bfa5eba6-188d-4ce7-aa43-fa4ad5ee61cc tags:

``` python

plt.imshow(im.numpy().transpose(1,2,0))

plt.show()

```

%% Cell type:code id:e0bcbeda-537c-4b37-9615-092722ef5998 tags:

``` python

plt.imshow(y)

plt.show()

```

%% Cell type:markdown id:34d067cc-ca61-4a1f-b0fe-9ad62c72149e tags:

# Apprentissage

%% Cell type:code id:d1a21957-ee88-43a9-8c90-f6689fe4ca33 tags:

``` python

# 2. Build the task

model = SemanticSegmentation(

    #mobilenet est fait pour de l'embarqué, fonctionne en peu de parametre dans le monde des RDN. (quelques millions)

    #Mais pas très performant

    backbone="mobilenetv3_large_100",

    #backbone="resnet50",

    #backbone="mobilenetv3_large_100",

    backbone="resnet50",

    head="fpn",

    num_classes=datamodule.num_classes,

    #pretrained=True,

)

```

%% Cell type:code id:cd9bd250-f479-4fb4-9bb0-aec18071a098 tags:

``` python

# 3. Create the trainer and finetune the model

trainer = flash.Trainer(max_epochs=3, accelerator='tpu')#torch.cuda.device_count())

#model = model.load_from_checkpoint('semantic_segmentation_model.pt')

trainer.finetune(model, datamodule=datamodule, strategy="freeze")

```

%% Cell type:code id:1a28110f tags:

``` python

# Sauvegarde du modèle

trainer.save_checkpoint("model/modelMobileNetWithTestSplit.pt")

```

%% Cell type:code id: tags:

``` python

#Si le modèle est déjà crée, l'ouvrir et le charger

model = model.load_from_checkpoint('model/modelMobileNetWithTestSplit.pt')

trainer = flash.Trainer(max_epochs=3, gpus=0)

```

%% Output

    Using 'mobilenetv3_large_100' provided by qubvel/segmentation_models.pytorch (https://github.com/qubvel/segmentation_models.pytorch).

    Using 'fpn' provided by qubvel/segmentation_models.pytorch (https://github.com/qubvel/segmentation_models.pytorch).

    GPU available: False, used: False

    TPU available: False, using: 0 TPU cores

%% Cell type:markdown id: tags:

# Teste de quelques images

%% Cell type:code id:fa8678f6-384b-483f-b8f7-015d239bc64b tags:

``` python

# 4. Segment a few images!

datamodule = SemanticSegmentationData.from_files(

    predict_files=[

        "data/Town01/generated/images_rgb/2.png"

    ],

    batch_size=3,

)

predictions = trainer.predict(model, datamodule=datamodule)

```

%% Cell type:code id:946bfd23-b059-4cf6-8c5a-32d2b3f5e966 tags:

``` python

in_im_test = predictions[0][0]['input']

out_im_test = predictions[0][0]['preds']

in_im_test=in_im_test.numpy().transpose(1,2,0)

in_im_test=(in_im_test-np.min(in_im_test))/(np.max(in_im_test)-np.min(in_im_test))

out_im_test = torch.argmax(out_im_test,0)

#plt.imshow(in_im_test[:,:,0],cmap='gray')

plt.imshow(in_im_test)

plt.imshow(out_im_test,alpha=0.3)

plt.show()

```

%% Cell type:markdown id:a39daa95-b896-4d72-a492-73661dedcce4 tags:

# Verification des labels

%% Cell type:code id:128d9043-067e-475a-b1d0-7bcbac0f7011 tags:

``` python

datamodule = SemanticSegmentationData.from_files(

    test_files=[

        "data/Town01/generated/images_rgb/2.png",

        "data/Town01/generated/images_rgb/4.png",

        "data/Town01/generated/images_rgb/6.png",

    ],

    test_targets=[

        "data/Town01/generated/image_ss_new/2.png",

        "data/Town01/generated/image_ss_new/4.png",

        "data/Town01/generated/image_ss_new/6.png",

    ],

    transform_kwargs=dict(image_size=(256, 256)),

    num_classes=23,

    batch_size=3,

)

```

%% Cell type:code id:9495e01f-a0a1-49d9-8fab-7b858dbe1ea9 tags:

``` python

data = next(iter(datamodule.test_dataloader()))

im = data['input'][0]

y = data['target'][0]

plt.imshow(im.numpy().transpose(1,2,0))

plt.show()

plt.imshow(y)

plt.show()

```

%% Cell type:markdown id: tags:

# Evaluation du modèle

%% Cell type:code id:c7db8c8f tags:

``` python

# Préparation de la donnée

X_test = X_test[:100]

y_test = y_test[:100]

print(len(X_test))

```

%% Output

    100

%% Cell type:code id:22428c13 tags:

``` python

modulePred = SemanticSegmentationData.from_files(

        predict_files=X_test,

        batch_size=1,

)

predictions = trainer.predict(model, datamodule=modulePred)

```

%% Output

    c:\Users\sacha\anaconda3\envs\deepLearningVehicule\lib\site-packages\pytorch_lightning\utilities\distributed.py:69: UserWarning: The dataloader, predict dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 12 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.

      warnings.warn(*args, **kwargs)

%% Cell type:code id: tags:

``` python

accuracy = 0

accuracyMetric = torchmetrics.Accuracy(multiclass=True,num_classes=23)

moduleTest = SemanticSegmentationData.from_files(

        test_files=X_test,

        test_targets=y_test,

        batch_size=1,

    )

toPrint = [0,10,20,30,40,50,60,70,80,90]

for i in range(100):

    prediction = torch.argmax(predictions[i][0]['preds'], 0).flatten()

    true = moduleTest.test_dataloader().dataset[i]['target'].flatten()

    #Conversion de ce qui ce trouve dans le tensor predictions et true en int

    prediction = prediction.numpy().astype(int)

    true = true.numpy().astype(int)

    #Converstion de predictions et true en tensor

    prediction = torch.from_numpy(prediction)

    true = torch.from_numpy(true)

    #print(true.flatten().shape)

    accuracy += accuracyMetric(prediction, true)

    if(i in toPrint):

        print("Accuracy actuelle: ",accuracyMetric(prediction, true))

        print("Accuracy moyenne: ",accuracy/(i+1))

print(accuracy/len(X_test))

```

%% Output

    Accuracy actuelle:  tensor(0.9234)

    Accuracy moyenne:  tensor(0.9234)

    Accuracy actuelle:  tensor(0.9080)

    Accuracy moyenne:  tensor(0.9140)

    Accuracy actuelle:  tensor(0.9097)

    Accuracy moyenne:  tensor(0.9125)

    Accuracy actuelle:  tensor(0.9055)

    Accuracy moyenne:  tensor(0.9147)

    Accuracy actuelle:  tensor(0.9032)

    Accuracy moyenne:  tensor(0.9099)

    Accuracy actuelle:  tensor(0.8659)

    Accuracy moyenne:  tensor(0.9082)

    Accuracy actuelle:  tensor(0.9141)

    Accuracy moyenne:  tensor(0.9083)

    Accuracy actuelle:  tensor(0.9466)

    Accuracy moyenne:  tensor(0.9094)

    Accuracy actuelle:  tensor(0.9140)

    Accuracy moyenne:  tensor(0.9093)

    Accuracy actuelle:  tensor(0.9300)

    Accuracy moyenne:  tensor(0.9101)

    tensor(0.9099)

%% Cell type:markdown id: tags:

Modification image detection sémantique

%% Cell type:code id: tags:

``` python

#Prend pixel en couleurs, ex:128*128*128*3 (1392*1024)

#Prendre la matrice et multiplier par un tableau

#Charger l'image avec matplotlib

im = PIL.Image.open('data3/Town01/generated/images_ss/2.png')

plt.imshow(im)

```

%% Cell type:code id: tags:

``` python

im2 = np.unique(np.sum(im*np.array([256**3,256**2,256**1,256**0])[None,None,:],axis=-1))

print(im2)

print(im)

```

%% Cell type:code id: tags:

``` python

#TODO

#Boucler sur tout les pixel de l'image im (voir avec map,flatten)

for i in range(im.shape[0]):

    for j in range(im.shape[1]):

        #Si la couleurs du pixel est (0,0,0) alors la classe est None (0)

        if im[i,j,0]==0 and im[i,j,1]==0 and im[i,j,2]==0:

            im[i,j,0]=0

            im[i,j,1]=0

            im[i,j,2]=0

        #Si la couleurs du pixel est (70,70,70) alors la classe est Building (1)

        elif im[i,j,0]==70 and im[i,j,1]==70 and im[i,j,2]==70:

            im[i,j,0]=1

            im[i,j,1]=1

            im[i,j,2]=1

#Faire de même avec map

newim = np.map(im,lambda x: 0 if x==[0,0,0] else 1 if x==[70,70,70] else 2 if x==[190,153,153] else 3 if x==[72,0,90] else 4 if x==[220,20,60] else 5 if x==[153,153,153] else 6 if x==[157,234,50] else 7 if x==[128,64,128] else 8 if x==[244,35,232] else 9 if x==[107,142,35] else 10 if x==[0,0,142] else 11 if x==[102,102,156] else 12 if x==[220,220,0] else 13 if x==[70,130,180] else 14 if x==[81,0,81] else 15 if x==[150,100,100] else 16 if x==[230,150,140] else 17 if x==[180,165,180] else 18 if x==[250,170,30] else 19 if x==[110,190,160] else 20 if x==[170,120,50] else 21 if x==[45,60,150] else 22 if x==[145,170,100] else 23 if x==[45,75,0] else 24 if x==[145,170,100] else 25 if x==[75,0,75] else 26 if x==[74,11,11] else 27 if x==[0,0,230] else 28 if x==[119,11,32] else 29 if x==[0,60,100] else 30 if x==[0,0,142] else 31 if x==[0,0,70] else 32 if x==[0,60,100] else 33 if x==[0,0,90] else 34 if x==[0,0,110] else 35 if x==[0,80,100] else 36 if x==[0,0,230] else 37 if x==[119,11,32] else 38) # Ne pas utiliser de lambda fonction car trop lente

```

%% Cell type:code id: tags:

``` python

for i in range(2,10001):

    im = PIL.Image.open('data3/Town01/generated/images_ss/' + str(i) + '.png')

    width, height = im.size

    #Créer une nouvelle image

    newim = PIL.Image.new('RGBA',im.size)

    for x in range(width):

        for y in range(height):

            rgb = im.getpixel((x, y))

            if rgb[0] == 0 and rgb[1] == 0 and rgb[2] == 0:

                newim.putpixel((x,y),(0,0,0))

            if rgb[0] == 70 and rgb[1] == 70 and rgb[2] == 70:

                newim.putpixel((x,y),(1,0,0))

            if rgb[0] == 100 and rgb[1] == 40 and rgb[2] == 40:

                newim.putpixel((x,y),(2,0,0))

            if rgb[0] == 55 and rgb[1] == 90 and rgb[2] == 80:

                newim.putpixel((x,y),(3,0,0))

            if rgb[0] == 220 and rgb[1] == 20 and rgb[2] == 60:

                newim.putpixel((x,y),(4,0,0))

            if rgb[0] == 153 and rgb[1] == 153 and rgb[2] == 153:

                newim.putpixel((x,y),(5,0,0))

            if rgb[0] == 157 and rgb[1] == 234 and rgb[2] == 50:

                newim.putpixel((x,y),(6,0,0))

            if rgb[0] == 128 and rgb[1] == 64 and rgb[2] == 128:

                newim.putpixel((x,y),(7,0,0))

            if rgb[0] == 244 and rgb[1] == 35 and rgb[2] == 232:

                newim.putpixel((x,y),(8,0,0))

            if rgb[0] == 107 and rgb[1] == 142 and rgb[2] == 35:

                newim.putpixel((x,y),(9,0,0))

            if rgb[0] == 0 and rgb[1] == 0 and rgb[2] == 142:

                newim.putpixel((x,y),(10,0,0))

            if rgb[0] == 102 and rgb[1] == 102 and rgb[2] == 156:

                newim.putpixel((x,y),(11,0,0))

            if rgb[0] == 220 and rgb[1] == 220 and rgb[2] == 0:

                newim.putpixel((x,y),(12,0,0))

            if rgb[0] == 70 and rgb[1] == 130 and rgb[2] == 180:

                newim.putpixel((x,y),(13,0,0))

            if rgb[0] == 81 and rgb[1] == 0 and rgb[2] == 81:

                newim.putpixel((x,y),(14,0,0))

            if rgb[0] == 150 and rgb[1] == 100 and rgb[2] == 100:

                newim.putpixel((x,y),(15,0,0))

            if rgb[0] == 230 and rgb[1] == 150 and rgb[2] == 140:

                newim.putpixel((x,y),(16,0,0))

            if rgb[0] == 180 and rgb[1] == 165 and rgb[2] == 180:

                newim.putpixel((x,y),(17,0,0))

            if rgb[0] == 250 and rgb[1] == 170 and rgb[2] == 30:

                newim.putpixel((x,y),(18,0,0))

            if rgb[0] == 110 and rgb[1] == 190 and rgb[2] == 160:

                newim.putpixel((x,y),(19,0,0))

            if rgb[0] == 170 and rgb[1] == 120 and rgb[2] == 50:

                newim.putpixel((x,y),(20,0,0))

            if rgb[0] == 45 and rgb[1] == 60 and rgb[2] == 150:

                newim.putpixel((x,y),(21,0,0))

            if rgb[0] == 145 and rgb[1] == 170 and rgb[2] == 100:

                newim.putpixel((x,y),(22,0,0))

    #Sauvegarder l'image

    newim.save('dataTest/' + str(i) + '.png')

```

%% Cell type:code id: tags:

``` python

def map_rgb(rgb):

    if rgb == (0,0,0):

        return (0,0,0,255)

    elif rgb == (70,70,70):

        return (1,0,0,255)

    elif rgb == (100,40,40):

        return (2,0,0,255)

    elif rgb == (55,90,80):

        return (3,0,0,255)

    elif rgb == (220,20,60):

        return (4,0,0,255)

    elif rgb == (153,153,153):

        return (5,0,0,255)

    elif rgb == (157,234,50):

        return (6,0,0,255)

    elif rgb == (128,64,128):

        return (7,0,0,255)

    elif rgb == (244,35,232):

        return (8,0,0,255)

    elif rgb == (107,142,35):

        return (9,0,0,255)

    elif rgb == (0,0,142):

        return (10,0,0,255)

    elif rgb == (102,102,156):

        return (11,0,0,255)

    elif rgb == (220,220,0):

        return (12,0,0,255)

    elif rgb == (70,130,180):

        return (13,0,0,255)

    elif rgb == (81,0,81):

        return (14,0,0,255)

    elif rgb == (150,100,100):

        return (15,0,0,255)

    elif rgb == (230,150,140):

        return (16,0,0,255)

    elif rgb == (180,165,180):

        return (17,0,0,255)

    elif rgb == (250,170,30):

        return (18,0,0,255)

    elif rgb == (110,190,160):

        return (19,0,0,255)

    elif rgb == (170,120,50):

        return (20,0,0,255)

    elif rgb == (45,60,150):

        return (21,0,0,255)

    elif rgb == (145,170,100):

        return (22,0,0,255)

    else:

        return (255,255,255,255)

```

%% Cell type:code id: tags:

``` python

im = PIL.Image.open('data3/Town01/generated/images_ss/2.png')

#Créer un dataset avec les images, et ajouter la transformation pour changer les image pil en grey, puis en tensor

#np.unique pour récupérer les occurences

#Changer les valeurs de l'occurences par les numéro de l'occurence

#Boucle for qui parcours les valeur de l'occurences

#Si la valeur de l'occurence est égale à la valeur de l'image, alors on change la valeur de l'image par le numéro de l'occurence

#En gros le faire avec dataset de pytorch

```

%% Cell type:code id: tags:

``` python

im = PIL.Image.open('data3/Town01/generated/images_ss/2.png')

im_iter = im.getdata()

new_im = map(map_rgb,im_iter)

#Récupérer image im_iter en numpy array

new_im = np.array(new_im)

#new_im = im.point(map_rgb)

new_im.save('imTest2.png')

```

%% Cell type:code id: tags:

``` python

im = PIL.Image.open('dataset_noah/All_Image/segmentation/image83.png')

width, height = im.size

for i in range(width):

    for j in range(height):

        print(im.getpixel((i,j)))

```

%% Cell type:code id: tags:

``` python

transform = transforms.Compose([

    transforms.Grayscale(),

    transforms.ToTensor()

])

```

%% Cell type:code id: tags:

``` python

train_dataset = datasets.ImageFolder('data3/Town01/generated/images_ss', transform=transform)

```

%% Cell type:markdown id: tags: