Loading distance.py 0 → 100644 +372 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 import sys import time import random import Levenshtein as leven import jaro import copy import statistics import collections import operator class Robustness: def __init__(self, pwdContainer=None): """ - pwdContainer: (String or list) - is the name of the file where passwords are stored or the list herself. """ self.l = list() self.distance = leven.distance self.algo = self.random_permutation if type(pwdContainer) == str: with open(pwdContainer) as inbuff: for word in inbuff: word = word.strip() self.l.append(word) else: self.l = pwdContainer self.k = len(self.l) def swap(self, i, j): """ Perform swap between two indice in the list. Keyword arguements: -i,j: (Integer) - swap between i and j. """ self.l[i], self.l[j] = self.l[j], self.l[i] def random_permutation(self): """ Generator for a random permutation of the list self.l using Fisher-Yates algorithm. """ d = len(self.l) for i in range(d): r = random.randint(i, d-1) self.swap(i, r) yield self.l[i] def cost_rate(self): """ Run a random_permutation and return its cost rate. Parameters are fixed using object members (l, k, algo, distance). Return values: - cost: (Integer) - Cost of the current excursion. """ before = time.time() dist = dict() c = 0 k = self.k ratio = len(self.l) / self.k perm = self.algo() w0 = next(perm) for w in perm: c += self.distance(w0, w) w0 = w if k == 0: break k -= 1 after = time.time() cost = int(c * ratio) #print("Run an excursion and its cost rate in ", after-before, "seconds") return cost def min_cost(self, n=10): """ Perfoms n times the random_permutation function, and return the one with the lowest cost. Keyword arguments: - n: (Interger) - Number of cost rate, that we are going to do. Return values: - min_c: (Interger) - Minimum cost found among the random_permutation. """ before = time.time() min_c = self.cost_rate() after = time.time() #print("First excursion run in ", after-before, "seconds") for i in range(n-1): before = time.time() min_c = min(min_c, self.cost_rate()) after = time.time() #print("Run the %dth excursion in " %i, after-before, "seconds") return min_c def closest_word(self): """ Generate a permutation where every word is the closest word to w0 (randomly choosen). Return values: - word: (String) - n th word closest to w_0. """ d = len(self.l) i0 = random.randint(0, d-1) w0 = self.l[i0] self.swap(0, i0) cost = 0 yield w0, cost for i in range(1, d): before = time.time() lowest_distance = self.distance(w0, self.l[i]) lowest_index = i for j in range(i+1, d): dist = self.distance(w0, self.l[j]) if dist < lowest_distance: lowest_distance = dist lowest_index = j cost += lowest_distance self.swap(i, lowest_index) after = time.time() word = self.l[i] #print("Found the %dth closest word in D, in %d seconds" % (i, after-before)) yield word, cost def closest_word_among(self, m=10): """ Generate a permutation where w_0 is randomly choosen and w_n+1 is the closest neighbor of w_n among n randomly choosen words. Keyword arguments: - m: (Integer) - Number of word randomly choosen to pick w_n+1. Return arguements: - (String) - wich is the closest word to w_0. """ # select the first word w0 d = len(self.l) i0 = random.randint(0, d-1) w0 = self.l[i0] self.swap(0, i0) cost = 0 yield w0, cost # select all the remaining ones for i in range(1, d): # pick the closest one among m random ones before = time.time() i_n = random.randint(i, d-1) w_n = self.l[i_n] lowest_distance = self.distance(w0, w_n) lowest_index = i_n for j in range(m-1): i_n = random.randint(1, d-1) w_n = self.l[i_n] dist = self.distance(w0, w_n) if dist < lowest_distance: lowest_distance = dist lowest_index = i_n cost += lowest_distance self.swap(i, lowest_index) after = time.time() #print("Found the %dth closest word in m, in" %i, after-before, "seconds") yield self.l[i], cost def all_distances(self): """ Computes and stores all the distances between every couple of words. """ ad = dict() for i, w in enumerate(self.l): before = time.time() for v in self.l[i:]: if w not in ad: ad[w] = dict() ad[w][v] = self.distance(w, v) after = time.time() print(w, after-before) print(ad) def avg_wordlist(self, l): """ Perfoms an average word of a word list. This function return a string (word). Keyword arguments: - l: (List of word (string)) - Contains any word. Return values: - word: (string) - The average word found for the list l. """ avg_word=[] avg_len = 0 #Average lenght of the word for i in l: avg_len += len(i) avg_len = int(avg_len/len(l)) for i in range(avg_len): letters = [] for wrd in l: if len(wrd) > i: letters.append(wrd[i]) #Add the most avg letters to the word #Between ('a', 1) and ('b', 1) he will always choose 'a', Here is a solution if needed (uncomment this two line): #This will shuffle the tuple, and choose the first one max """ letters = list(dict(collections.Counter(letters).items())) random.shuffle(letters) """ avg_word.append(max(dict(collections.Counter(letters)).items(), key=operator.itemgetter(1))[0]) word =''.join(avg_word) return word def k_means(self, k=2, max_iters=100, algo=2): """ Performs k-means clustering with Levenshtein distance. This function return the cluster assigment. This function indicate also the number of iterations to find the optimal solutions. Keyword arguments: - k: (Integer) - Number of clusters. - max_iters: (Integer) - Number of round to do. - algo: (Integer) - Algo choosen for the cost. 1: Random_permutation(). 2: Closest_word(). Return values: - cluster: (List) - All the cluster corresponding. """ #Centroids print(self.l) Centroids = random.sample(self.l, k) print("First centroids :", Centroids,"\n") #Current Average average = [[] for i in range(k)] #Converge ? ex_average = [[] for i in range(k)] for y in range(max_iters): #Random permutations perm = self.algo() #=> problèmes que se soit une permutations aléatoire a chaque nouveau tour ? cluster = [[] for i in range(k)] #For every word for w in perm: clostest_word ='' mini_dist = 30000 #Wich centroids is closest for i in Centroids: if self.distance(i,w) < mini_dist: closest_word = i mini_dist = self.distance(i,w) cluster[Centroids.index(closest_word)].append(w) #Update average for next turn ex_average = copy.deepcopy(average) #Update average (each average is a cost_rate function) for i in range(0, len(cluster)): if algo == 1: average[i] = Robustness(cluster[i]).cost_rate()/len(cluster[i]) #=> cost_rate() cout différent (presque) a chaque nouveau lancer else: closest = Robustness(cluster[i]).closest_word() #=> Closest_word() qui choisis le premier mot en random problème ? for wrd, cost in closest: avg = cost average[i] = avg/len(cluster[i]) #Converge ? If yes, then break if ex_average == average: print("Breaked at the %dth iterations" % (y)) return cluster #Update centroids (middle of each cluster) print("Cluster", cluster) for i in range (len(Centroids)): Centroids[i] = self.avg_wordlist(cluster[i]) print("Update centroids :", Centroids,"\n") print("Optimal solutions not found after %d th iterations" %(y)) return cluster, average def sampling(self, t, algo=1): """ Performs sampling algorithm, with d/t word randomly choosen. Keyword arguments: - t: (Integer) - Number (this will determine the lenght of the sample). - algo: (Integer) - Algo choosen for the cost. Algorithme choosen will start with his default values. 1: closest_word(). 2: Closest_word_among(). 3: cost_rate(). 4: min_cost(). 5 : k_means(). Return values: - cost: (Integer) - cost find for one sample, then multiply by t. """ newl = [] d = len(self.l) perm = self.algo() for i in range(int(d/t)): newl.append(next(perm)) smp = Robustness(newl) if algo == 1: clt_wrd = smp.closest_word() for i in clt_wrd: cost = i[1] elif algo == 2: clt_wrd = smp.closest_word_among() for i in clt_wrd: cost = i[1] elif algo == 3: cost = smp.cost_rate() elif algo == 4: cost = smp.min_cost() elif algo == 5: cost = 0 clt = smp.k_means()[0] avg = smp.k_means()[1] for i in range(len(clt[1])): cost += avg[i]*len(clt[i]) return cost*t if __name__ == '__main__': before = time.time() R = Robustness(sys.argv[1]) after = time.time() print("File loaded in %d seconds" % (after-before)) print("=============================================================") before = time.time() cluster = R.k_means() print(cluster) after = time.time() print("One complete random_permutation in %d seconds" % (after-before)) print("=============================================================") estimation.pdf 0 → 100644 +106 KiB File added.No diff preview for this file type. View file Loading
distance.py 0 → 100644 +372 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 import sys import time import random import Levenshtein as leven import jaro import copy import statistics import collections import operator class Robustness: def __init__(self, pwdContainer=None): """ - pwdContainer: (String or list) - is the name of the file where passwords are stored or the list herself. """ self.l = list() self.distance = leven.distance self.algo = self.random_permutation if type(pwdContainer) == str: with open(pwdContainer) as inbuff: for word in inbuff: word = word.strip() self.l.append(word) else: self.l = pwdContainer self.k = len(self.l) def swap(self, i, j): """ Perform swap between two indice in the list. Keyword arguements: -i,j: (Integer) - swap between i and j. """ self.l[i], self.l[j] = self.l[j], self.l[i] def random_permutation(self): """ Generator for a random permutation of the list self.l using Fisher-Yates algorithm. """ d = len(self.l) for i in range(d): r = random.randint(i, d-1) self.swap(i, r) yield self.l[i] def cost_rate(self): """ Run a random_permutation and return its cost rate. Parameters are fixed using object members (l, k, algo, distance). Return values: - cost: (Integer) - Cost of the current excursion. """ before = time.time() dist = dict() c = 0 k = self.k ratio = len(self.l) / self.k perm = self.algo() w0 = next(perm) for w in perm: c += self.distance(w0, w) w0 = w if k == 0: break k -= 1 after = time.time() cost = int(c * ratio) #print("Run an excursion and its cost rate in ", after-before, "seconds") return cost def min_cost(self, n=10): """ Perfoms n times the random_permutation function, and return the one with the lowest cost. Keyword arguments: - n: (Interger) - Number of cost rate, that we are going to do. Return values: - min_c: (Interger) - Minimum cost found among the random_permutation. """ before = time.time() min_c = self.cost_rate() after = time.time() #print("First excursion run in ", after-before, "seconds") for i in range(n-1): before = time.time() min_c = min(min_c, self.cost_rate()) after = time.time() #print("Run the %dth excursion in " %i, after-before, "seconds") return min_c def closest_word(self): """ Generate a permutation where every word is the closest word to w0 (randomly choosen). Return values: - word: (String) - n th word closest to w_0. """ d = len(self.l) i0 = random.randint(0, d-1) w0 = self.l[i0] self.swap(0, i0) cost = 0 yield w0, cost for i in range(1, d): before = time.time() lowest_distance = self.distance(w0, self.l[i]) lowest_index = i for j in range(i+1, d): dist = self.distance(w0, self.l[j]) if dist < lowest_distance: lowest_distance = dist lowest_index = j cost += lowest_distance self.swap(i, lowest_index) after = time.time() word = self.l[i] #print("Found the %dth closest word in D, in %d seconds" % (i, after-before)) yield word, cost def closest_word_among(self, m=10): """ Generate a permutation where w_0 is randomly choosen and w_n+1 is the closest neighbor of w_n among n randomly choosen words. Keyword arguments: - m: (Integer) - Number of word randomly choosen to pick w_n+1. Return arguements: - (String) - wich is the closest word to w_0. """ # select the first word w0 d = len(self.l) i0 = random.randint(0, d-1) w0 = self.l[i0] self.swap(0, i0) cost = 0 yield w0, cost # select all the remaining ones for i in range(1, d): # pick the closest one among m random ones before = time.time() i_n = random.randint(i, d-1) w_n = self.l[i_n] lowest_distance = self.distance(w0, w_n) lowest_index = i_n for j in range(m-1): i_n = random.randint(1, d-1) w_n = self.l[i_n] dist = self.distance(w0, w_n) if dist < lowest_distance: lowest_distance = dist lowest_index = i_n cost += lowest_distance self.swap(i, lowest_index) after = time.time() #print("Found the %dth closest word in m, in" %i, after-before, "seconds") yield self.l[i], cost def all_distances(self): """ Computes and stores all the distances between every couple of words. """ ad = dict() for i, w in enumerate(self.l): before = time.time() for v in self.l[i:]: if w not in ad: ad[w] = dict() ad[w][v] = self.distance(w, v) after = time.time() print(w, after-before) print(ad) def avg_wordlist(self, l): """ Perfoms an average word of a word list. This function return a string (word). Keyword arguments: - l: (List of word (string)) - Contains any word. Return values: - word: (string) - The average word found for the list l. """ avg_word=[] avg_len = 0 #Average lenght of the word for i in l: avg_len += len(i) avg_len = int(avg_len/len(l)) for i in range(avg_len): letters = [] for wrd in l: if len(wrd) > i: letters.append(wrd[i]) #Add the most avg letters to the word #Between ('a', 1) and ('b', 1) he will always choose 'a', Here is a solution if needed (uncomment this two line): #This will shuffle the tuple, and choose the first one max """ letters = list(dict(collections.Counter(letters).items())) random.shuffle(letters) """ avg_word.append(max(dict(collections.Counter(letters)).items(), key=operator.itemgetter(1))[0]) word =''.join(avg_word) return word def k_means(self, k=2, max_iters=100, algo=2): """ Performs k-means clustering with Levenshtein distance. This function return the cluster assigment. This function indicate also the number of iterations to find the optimal solutions. Keyword arguments: - k: (Integer) - Number of clusters. - max_iters: (Integer) - Number of round to do. - algo: (Integer) - Algo choosen for the cost. 1: Random_permutation(). 2: Closest_word(). Return values: - cluster: (List) - All the cluster corresponding. """ #Centroids print(self.l) Centroids = random.sample(self.l, k) print("First centroids :", Centroids,"\n") #Current Average average = [[] for i in range(k)] #Converge ? ex_average = [[] for i in range(k)] for y in range(max_iters): #Random permutations perm = self.algo() #=> problèmes que se soit une permutations aléatoire a chaque nouveau tour ? cluster = [[] for i in range(k)] #For every word for w in perm: clostest_word ='' mini_dist = 30000 #Wich centroids is closest for i in Centroids: if self.distance(i,w) < mini_dist: closest_word = i mini_dist = self.distance(i,w) cluster[Centroids.index(closest_word)].append(w) #Update average for next turn ex_average = copy.deepcopy(average) #Update average (each average is a cost_rate function) for i in range(0, len(cluster)): if algo == 1: average[i] = Robustness(cluster[i]).cost_rate()/len(cluster[i]) #=> cost_rate() cout différent (presque) a chaque nouveau lancer else: closest = Robustness(cluster[i]).closest_word() #=> Closest_word() qui choisis le premier mot en random problème ? for wrd, cost in closest: avg = cost average[i] = avg/len(cluster[i]) #Converge ? If yes, then break if ex_average == average: print("Breaked at the %dth iterations" % (y)) return cluster #Update centroids (middle of each cluster) print("Cluster", cluster) for i in range (len(Centroids)): Centroids[i] = self.avg_wordlist(cluster[i]) print("Update centroids :", Centroids,"\n") print("Optimal solutions not found after %d th iterations" %(y)) return cluster, average def sampling(self, t, algo=1): """ Performs sampling algorithm, with d/t word randomly choosen. Keyword arguments: - t: (Integer) - Number (this will determine the lenght of the sample). - algo: (Integer) - Algo choosen for the cost. Algorithme choosen will start with his default values. 1: closest_word(). 2: Closest_word_among(). 3: cost_rate(). 4: min_cost(). 5 : k_means(). Return values: - cost: (Integer) - cost find for one sample, then multiply by t. """ newl = [] d = len(self.l) perm = self.algo() for i in range(int(d/t)): newl.append(next(perm)) smp = Robustness(newl) if algo == 1: clt_wrd = smp.closest_word() for i in clt_wrd: cost = i[1] elif algo == 2: clt_wrd = smp.closest_word_among() for i in clt_wrd: cost = i[1] elif algo == 3: cost = smp.cost_rate() elif algo == 4: cost = smp.min_cost() elif algo == 5: cost = 0 clt = smp.k_means()[0] avg = smp.k_means()[1] for i in range(len(clt[1])): cost += avg[i]*len(clt[i]) return cost*t if __name__ == '__main__': before = time.time() R = Robustness(sys.argv[1]) after = time.time() print("File loaded in %d seconds" % (after-before)) print("=============================================================") before = time.time() cluster = R.k_means() print(cluster) after = time.time() print("One complete random_permutation in %d seconds" % (after-before)) print("=============================================================")
