Implements OpenMP multi-threading. Implements #3.

CMakeLists.txt

 # Version de cmake demandée.
 cmake_minimum_required(VERSION 2.8)
 
-find_package(Threads REQUIRED)
+find_package(OpenMP)
 
 # Chemin des répertoires contenant les fichiers entêtes.
 include_directories(include)
@@ -10,8 +10,7 @@ include_directories(include)
 set(EXECUTABLE_OUTPUT_PATH bin/${CMAKE_BUILD_TYPE})
 
 # Option du compilateur pour supporter C 2011.
-set(CMAKE_CXX_FLAGS "-pedantic -Wall -std=c++11 -Wextra -O3")
-set(CMAKE_LD_FLAGS "-lpthread")
+set(CMAKE_CXX_FLAGS "-pedantic -Wall -std=c++11 -Wextra -O3 -fopenmp")
 
 add_executable(cppack
 				src/core/Statsgen.cpp

include/MainWindow.h

@@ -26,21 +26,14 @@ class ProgressThread : public QThread
 {
     Q_OBJECT
     void run() {
-        unsigned int progress = 0;
-        uint64_t processed = 0;
-        const std::vector<ThreadData>& td = _s.getThreadsData();
-
+        int progress = 0;
         while(!_s.allStarted()){
             msleep(500);
         }
-        const uint64_t nblines = td.back().lineEnd;
+        const uint64_t nblines = _s.getNbLines();
 
         while(!_s.allFinished()){
-            processed = 0;
-            for(ThreadData t : td){
-                processed += t.total_counter;
-            }
-            progress = (int) percentage(processed, nblines);
+            progress = (int) percentage(_s.getProcessed(), nblines);
             _qpb.setValue(progress);
             msleep(500);
         }

include/Statsgen.h

@@ -17,7 +17,6 @@
 
 #include <string>
 #include <iostream>
-#include <thread>
 
 #include "Policy.h"
 #include "SecurityRules.h"
@@ -39,123 +38,38 @@ public:
 	Statsgen(){}
 	Statsgen(const std::string& name);
 
-	/**
-	 * @brief useful to hide statistics below 1%
-	 * @param hr: 1 to hide, else 0
-	 */
-	inline void setHiderare(const int& hr) { hiderare = hr; }
+	int generate_stats();
+	void print_stats() const;
 
-	/**
-	 * @brief Defining the number of best results the user wants to see,
-	 * and so the result should be clearer
-	 * @param t: number of interesting results
-	 */
+	inline void setHiderare(const int& hr) { hiderare = hr; }
 	inline void setTop(const int& t) { top = t; }
-
-	/**
-	 * @brief Defining a regular expression to analyse only
-	 * some interesting passwords
-	 * @param reg: regular expression
-	 */
 	inline void setRegex(const std::string& reg) {
 		current_regex = reg;
 		use_regex = true;
 	}
-
-	/**
-	 * @brief Useful to know if the database uses the format withcount
-	 * @param var: true if database uses the format withcount, else false
-	 */
 	inline void setWithcount(const bool& wc) { withcount = wc; }
-
-	/**
-	 * @brief Filter for the size of the simple masks, can be
-	 * used as an optimisation option
-	 * @param limit: number that limit the size of the simple masks
-	 */
 	inline void setLimitSimplemask(const int& limit) { limitSimplemask = limit; }
-
-	/**
-	 * @brief Filter for the size of the advanced masks, can be
-	 * used as an optimisation option
-	 * @param limit: number that limit the size of the advanced masks
-	 */
 	inline void setLimitAdvancedmask(const int& limit) { limitAdvancedmask = limit; }
-
-	/**
-	 * @brief Number of threads the user wants to use
-	 * @param nb: number of usable threads
-	 */
-	inline void setNbThread(const int& nb) {
-		int max = std::thread::hardware_concurrency();
-		if (nb > max) {
-			nbThread = max;
-		} else {
-			nbThread = nb;
-		}
-	}
-
+	inline void setNbThread(const int& nb) { nbThread = nb; }
 	void configureThread(ThreadData& td) const;
-
-
-	/**
-	 * @brief Defining all security rules
-	 */
-	void askSecurityRules();
 	void setSecurityRules(const uint& length, const uint& special, const uint& digit, const uint& upper, const uint& lower);
-
-	/**
-	 * @brief Where to write masks
-	 * @param outfile: the file where to write masks
-	 */
 	inline void setOutfile(const std::string& outfile) { outfile_name = outfile; }
-
-	/**
-	 * @brief enable debugging
-	 */
 	inline void enableDebug() { debug_enabled = true; }
 
-
-	/**
-	 * @brief Calculate all statistics for a database
-	 * @return 0 if error, 1 if success
-	 */
-	int generate_stats();
-
-
-	/**
-	 * @brief Print all calculated statistics
-	 */
-	void print_stats();
-
-	inline uint64_t getTotalCounter() const { return td[0].total_counter; }
-	inline uint64_t getTotalFilter() const { return td[0].total_filter; }
-	inline uint64_t getNbSecurePasswords() const { return td[0].sr.nbSecurePassword; }
 	inline int getNbThreads() const { return nbThread; }
-	inline const IntOccurrence& getStatsLength() const { return td[0].length; }
-	inline const StringOccurrence& getStatsCharsets() const { return td[0].charactersets; }
-	inline const StringOccurrence& getStatsSimple() const { return td[0].simplemasks; }
-	inline const StringOccurrence& getStatsAdvanced() const { return td[0].advancedmasks; }
-	inline const std::vector<ThreadData>& getThreadsData() const { return td; }
-	inline bool allFinished() const { return finished == nbThread; }
+	inline uint64_t getNbLines() const { return nblines; }
+	inline uint64_t getProcessed() const { return processed; }
+	inline const ThreadData& getResults() const { return results; }
+	inline bool allFinished() const { return finished; }
 	inline bool allStarted() const { return started; }
 	bool operator==(const Statsgen& other) const;
 
-	/**
-	* @brief Given parameters in threadarg, compute statistics on partition of the file
-	* @param threadarg : parameters and result storage
-	*/
-	static void* generate_stats_thread(void * threadarg);
-
 private:
 	std::string filename;
 	// results of the computation
 	ThreadData results;
-	// Data computed from within a thread, for read-accessibility
-	std::vector<ThreadData> td;
 
 	// Filters
-
 	int hiderare = 0; 				// Hide low statistics
 	int top = 10;					// Show only a top of statistics
 	std::regex current_regex;		// Regex for the interesting passwords
@@ -170,10 +84,17 @@ private:
 	// Security policy
 	SecurityRules _sr = { 0, 8, 0, 1, 1, 1	};
 
-	// threads which have finished
-	uint finished = 0;
+	// nb lines processed
+	uint64_t nblines = 0;
+	// nb lines processed
+	uint64_t processed = 0;
+	// all threads have finished
+	uint finished = false;
 	// all threads have been started
 	bool started = false;
+
+	void handle_password(const std::string& password, const uint64_t& nbPasswords, ThreadData& td) const;
+	std::pair<uint, uint> get_masks(const std::string& password, PasswordStats& c) const;
 };
 
 #endif

include/ThreadData.h

@@ -11,6 +11,7 @@
 typedef std::unordered_map<std::string, uint64_t> StringOccurrence;
 typedef std::unordered_map<int, uint64_t> IntOccurrence;
 
+
 /**
  * @brief minimal number of digits of a password, etc.
  */
@@ -61,4 +62,6 @@ struct ThreadData {
 	SecurityRules sr;
 };
 
+#pragma omp declare reduction(dataSum: ThreadData : omp_out += omp_in ) initializer(omp_priv(omp_orig))
+
 #endif // THREAD_DATA_H
\ No newline at end of file

include/Utils.h

@@ -37,8 +37,8 @@ using MapIterator = typename std::map<K, V>::const_iterator;
  * @return ordered map
  */
 template<typename A>
-std::map<uint64_t, A, std::greater<uint64_t>> flip_map(const std::unordered_map<A, uint64_t> & src) {
-	std::map<uint64_t, A, std::greater<uint64_t>> dst;
+std::multimap<uint64_t, A, std::greater<uint64_t>> flip_map(const std::unordered_map<A, uint64_t> & src) {
+	std::multimap<uint64_t, A, std::greater<uint64_t>> dst;
 
 	for(UnorderedMapIterator<A, uint64_t> it = src.begin(); it != src.end(); ++it)
 		dst.insert(std::make_pair(it->second, it->first));
@@ -56,6 +56,7 @@ std::map<uint64_t, A, std::greater<uint64_t>> flip_map(const std::unordered_map<
  */
 template<typename Type>
 void readResult(const uint64_t & res, const Type& carac, int & count, const uint64_t & total_counter, const int & hiderare) {
+	if(res == 0) return;
 	std::ostringstream ss;
 	float perc = percentage(res, total_counter);
 
@@ -83,7 +84,7 @@ void readResult(const uint64_t & res, const Type& carac, int & count, const uint
 template<typename Type>
 void showMap(const std::unordered_map<Type, uint64_t> & stats, const int & top, const uint64_t & total_counter, const int & hiderare, int & count) {
 	count = 0;
-	std::map<uint64_t, Type, std::greater<uint64_t>> reverse = flip_map<Type>(stats);
+	std::multimap<uint64_t, Type, std::greater<uint64_t>> reverse = flip_map<Type>(stats);
 	std::pair<uint64_t, Type> it;
 	for(std::pair<uint64_t, Type> it : reverse) {
 		readResult<Type>(it.first, it.second, count, total_counter, hiderare);

src/cli/Main.cpp

@@ -61,6 +61,24 @@ void showHelp() {
 	exit(EXIT_SUCCESS);
 }
 
+void askSecurityRules(Statsgen& stats){
+	uint length, special, digit, upper, lower;
+	cout << "Minimal length of a password:" << endl;
+	cin >> length;
+
+	cout << "Minimum of special characters in a password:" << endl;
+	cin >> special;
+
+	cout << "Minimum of digits in a password:" << endl;
+	cin >> digit;
+
+	cout << "Minimum of lower characters in a password:" << endl;
+	cin >> lower;
+
+	cout << "Minimum of upper characters in a password:" << endl;
+	cin >> upper;
+	stats.setSecurityRules(length, special, digit, upper, lower);
+}
 
 int main(int argc,char* argv[]) {
 	locale::global(locale(""));
@@ -128,7 +146,7 @@ int main(int argc,char* argv[]) {
 			return EXIT_FAILURE;
 		}
 		if(arg == "-s" || arg == "--security"){
-			statsgen.askSecurityRules(); continue;
+			askSecurityRules(statsgen); continue;
 		}
 		else {
 			unknownArg(argv[i]);

src/core/Statsgen.cpp

@@ -14,7 +14,7 @@
 #include <fstream>
 #include <iomanip>
 #include <algorithm>
-#include <thread>
+#include <omp.h>
 
 
 #include "Statsgen.h"
@@ -23,26 +23,6 @@ using namespace std;
 
 Statsgen::Statsgen(const std::string& name):filename(name){}
 
-
-void Statsgen::askSecurityRules() {
-	uint length, special, digit, upper, lower;
-	cout << "Minimal length of a password:" << endl;
-	cin >> length;
-
-	cout << "Minimum of special characters in a password:" << endl;
-	cin >> special;
-
-	cout << "Minimum of digits in a password:" << endl;
-	cin >> digit;
-
-	cout << "Minimum of lower characters in a password:" << endl;
-	cin >> lower;
-
-	cout << "Minimum of upper characters in a password:" << endl;
-	cin >> upper;
-	setSecurityRules(length, special, digit, upper, lower);
-}
-
 void Statsgen::setSecurityRules(const uint& length, const uint& special, const uint& digit, const uint& upper, const uint& lower) {
 	_sr = { _sr.nbSecurePassword, length, special, digit, upper, lower };
 }
@@ -58,55 +38,48 @@ void Statsgen::configureThread(ThreadData& td) const {
 }
 
 int Statsgen::generate_stats() {
-	td = vector<ThreadData>(nbThread);
+	processed = 0;
+	finished = false;
 	results = ThreadData();
-	finished = 0;
-	started = false;
-	uint64_t nbline = 0;
-	nbline = nbline_file(filename);
-	if (!nbline){ // error reading the file
+	configureThread(results);
+	omp_set_num_threads(nbThread);
+	nblines = nbline_file(filename);
+	if (!nblines){ // error reading the file
 		cerr << "[ERROR] Empty file or not existing file" << endl;
 		return 0;
 	}
 
-	vector<pthread_t> threads(nbThread);
-
-	for(uint i = 0; i < nbThread; i++ ) {
-		td[i].thread_id = i + 1;
-		configureThread(td[i]);
-
-		if(i == 0){
-			td[i].lineBegin = 0;
+	started = true;
+#pragma omp parallel reduction(dataSum:results)
+	{
+#pragma omp single
+		{
+			nbThread = omp_get_num_threads();
+		}
+		uint thread_id = omp_get_thread_num();
+		ifstream inputfile(filename);
+		string line;
+		int nbpasswords = 1;
+		istringstream iss;
+		for(uint numline = 0; numline < nblines; ++numline){
+			getline(inputfile, line);
+			if(withcount){
+				iss = istringstream(line);
+				iss >> nbpasswords;
+				iss >> line;
 			}
 			else {
-			td[i].lineBegin =  td[i-1].lineEnd + 1;
+				nbpasswords = 1;
 			}
-		td[i].lineEnd = (i+1)*nbline/nbThread;
-
-		if(debug_enabled){
-			cerr << "[DEBUG] " << "Thread " << td[i].thread_id << " analyse : " << td[i].lineBegin << " --> " << td[i].lineEnd << endl;
+			if((numline % nbThread) == thread_id){
+				handle_password(line, nbpasswords, results);
+#pragma omp atomic
+				++processed;
 			}
-
-		int rc = pthread_create(&threads[i], NULL, generate_stats_thread, (void *)&td[i] );
-
-		if (rc) {
-			cerr << "[ERROR] unable to create thread," << rc << endl;
-			exit(-1);
 		}
 	}
 
-	started = true;
-
-	void *status;
-	for(uint i = 0; i < nbThread; i++ ) {
-		int rc = pthread_join(threads[i], &status);
-		if (rc) {
-			cerr << "[ERROR] unable to join," << rc << endl;
-			exit(-1);
-		}
-		results += td[i];
-		finished++;
-	}
+	finished = true;
 
 	if (!results.total_counter) {
 		cerr << "[ERROR] Empty file or not existing file" << endl;
@@ -116,7 +89,7 @@ int Statsgen::generate_stats() {
 	return 1;
 }
 
-void Statsgen::print_stats() {
+void Statsgen::print_stats() const {
 	int count;
 	float perc = percentage(results.total_filter, results.total_counter);
 
@@ -159,7 +132,10 @@ void Statsgen::print_stats() {
 
 	if (limitSimplemask > 0) {
 		cout << endl;
-		readResult(results.simplemasks["othermasks"], "othermasks", count, results.total_counter, hiderare);
+		auto r = results.simplemasks.find("othermasks");
+		if(r != results.simplemasks.end()){
+			readResult(r->second, r->first, count, results.total_counter, hiderare);
+		}
 	}
 
 
@@ -169,7 +145,7 @@ void Statsgen::print_stats() {
 	if (! outfile_name.empty()){
 		locale::global(locale("C"));
 		ofstream outfile_stream(outfile_name);
-		map<uint64_t, string, greater<uint64_t>> reverse = flip_map(results.advancedmasks);
+		multimap<uint64_t, string, greater<uint64_t>> reverse = flip_map(results.advancedmasks);
 		for(pair<uint64_t, string> it : reverse){
 			if(it.second == "othermasks") continue;
 			outfile_stream << it.second << "," << it.first << endl;
@@ -179,12 +155,15 @@ void Statsgen::print_stats() {
 
 	if (limitAdvancedmask > 0) {
 		cout << endl;
-		readResult(results.advancedmasks["othermasks"], "othermasks", count, results.total_counter, hiderare);
+		auto r = results.advancedmasks.find("othermasks");
+		if(r != results.advancedmasks.end()){
+			readResult(r->second, r->first, count, results.total_counter, hiderare);
+		}
 	}
 }
 
 
-pair<uint, uint> get_masks(const string& password, PasswordStats& c){
+pair<uint, uint> Statsgen::get_masks(const string& password, PasswordStats& c) const {
 	char last_simplemask = 'a';
 	uint sizeSimpleMask = 0;
 	uint sizeAdvancedMask = 0;
@@ -232,63 +211,32 @@ pair<uint, uint> get_masks(const string& password, PasswordStats& c){
 	return make_pair(sizeSimpleMask, sizeAdvancedMask);
 }
 
-void handle_password(const string& password, const uint64_t& nbPasswords, ThreadData* my_data){
-	my_data->total_counter += nbPasswords;
-	if(my_data->use_regex && !regex_match(password,my_data->current_regex)){
+void Statsgen::handle_password(const string& password, const uint64_t& nbPasswords, ThreadData& my_data) const {
+	my_data.total_counter += nbPasswords;
+	if(my_data.use_regex && !regex_match(password,my_data.current_regex)){
 		return;
 	}
 	PasswordStats c;
 
 	pair<uint, uint> masks = get_masks(password, c);
 
-	if(c.pol.satisfies(my_data->sr, password.size())){
-		my_data->sr.nbSecurePassword++;
+	if(c.pol.satisfies(my_data.sr, password.size())){
+		my_data.sr.nbSecurePassword++;
 	}
 
-	if (masks.first > my_data->limitSimplemask) {
+	if (masks.first > my_data.limitSimplemask) {
 		c.simplemask_string = "othermasks";
 	}
-	if (masks.second > my_data->limitAdvancedmask) {
+	if (masks.second > my_data.limitAdvancedmask) {
 		c.advancedmask_string = "othermasks";
 	}
 
-	my_data->total_filter += nbPasswords;
-	my_data->length[ password.size() ] += nbPasswords;
-	my_data->charactersets[ c.pol ] += nbPasswords;
-	my_data->simplemasks[ c.simplemask_string ] += nbPasswords;
-	my_data->advancedmasks[ c.advancedmask_string ] += nbPasswords;
-	my_data->minMaxValue.updateMinMax(c.pol);
-}
-
-void* Statsgen::generate_stats_thread(void* threadarg) {
-	ThreadData* my_data = (ThreadData *) threadarg;
-
-	ifstream readfile(my_data->filename);
-	uint64_t nbline = 0;
-	string line;
-	string password;
-	uint64_t nbPasswords;
-
-	while(getline(readfile, line)){
-		++nbline;
-		if (nbline < my_data->lineBegin){ continue; }
-		if (nbline > my_data->lineEnd){ break; }
-		if (line.size() == 0){ continue; }
-
-		if (my_data->withcount) {
-			istringstream is(line);
-			is >> nbPasswords;
-			is >> password;
-		}
-		else {
-			nbPasswords = 1;
-			password = line;
-		}
-		handle_password(password, nbPasswords, my_data);
-	}
-
-	readfile.close();
-	pthread_exit(NULL);
+	my_data.total_filter += nbPasswords;
+	my_data.length[ password.size() ] += nbPasswords;
+	my_data.charactersets[ c.pol ] += nbPasswords;
+	my_data.simplemasks[ c.simplemask_string ] += nbPasswords;
+	my_data.advancedmasks[ c.advancedmask_string ] += nbPasswords;
+	my_data.minMaxValue.updateMinMax(c.pol);
 }
 
 bool Statsgen::operator==(const Statsgen& o) const {

src/gui/MainWindow.cpp

@@ -156,13 +156,14 @@ void MainWindow::disableWithCount()
 }
 
 double MainWindow::initGraphicalStats(QBarSeries * barLength, QPieSeries * pieCharset, QPieSeries* pieSimple, QPieSeries* pieAdvanced, double & percentageTotal, double & percentageSecurity) {
-    double total = stats.getTotalCounter();
-    double filter = stats.getTotalFilter();
+    const ThreadData& results = stats.getResults();
+    double total = results.total_counter;
+    double filter = results.total_filter;
     percentageTotal = percentage(filter, total);
-    percentageSecurity = percentage(stats.getNbSecurePasswords(), total);
+    percentageSecurity = percentage(results.sr.nbSecurePassword, total);
 
     /* LENGTH HISTOGRAM */
-    map<uint64_t, int, greater<uint64_t>> reverseL = flip_map<int>(stats.getStatsLength());
+    multimap<uint64_t, int, greater<uint64_t>> reverseL = flip_map<int>(results.length);
     double percentageL;
     double maxPercLength = 0;
     uint64_t nbHideL = 0;
@@ -187,7 +188,7 @@ double MainWindow::initGraphicalStats(QBarSeries * barLength, QPieSeries * pieCh
 
 
     /* CHARSET PIECHART */
-    map<uint64_t, string, greater<uint64_t>> reverseC = flip_map(stats.getStatsCharsets());
+    multimap<uint64_t, string, greater<uint64_t>> reverseC = flip_map(results.charactersets);
     uint64_t top = 0;
     uint64_t nbHideC = 0;
     pieCharset->clear();
@@ -203,7 +204,7 @@ double MainWindow::initGraphicalStats(QBarSeries * barLength, QPieSeries * pieCh
     pieCharset->append("Other charsets", nbHideC);
 
     /* SIMPLE PIECHART */
-    map<uint64_t, string, greater<uint64_t>> reverseS = flip_map(stats.getStatsSimple());
+    multimap<uint64_t, string, greater<uint64_t>> reverseS = flip_map(results.simplemasks);
     uint64_t top_simple = 0;
     uint64_t nbHideS = 0;
     pieSimple->clear();
@@ -219,7 +220,7 @@ double MainWindow::initGraphicalStats(QBarSeries * barLength, QPieSeries * pieCh
     pieSimple->append("Other Masks", nbHideS);
 
     /* ADVANCED PIECHART */
-    map<uint64_t, string, greater<uint64_t>> reverseA = flip_map(stats.getStatsAdvanced());
+    multimap<uint64_t, string, greater<uint64_t>> reverseA = flip_map(results.advancedmasks);
     uint64_t top_advanced = 0;
     uint64_t nbHideA = 0;
     pieAdvanced->clear();
@@ -261,7 +262,7 @@ QVBoxLayout* MainWindow::drawPieChart(QPieSeries* qps, QVBoxLayout* layout, cons
 
 void MainWindow::handleResults()
 {
-
+    const ThreadData& results = stats.getResults();
     progressThread->terminate();
     progressThread->wait();
     ui->progressBar->setValue(100);
@@ -313,9 +314,9 @@ void MainWindow::handleResults()
     /* LABELS */
 
     ui->AnalyzedLabel->setText("Number of analyzed passwords: "
-                               + QString::number(stats.getTotalFilter(), 'g', 2)
+                               + QString::number(results.total_filter, 'g', 2)
                                + " on a total of "
-                               + QString::number(stats.getTotalFilter(), 'g', 2)
+                               + QString::number(results.total_counter, 'g', 2)
                                + " passwords (" + QString::number(percentageTotal) + "%)");