Commit c42ba229 authored by Mathieu Valois's avatar Mathieu Valois
Browse files

Merge OMP branch

parents dfc1fc5b ce535e43

CMakeLists.txt

+3 −4
Original line number Diff line number Diff line 
# Version de cmake demandée.

cmake_minimum_required(VERSION 2.8)



find_package(Threads REQUIRED)

find_package(OpenMP)



# Chemin des répertoires contenant les fichiers entêtes.

include_directories(include)
@@ -10,8 +10,7 @@ include_directories(include) 
set(EXECUTABLE_OUTPUT_PATH bin/${CMAKE_BUILD_TYPE})



# Option du compilateur pour supporter C 2011.

set(CMAKE_CXX_FLAGS "-pedantic -Wall -std=c++11 -Wextra -O3")

set(CMAKE_LD_FLAGS "-lpthread")

set(CMAKE_CXX_FLAGS "-pedantic -Wall -std=c++11 -Wextra -O3 -fopenmp")



add_executable(cppack

				src/core/Statsgen.cpp

include/MainWindow.h

+3 −10
Original line number Diff line number Diff line
@@ -26,21 +26,14 @@ class ProgressThread : public QThread 
{

    Q_OBJECT

    void run() {

        unsigned int progress = 0;

        uint64_t processed = 0;

        const std::vector<ThreadData>& td = _s.getThreadsData();



        int progress = 0;

        while(!_s.allStarted()){

            msleep(500);

        }

        const uint64_t nblines = td.back().lineEnd;

        const uint64_t nblines = _s.getNbLines();



        while(!_s.allFinished()){

            processed = 0;

            for(ThreadData t : td){

                processed += t.total_counter;

            }

            progress = (int) percentage(processed, nblines);

            progress = (int) percentage(_s.getProcessed(), nblines);

            _qpb.setValue(progress);

            msleep(500);

        }

include/Statsgen.h

+12 −29
Original line number Diff line number Diff line
@@ -17,7 +17,6 @@ 


#include <string>

#include <iostream>

#include <thread>



#include "Policy.h"

#include "SecurityRules.h"
@@ -86,14 +85,7 @@ public: 
	 * @brief Number of threads the user wants to use

	 * @param nb: number of usable threads

	 */

	inline void setNbThread(const int& nb) {

		int max = std::thread::hardware_concurrency();

		if (nb > max) {

			nbThread = max;

		} else {

			nbThread = nb;

		}

	}

	inline void setNbThread(const int& nb) { nbThread = nb; }



	void configureThread(ThreadData& td) const;
@@ -128,34 +120,21 @@ public: 
	 */

	void print_stats();



	inline uint64_t getTotalCounter() const { return td[0].total_counter; }

	inline uint64_t getTotalFilter() const { return td[0].total_filter; }

	inline uint64_t getNbSecurePasswords() const { return td[0].sr.nbSecurePassword; }

	inline int getNbThreads() const { return nbThread; }

	inline const IntOccurrence& getStatsLength() const { return td[0].length; }

	inline const StringOccurrence& getStatsCharsets() const { return td[0].charactersets; }

	inline const StringOccurrence& getStatsSimple() const { return td[0].simplemasks; }

	inline const StringOccurrence& getStatsAdvanced() const { return td[0].advancedmasks; }

	inline const std::vector<ThreadData>& getThreadsData() const { return td; }

	inline bool allFinished() const { return finished == nbThread; }

	inline uint64_t getNbLines() const { return nblines; }

	inline uint64_t getProcessed() const { return processed; }

	inline const ThreadData& getResults() const { return results; }

	inline bool allFinished() const { return finished; }

	inline bool allStarted() const { return started; }

	bool operator==(const Statsgen& other) const;



	/**

	* @brief Given parameters in threadarg, compute statistics on partition of the file

	* @param threadarg : parameters and result storage

	*/

	static void* generate_stats_thread(void * threadarg);

	void handle_password(const std::string& password, const uint64_t& nbPasswords, ThreadData& td) const;



private:

	std::string filename;

	// results of the computation

	ThreadData results;

	// Data computed from within a thread, for read-accessibility

	std::vector<ThreadData> td;



	// Filters



	int hiderare = 0; 				// Hide low statistics

	int top = 10;					// Show only a top of statistics

	std::regex current_regex;		// Regex for the interesting passwords
@@ -170,8 +149,12 @@ private: 
	// Security policy

	SecurityRules _sr = { 0, 8, 0, 1, 1, 1	};



	// threads which have finished

	uint finished = 0;

	// nb lines processed

	uint64_t nblines = 0;

	// nb lines processed

	uint64_t processed = 0;

	// all threads have finished

	uint finished = false;

	// all threads have been started

	bool started = false;

};

include/ThreadData.h

+3 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
typedef std::unordered_map<std::string, uint64_t> StringOccurrence;

typedef std::unordered_map<int, uint64_t> IntOccurrence;





/**

 * @brief minimal number of digits of a password, etc.

 */
@@ -61,4 +62,6 @@ struct ThreadData { 
	SecurityRules sr;

};



#pragma omp declare reduction(dataSum: ThreadData : omp_out += omp_in ) initializer(omp_priv(omp_orig))



#endif // THREAD_DATA_H
 
 No newline at end of file

src/core/Statsgen.cpp

+37 −85
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ 
#include <fstream>

#include <iomanip>

#include <algorithm>

#include <thread>

#include <omp.h>





#include "Statsgen.h"
@@ -58,55 +58,38 @@ void Statsgen::configureThread(ThreadData& td) const { 
}



int Statsgen::generate_stats() {

	td = vector<ThreadData>(nbThread);

	processed = 0;

	finished = false;

	results = ThreadData();

	finished = 0;

	started = false;

	uint64_t nbline = 0;

	nbline = nbline_file(filename);

	if (!nbline){ // error reading the file

	configureThread(results);

	omp_set_num_threads(nbThread);

	nblines = nbline_file(filename);

	if (!nblines){ // error reading the file

		cerr << "[ERROR] Empty file or not existing file" << endl;

		return 0;

	}



	vector<pthread_t> threads(nbThread);



	for(uint i = 0; i < nbThread; i++ ) {

		td[i].thread_id = i + 1;

		configureThread(td[i]);



		if(i == 0){

			td[i].lineBegin = 0;

		}

		else{

			td[i].lineBegin =  td[i-1].lineEnd + 1;

		}

		td[i].lineEnd = (i+1)*nbline/nbThread;



		if(debug_enabled){

			cerr << "[DEBUG] " << "Thread " << td[i].thread_id << " analyse : " << td[i].lineBegin << " --> " << td[i].lineEnd << endl;

	started = true;

#pragma omp parallel reduction(dataSum:results)

	{

#pragma omp single

		{

			nbThread = omp_get_num_threads();

		}

		uint thread_id = omp_get_thread_num();

		ifstream inputfile(filename);

		string line;

		for(uint numline = 0; numline < nblines; ++numline){

			getline(inputfile, line);

			if((numline % nbThread) == thread_id){

				handle_password(line, 1, results);

#pragma omp atomic

				++processed;

			}



		int rc = pthread_create(&threads[i], NULL, generate_stats_thread, (void *)&td[i] );



		if (rc) {

			cerr << "[ERROR] unable to create thread," << rc << endl;

			exit(-1);

		}

	}



	started = true;



	void *status;

	for(uint i = 0; i < nbThread; i++ ) {

		int rc = pthread_join(threads[i], &status);

		if (rc) {

			cerr << "[ERROR] unable to join," << rc << endl;

			exit(-1);

		}

		results += td[i];

		finished++;

	}

	finished = true;



	if (!results.total_counter) {

		cerr << "[ERROR] Empty file or not existing file" << endl;
@@ -232,63 +215,32 @@ pair<uint, uint> get_masks(const string& password, PasswordStats& c){ 
	return make_pair(sizeSimpleMask, sizeAdvancedMask);

}



void handle_password(const string& password, const uint64_t& nbPasswords, ThreadData* my_data){

	my_data->total_counter += nbPasswords;

	if(my_data->use_regex && !regex_match(password,my_data->current_regex)){

void Statsgen::handle_password(const string& password, const uint64_t& nbPasswords, ThreadData& my_data) const {

	my_data.total_counter += nbPasswords;

	if(my_data.use_regex && !regex_match(password,my_data.current_regex)){

		return;

	}

	PasswordStats c;



	pair<uint, uint> masks = get_masks(password, c);



	if(c.pol.satisfies(my_data->sr, password.size())){

		my_data->sr.nbSecurePassword++;

	if(c.pol.satisfies(my_data.sr, password.size())){

		my_data.sr.nbSecurePassword++;

	}



	if (masks.first > my_data->limitSimplemask) {

	if (masks.first > my_data.limitSimplemask) {

		c.simplemask_string = "othermasks";

	}

	if (masks.second > my_data->limitAdvancedmask) {

	if (masks.second > my_data.limitAdvancedmask) {

		c.advancedmask_string = "othermasks";

	}



	my_data->total_filter += nbPasswords;

	my_data->length[ password.size() ] += nbPasswords;

	my_data->charactersets[ c.pol ] += nbPasswords;

	my_data->simplemasks[ c.simplemask_string ] += nbPasswords;

	my_data->advancedmasks[ c.advancedmask_string ] += nbPasswords;

	my_data->minMaxValue.updateMinMax(c.pol);

}



void* Statsgen::generate_stats_thread(void* threadarg) {

	ThreadData* my_data = (ThreadData *) threadarg;



	ifstream readfile(my_data->filename);

	uint64_t nbline = 0;

	string line;

	string password;

	uint64_t nbPasswords;



	while(getline(readfile, line)){

		++nbline;

		if (nbline < my_data->lineBegin){ continue; }

		if (nbline > my_data->lineEnd){ break; }

		if (line.size() == 0){ continue; }



		if (my_data->withcount) {

			istringstream is(line);

			is >> nbPasswords;

			is >> password;

		}

		else {

			nbPasswords = 1;

			password = line;

		}

		handle_password(password, nbPasswords, my_data);

	}



	readfile.close();

	pthread_exit(NULL);

	my_data.total_filter += nbPasswords;

	my_data.length[ password.size() ] += nbPasswords;

	my_data.charactersets[ c.pol ] += nbPasswords;

	my_data.simplemasks[ c.simplemask_string ] += nbPasswords;

	my_data.advancedmasks[ c.advancedmask_string ] += nbPasswords;

	my_data.minMaxValue.updateMinMax(c.pol);

}



bool Statsgen::operator==(const Statsgen& o) const {