Commit 5d896fa1 authored by Mathieu Valois's avatar Mathieu Valois
Browse files

Producer/Consumer

parent 117ab72d

Statsgen/include/Queue.h

0 → 100644
+66 −0
Original line number Diff line number Diff line 
//

// Copyright (c) 2013 Juan Palacios juan.palacios.puyana@gmail.com

// Subject to the BSD 2-Clause License

// - see < http://opensource.org/licenses/BSD-2-Clause>

//



#ifndef CONCURRENT_QUEUE_

#define CONCURRENT_QUEUE_



#include <queue>

#include <thread>

#include <mutex>

#include <condition_variable>



template <typename T>

class Queue

{

 public:



  T pop() 

  {

    std::unique_lock<std::mutex> mlock(mutex_);

    while (queue_.empty())

    {

      cond_.wait(mlock);

    }

    auto val = queue_.front();

    queue_.pop();

    return val;

  }



  void pop(T& item)

  {

    std::unique_lock<std::mutex> mlock(mutex_);

    while (queue_.empty())

    {

      cond_.wait(mlock);

    }

    item = queue_.front();

    queue_.pop();

  }



  void push(const T& item)

  {

    std::unique_lock<std::mutex> mlock(mutex_);

    queue_.push(item);

    mlock.unlock();

    cond_.notify_one();

  }



  bool empty()

  {

    return queue_.empty();

  }

  

  Queue()=default;

  Queue(const Queue&) = delete;            // disable copying

  Queue& operator=(const Queue&) = delete; // disable assignment

  

 private:

  std::queue<T> queue_;

  std::mutex mutex_;

  std::condition_variable cond_;

};



#endif

Statsgen/include/statsgen.h

+20 −13
Original line number Diff line number Diff line
@@ -21,6 +21,8 @@ 
#include <regex>

#include <queue>



#include "Queue.h"



#define MAX_THREADS 32
@@ -80,13 +82,27 @@ struct SecurityRules { 
 * @brief All needed variables to give to each thread

 */

struct thread_data {

	int thread_id;

	std::string filename;

	uint64_t lineBegin;

	uint64_t lineEnd;

	std::regex current_regex;

	bool use_regex = false;

	bool withcount = false;

	bool hiderare = false;

	int top = 10;



	int limitSimplemask;

	int limitAdvancedmask;



	SecurityRules sr;



	std::string line;



	int nbThread;

	bool endOfWork = false;

};



struct thread_result {

	uint64_t total_counter = 0;

	uint64_t total_filter = 0;

	uint64_t nbSecurePassword = 0;



	std::unordered_map<int, uint64_t> length;

	std::unordered_map<std::string, uint64_t> simplemasks;
@@ -95,15 +111,6 @@ struct thread_data { 
	std::queue<std::string> password_queue;



	minMax minMaxValue;



	std::regex current_regex;

	bool use_regex = false;

	bool withcount = false;



	int limitSimplemask;

	int limitAdvancedmask;



	SecurityRules sr;

};

Statsgen/src/core/statsgen.cpp

+117 −205
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 


#include "statsgen.h"

#include "utils.h"

#include "Queue.h"

using namespace std;
@@ -152,148 +153,172 @@ void Statsgen::setSecurityRules(int length,int special,int digit,int upper,int l 
}







int Statsgen::generate_stats() {

void produce(Queue<string>& work_queue, const string filename, struct thread_data params){

	ifstream readfile(filename);

	std::string line;

	uint64_t nbline = 0;

	if (!is_stdin){

		nbline = nbline_file(filename);

		if (!nbline){ // error reading the file

			return 0;

	while(readfile.good()){

		nbline++;

		getline(readfile, line);

		work_queue.push(line);

	}

	readfile.close();

	for (int i=0; i < params.nbThread; i++){

		work_queue.push("END");

	}

}



	int rc;

	int i;

	pthread_t threads[MAX_THREADS];

	struct thread_data td[MAX_THREADS];



	pthread_attr_t attr;

	void *status;



	pthread_attr_init(&attr);

	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);

void consume(std::tuple<struct thread_data&, Queue<string>&,

		Queue<struct thread_result>&> queues){



	// split stdin into nbThread files on disk

	if (is_stdin){

	struct thread_data& params = get<0>(queues);

	Queue<string>& work_queue = get<1>(queues);

	Queue<struct thread_result>& result_queue = get<2>(queues);

	bool _continue = true;

	struct thread_result result;

	string line;

		nbline = 0;

		while(getline(cin, line)){

			td[nbline%nbThread].password_queue.push(line);

			nbline++;

	string password;

	uint64_t nbPasswords;

	while (_continue){

		line = work_queue.pop();

		if (line.compare("END") == 0){

			_continue = false;

			break;

		}

		if (line.empty()){

			continue;

		}



	for( i = 0; i < nbThread; i++ ) {

		td[i].filename = filename;

		td[i].thread_id = i + 1;

		td[i].current_regex = current_regex;

		td[i].use_regex = use_regex;

		td[i].withcount = withcount;

		td[i].limitSimplemask = limitSimplemask;

		td[i].limitAdvancedmask = limitAdvancedmask;

		Container c;



		td[i].lineBegin = i*(nbline/nbThread) + 1;

		td[i].lineEnd = (i+1)*nbline/nbThread;

		if (i > 0) {

			while (td[i].lineBegin <= td[i-1].lineEnd) {

				td[i].lineBegin++;

		if (params.withcount) {

			// TODO opti split

			uint64_t i = 0;

			bool number=false;

			for(i=0; i < line.length(); i++) {

				if(iswdigit(line.at(i))) {

					number=true;

				}

				else if (!iswdigit(line.at(i)) && number) {

					break;

				}





		td[i].sr.nbSecurePassword = 0;

		td[i].sr.minLength = minLength;

		td[i].sr.minSpecial = minSpecial;

		td[i].sr.minDigit = minDigit;

		td[i].sr.minLower = minLower;

		td[i].sr.minUpper = minUpper;



		debug("Thread", td[i].thread_id, "analyse :", td[i].lineBegin, " -->", td[i].lineEnd);

		// We use std::queue if input is from stdin

		if (is_stdin) {

			rc = pthread_create(&threads[i], NULL, generate_stats_thread_queue, (void *)&td[i] );

			}

		// Else we split the file in nbThread threads

		else {

			rc = pthread_create(&threads[i], NULL, generate_stats_thread, (void *)&td[i] );

			password = line.substr(i+1,line.length());

			nbPasswords = stoi(line.substr(0,i));

		}



		if (rc) {

			cout << "Error:unable to create thread," << rc << endl;

			exit(-1);

		else {

			password = line;

			nbPasswords = 1;

		}

		result.total_counter += nbPasswords;

		if ( !params.use_regex || (params.use_regex && regex_match(password, params.current_regex)) ) {

			result.total_filter += nbPasswords;

			analyze_password(password, c, params.sr,params.limitAdvancedmask, params.limitSimplemask);

			result.length[ c.pass_length ] += nbPasswords;

			result.charactersets[ c.characterset ] += nbPasswords;

			result.simplemasks[ c.simplemask_string ] += nbPasswords;

			result.advancedmasks[ c.advancedmask_string ] += nbPasswords;

		}



	pthread_attr_destroy(&attr);

	for( i = 0; i < nbThread; i++ ) {

		rc = pthread_join(threads[i], &status);

		if (rc) {

			cout << "Error:unable to join," << rc << endl;

			exit(-1);

		updateMinMax(result.minMaxValue, c.pol);

	}

	result_queue.push(result);

}



int Statsgen::generate_stats() {

	Queue<string> work_queue;

	Queue<struct thread_result> result_queue;

	struct thread_data params;

	params.hiderare = hiderare;

	params.top = top;

	params.current_regex = current_regex;

	params.use_regex = use_regex;

	params.withcount = withcount;

	params.limitSimplemask = limitSimplemask;

	params.limitAdvancedmask = limitAdvancedmask;

	params.nbThread = nbThread;

	params.sr.minLength = minLength;

	params.sr.minSpecial = minSpecial;

	params.sr.minDigit = minDigit;

	params.sr.minLower = minLower;

	params.sr.minUpper = minUpper;



	std::tuple<

		struct thread_data&,

		Queue<string>&,

		Queue<struct thread_result>&> queues(params,

						work_queue, result_queue);





	std::thread producer(std::bind(produce,

						std::ref(work_queue),

						filename,

						params));

	std::vector<std::thread> consumers;



	for (int i=0; i < nbThread; i++){

		std::thread consumer(std::bind(&consume,

							 std::ref(queues)));

		consumers.push_back(std::move(consumer));

	}



	producer.join();

	for (auto& consumer: consumers){

		consumer.join();

	}



	Policy policyMin;

	Policy policyMax;

	struct thread_result result;

	while(!result_queue.empty()){

		result = result_queue.pop();

		total_counter+=result.total_counter;

		total_filter+=result.total_filter;



	for(i=0;i<nbThread;i++)	{

		total_counter+=td[i].total_counter;

		total_filter+=td[i].total_filter;



		policyMin.digit = td[i].minMaxValue.mindigit;

		policyMin.lower = td[i].minMaxValue.minlower;

		policyMin.upper = td[i].minMaxValue.minupper;

		policyMin.special = td[i].minMaxValue.minspecial;

		policyMin.digit = result.minMaxValue.mindigit;

		policyMin.lower = result.minMaxValue.minlower;

		policyMin.upper = result.minMaxValue.minupper;

		policyMin.special = result.minMaxValue.minspecial;



		policyMax.digit = td[i].minMaxValue.maxdigit;

		policyMax.lower = td[i].minMaxValue.maxlower;

		policyMax.upper = td[i].minMaxValue.maxupper;

		policyMax.special = td[i].minMaxValue.maxspecial;

		policyMax.digit = result.minMaxValue.maxdigit;

		policyMax.lower = result.minMaxValue.maxlower;

		policyMax.upper = result.minMaxValue.maxupper;

		policyMax.special = result.minMaxValue.maxspecial;



		nbSecurePassword += td[i].sr.nbSecurePassword;

		nbSecurePassword += result.nbSecurePassword;



		updateMinMax(minMaxValue, policyMin);

		updateMinMax(minMaxValue, policyMax);





		for(auto it=td[i].length.begin();it!=td[i].length.end();it++)

		for(auto it=result.length.begin();it!=result.length.end();it++)

		{

			stats_length[it->first]+=it->second;

		}

		td[i].length.clear();

		for(auto it=td[i].charactersets.begin();it!=td[i].charactersets.end();it++)

		result.length.clear();

		for(auto it=result.charactersets.begin();it!=result.charactersets.end();it++)

		{

			stats_charactersets[it->first]+=it->second;

		}

		td[i].charactersets.clear();

		for(auto it=td[i].simplemasks.begin();it!=td[i].simplemasks.end();it++)

		result.charactersets.clear();

		for(auto it=result.simplemasks.begin();it!=result.simplemasks.end();it++)

		{

			stats_simplemasks[it->first]+=it->second;

		}

		td[i].simplemasks.clear();

		for(auto it=td[i].advancedmasks.begin();it!=td[i].advancedmasks.end();it++)

		result.simplemasks.clear();

		for(auto it=result.advancedmasks.begin();it!=result.advancedmasks.end();it++)

		{

			stats_advancedmasks[it->first]+=it->second;

		}

		td[i].advancedmasks.clear();

		result.advancedmasks.clear();

	}







	if (!total_counter) {

		wcerr << "Empty file or not existing file" << endl;

		return 0;

	}



	cout << "total counter " << total_counter << endl;



	return 1;

}





void Statsgen::print_stats() {

	int count;

	cout << total_filter << " " << total_counter << endl;

	float perc = (float) 100 * (total_filter / total_counter);



	wcout << "\n\tSelected " << total_filter << " on " << total_counter << " passwords\t("
@@ -360,11 +385,6 @@ void Statsgen::print_stats() { 
	}

}













void analyse_letter(const char & letter, char & last_simplemask, string & simplemask_string, string & advancedmask_string, Policy & policy, int & sizeAdvancedMask, int & sizeSimpleMask) {

	sizeAdvancedMask++;
@@ -523,114 +543,6 @@ void updateMinMax(minMax & minMaxValue, const Policy & pol) { 
	}

}





void * generate_stats_thread_queue(void * threadarg) {

	struct thread_data *my_data;

	my_data = (struct thread_data *) threadarg;



	string line;

	uint64_t nbline = 0;

	while(!my_data->password_queue.empty()) {

		++nbline;

		line = my_data->password_queue.front();

		my_data->password_queue.pop();



		if (line.size() == 0) {

			continue;

		}



		Container c;



		my_data->total_counter++;

		if ( !my_data->use_regex || (my_data->use_regex && regex_match(line,my_data->current_regex)) ) {

			analyze_password(line, c, my_data->sr, my_data->limitAdvancedmask, my_data->limitSimplemask);



			my_data->total_filter++;



			my_data->length[ c.pass_length ] += 1;

			my_data->charactersets[ c.characterset ] += 1;

			my_data->simplemasks[ c.simplemask_string ] += 1;

			my_data->advancedmasks[ c.advancedmask_string ] += 1;

		}

		updateMinMax(my_data->minMaxValue, c.pol);

	}



	pthread_exit(NULL);

}





void * generate_stats_thread(void * threadarg) {

	struct thread_data *my_data;

	my_data = (struct thread_data *) threadarg;



	ifstream readfile(my_data->filename);

	string line;

	uint64_t nbline = 0;



	while(readfile.good()) {

		++nbline;

		getline(readfile, line);

		if (nbline < my_data->lineBegin) {

			continue;

		}

		if (nbline > my_data->lineEnd) {

			break;

		}





		if (line.size() == 0) {

			continue;

		}



		Container c;



		if (my_data->withcount) {

			uint64_t i = 0;

			bool number=false;

			for(i=0; i < line.length(); i++) {

				if(iswdigit(line.at(i))) {

					number=true;

				}

				else if (!iswdigit(line.at(i)) && number) {

					break;

				}

			}

			string password = line.substr(i+1,line.length());

			uint64_t nbPasswords = stoi(line.substr(0,i));



			my_data->total_counter += nbPasswords;



			if ( !my_data->use_regex || (my_data->use_regex && regex_match(password,my_data->current_regex)) ) {

				my_data->total_filter += nbPasswords;

				analyze_password(password, c, my_data->sr,my_data->limitAdvancedmask, my_data->limitSimplemask);



				my_data->length[ c.pass_length ] += nbPasswords;

				my_data->charactersets[ c.characterset ] += nbPasswords;

				my_data->simplemasks[ c.simplemask_string ] += nbPasswords;

				my_data->advancedmasks[ c.advancedmask_string ] += nbPasswords;

			}

		}

		else {

			my_data->total_counter++;

			if ( !my_data->use_regex || (my_data->use_regex && regex_match(line,my_data->current_regex)) ) {

				analyze_password(line, c, my_data->sr, my_data->limitAdvancedmask, my_data->limitSimplemask);



				my_data->total_filter++;



				my_data->length[ c.pass_length ] += 1;

				my_data->charactersets[ c.characterset ] += 1;

				my_data->simplemasks[ c.simplemask_string ] += 1;

				my_data->advancedmasks[ c.advancedmask_string ] += 1;

			}

		}

		updateMinMax(my_data->minMaxValue, c.pol);

	}



	readfile.close();



	pthread_exit(NULL);

}



uint64_t Statsgen::getTotalCounter() {

	return total_counter;

}