Commit f5d4a194 authored by Mathieu Valois's avatar Mathieu Valois
Browse files

stdin and threads working using a std::queue

parent fa7bbc8c
Loading
Loading
Loading
Loading
+9 −0
Changes for Statsgen/include/statsgen.h: 9 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@
#include <unordered_map>
#include <iostream>
#include <regex>
#include <queue>

#define MAX_THREADS 32

@@ -83,6 +84,7 @@ struct thread_data {
	std::string filename;
	int lineBegin;
	int lineEnd;
	int queue_full = 0;

	double total_counter = 0;
	double total_filter = 0;
@@ -91,6 +93,7 @@ struct thread_data {
	std::unordered_map<std::wstring, double> simplemasks;
	std::unordered_map<std::wstring, double> advancedmasks;
	std::unordered_map<std::wstring, double> charactersets;
	std::queue<std::wstring> password_queue;

	minMax minMaxValue;

@@ -286,5 +289,11 @@ void updateMinMax(minMax & minMaxValue, const Policy & pol);
*/
void * generate_stats_thread(void * threadarg);

/**
 * @brief Compute statistics by consumming password from a queue
 * @param threadarg : arguments required by the thread
 */
void * generate_stats_thread_queue(void * threadarg);


#endif
 No newline at end of file
+63 −23
Changes for Statsgen/src/statsgen.cpp: 63 added lines, 23 removed lines.
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@
#include <vector>
#include <thread>
#include <pthread.h>
#include <chrono>


#include "statsgen.h"
@@ -122,29 +123,24 @@ int Statsgen::generate_stats() {

	// split stdin into nbThread files on disk
	if (is_stdin){
		ofstream filestreams[nbThread];
		for (i=0; i < nbThread; i++){
			filestreams[i].open("statsgen_tmp" + to_string(i));
		}
		std::string line;
		wstring line;
		nbline = 0;
		while(getline(cin, line)){
			filestreams[nbline%nbThread] << line << endl;
		while(getline(wcin, line)){
			td[nbline%nbThread].password_queue.push(line);
			nbline++;
		}
		for (i=0; i < nbThread; i++){
			filestreams[i].close();
		}
		for(i=0; i < nbThread; i++) td[i].queue_full = 1;
	}

	for( i = 0; i < nbThread; i++ ) {
		if (is_stdin){
			td[i].filename = "statsgen_tmp" + to_string(i);
			td[i].lineBegin = 0;
			td[i].lineEnd = nbline_file(td[i].filename);
		}
		else {
		td[i].filename = filename;
		td[i].thread_id = i + 1;
		td[i].current_regex = current_regex;
		td[i].use_regex = use_regex;
		td[i].withcount = withcount;
		td[i].limitSimplemask = limitSimplemask;
		td[i].limitAdvancedmask = limitAdvancedmask;
		
		td[i].lineBegin = i*(nbline/nbThread) + 1;
		td[i].lineEnd = (i+1)*nbline/nbThread;
		if (i > 0) {
@@ -152,13 +148,6 @@ int Statsgen::generate_stats() {
				td[i].lineBegin++;
			}
		}
		}
		td[i].thread_id = i + 1;
		td[i].current_regex = current_regex;
		td[i].use_regex = use_regex;
		td[i].withcount = withcount;
		td[i].limitSimplemask = limitSimplemask;
		td[i].limitAdvancedmask = limitAdvancedmask;


		td[i].sr.nbSecurePassword = 0;
@@ -169,7 +158,14 @@ int Statsgen::generate_stats() {
		td[i].sr.minUpper = minUpper;

		wcout << "Thread " << td[i].thread_id << " analyse : " << td[i].lineBegin << " --> " << td[i].lineEnd << endl;
		// We use std::queue if input is from stdin
		if (is_stdin) {
			rc = pthread_create(&threads[i], NULL, generate_stats_thread_queue, (void *)&td[i] );
		}
		// Else we split the file in nbThread threads
		else {
			rc = pthread_create(&threads[i], NULL, generate_stats_thread, (void *)&td[i] );
		}
	  
		if (rc) {
			cout << "Error:unable to create thread," << rc << endl;
@@ -476,6 +472,50 @@ void updateMinMax(minMax & minMaxValue, const Policy & pol) {
}


void * generate_stats_thread_queue(void * threadarg) {
	struct thread_data *my_data;
	my_data = (struct thread_data *) threadarg;

	wstring line;
	int nbline = 0;
	while(1) {
		// there are passwords in stdin but they have not been pushed to the queue yet
		if (my_data->password_queue.empty() && !my_data->queue_full){
			this_thread::sleep_for(chrono::milliseconds(500));
			continue;
		}
		// there is no more password
		if (my_data->password_queue.empty() && my_data->queue_full){
			break;
		}
		++nbline;
		line = my_data->password_queue.front();
		my_data->password_queue.pop();

		if (line.size() == 0) {
			continue;
		}

		Container c;

		my_data->total_counter++;
		if ( !my_data->use_regex || (my_data->use_regex && regex_match(line,my_data->current_regex)) ) {
			analyze_password(line, c, my_data->sr, my_data->limitAdvancedmask, my_data->limitSimplemask);

			my_data->total_filter++;

			my_data->length[ c.pass_length ] += 1;
			my_data->charactersets[ c.characterset ] += 1;
			my_data->simplemasks[ c.simplemask_string ] += 1;
			my_data->advancedmasks[ c.advancedmask_string ] += 1;
		}
		updateMinMax(my_data->minMaxValue, c.pol);
	}
	wcout << my_data->total_counter << endl;

	pthread_exit(NULL);
}


void * generate_stats_thread(void * threadarg) {
	struct thread_data *my_data;