Commit a38768e7 authored by Mathieu Valois's avatar Mathieu Valois
Browse files

handle stdin with multiple threads

parent eed7e02f
Loading
Loading
Loading
Loading
+1 −0
Changes for Statsgen/include/statsgen.h: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -210,6 +210,7 @@ private:
	int limitAdvancedmask = 12;		// Limit the size of Advanced Mask
	int nbThread = 1;				// Number of usable threads, default 1
	std::string outfile_name;		// File where to write masks
	int is_stdin = 0;				// If filename is stdin


	// Dictionary
+39 −8
Changes for Statsgen/src/statsgen.cpp: 39 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -25,8 +25,13 @@ using namespace std;


void Statsgen::setFilename(std::string name) {
	if (name == "-"){
		is_stdin = 1;
	}
	else {
		filename = name;
	}
}


void Statsgen::setHiderare(int val) {
@@ -99,7 +104,10 @@ void Statsgen::setSecurityRules() {


int Statsgen::generate_stats() {
	int nbline = nbline_file(filename);
	int nbline = 0;
	if (!is_stdin){
		nbline = nbline_file(filename);
	}

	int rc;
	int i;
@@ -112,15 +120,31 @@ int Statsgen::generate_stats() {
	pthread_attr_init(&attr);
	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);

	// split stdin into nbThread files on disk
	if (is_stdin){
		ofstream filestreams[nbThread];
		for (i=0; i < nbThread; i++){
		td[i].filename = filename;
		td[i].thread_id = i + 1;
		td[i].current_regex = current_regex;
		td[i].use_regex = use_regex;
		td[i].withcount = withcount;
		td[i].limitSimplemask = limitSimplemask;
		td[i].limitAdvancedmask = limitAdvancedmask;
			filestreams[i].open("statsgen_tmp" + to_string(i));
		}
		std::string line;
		nbline = 0;
		while(getline(cin, line)){
			filestreams[nbline%nbThread] << line << endl;
			nbline++;
		}
		for (i=0; i < nbThread; i++){
			filestreams[i].close();
		}
	}

	for( i = 0; i < nbThread; i++ ) {
		if (is_stdin){
			td[i].filename = "statsgen_tmp" + to_string(i);
			td[i].lineBegin = 0;
			td[i].lineEnd = nbline_file(td[i].filename);
		}
		else {
			td[i].filename = filename;
			td[i].lineBegin = i*(nbline/nbThread) + 1;
			td[i].lineEnd = (i+1)*nbline/nbThread;
			if (i > 0) {
@@ -128,6 +152,13 @@ int Statsgen::generate_stats() {
					td[i].lineBegin++;
				}
			}
		}
		td[i].thread_id = i + 1;
		td[i].current_regex = current_regex;
		td[i].use_regex = use_regex;
		td[i].withcount = withcount;
		td[i].limitSimplemask = limitSimplemask;
		td[i].limitAdvancedmask = limitAdvancedmask;
		

		td[i].sr.nbSecurePassword = 0;