- Ajout de processing sur les textes indéxés (et centralisation de la configuration) (405ce24c) · Commits · open-source / oscar

module/Oscar/src/Oscar/Command/OscarCommandAbstract.php

+1 −0

Original line number	Diff line number	Diff line
		@@ -32,6 +32,7 @@ abstract class OscarCommandAbstract extends Command
		const COMMAND_ORGANIZATION_SEARCH_REINDEX = 'organization:search:reindex';

		const COMMAND_PERSON_SEARCH_REINDEX = 'person:search:reindex';
		const COMMAND_PROJECT_SEARCH_REINDEX_ALL = 'project:search-rebuild';

		/** @var ServiceManager ServiceManager */
		private $servicemanager;

module/Oscar/src/Oscar/Command/OscarProjectSearchRebuildCommand.php

0 → 100644

+38 −0

Original line number	Diff line number	Diff line
		<?php
		/**
		* Created by PhpStorm.
		* User: bouvry
		* Date: 04/10/19
		* Time: 11:49
		*/

		namespace Oscar\Command;

		use Exception;
		use Symfony\Component\Console\Input\InputInterface;
		use Symfony\Component\Console\Output\OutputInterface;

		class OscarProjectSearchRebuildCommand extends OscarAdvancedCommandAbstract
		{
		protected static $defaultName = OscarCommandAbstract::COMMAND_PROJECT_SEARCH_REINDEX_ALL;

		protected function configure()
		{
		parent::configure();
		$this
		->setDescription("Reconstruction de l'index de recherche pour TOUS les projet");
		}

		protected function execute(InputInterface $input, OutputInterface $output): int
		{
		parent::execute($input, $output);

		try {
		$this->getIO()->title("Reconstruction de l'index de recherche des projets");
		$this->getProjectGrantService()->searchIndex_rebuildProject();
		return $this->finalSuccess("Index de recherche mis à jour");
		} catch (Exception $e) {
		return $this->finalFatalError($e);
		}
		}
		}
		No newline at end of file

module/Oscar/src/Oscar/Controller/ProjectGrantController.php

+22 −5

Original line number	Diff line number	Diff line
		@@ -2745,17 +2745,17 @@ class ProjectGrantController extends AbstractOscarController implements UseNotif
		$filters = [
		"at" => [
		"key" => 'at',
		"label" => $this->oscarText("Type"),
		"label" => $this->oscarText("Activité - Type"),
		"raw" => 'at;;;'
		],
		"ap" => [
		"key" => 'ap',
		"label" => $this->oscarText("Personne"),
		"label" => $this->oscarText("Personne impliquée"),
		"raw" => 'ap;;;'
		],
		"ao" => [
		"key" => 'ao',
		"label" => $this->oscarText("Organisation"),
		"label" => $this->oscarText("Organisation impliquée"),
		"raw" => 'ao;;;'
		],
		"orp" => [
		@@ -2774,6 +2774,7 @@ class ProjectGrantController extends AbstractOscarController implements UseNotif
		];

		$rolesPersons = $this->getOscarUserContextService()->getAllRoleIdPersonInActivity();
		$rolesOrganizations = $this->getOscarUserContextService()->getRolesOrganizationInActivity();

		$directions = [
		'desc' => 'Décroissant',
		@@ -2800,14 +2801,30 @@ class ProjectGrantController extends AbstractOscarController implements UseNotif
		'activityTypes' => $this->getActivityTypeService()->getActivityTypesTree(true),
		'activityDocumentTypes' => $this->getProjectGrantSearchService()->getFilterOptionsMilestones(),
		'rolesPersons' => $rolesPersons,
		'rolesOrganizations' => $rolesOrganizations,
		'filters' => $filters,
		'sorts' => $sorts,
		'directions' => $directions,
		];

		$urlShow = $this->url()->fromRoute('contract/show', ['id' => '']);
		$urlShowOrganization = "";
		$urlShowPerson = "";
		$urlShowProject = $this->url()->fromRoute('project/show', ['id' => '']);

		if( $this->getOscarUserContextService()->hasPrivileges(Privileges::ORGANIZATION_SHOW) ){
		$urlShowOrganization = $this->url()->fromRoute('organization/show', ['id' => '']);
		}
		if( $this->getOscarUserContextService()->hasPrivileges(Privileges::PERSON_SHOW) ){
		$urlShowPerson = $this->url()->fromRoute('person/show', ['id' => '']);
		}

		return [
		'filtersOptions' => $filtersOptions,
		'urlShow' => $this->url()->fromRoute('contract/show', ['id' => '']),
		'urlShowProject' => $this->url()->fromRoute('project/show', ['id' => ''])
		'urlShow' => $urlShow,
		'urlShowProject' => $urlShowProject,
		'urlShowOrganization' => $urlShowOrganization,
		'urlShowPerson' => $urlShowPerson,
		];
		}

module/Oscar/src/Oscar/Service/ProjectGrantService.php

+88 −12

Original line number	Diff line number	Diff line
		@@ -257,7 +257,7 @@ class ProjectGrantService implements UseGearmanJobLauncherService, UseOscarConfi
		/** @var ActivityOrganization $organizationActivity */
		foreach ($activity->getOrganizationsDeep() as $organizationActivity) {
		if ($filterRoleOrganization == null \|\| in_array(
		$organizationActivity->getRoleObj()->getId(),
		$organizationActivity->getRoleObj() ? $organizationActivity->getRoleObj()->getId() : null,
		$filterRoleOrganization
		)) {
		$org = $organizationActivity->getOrganization();
		@@ -2265,8 +2265,79 @@ class ProjectGrantService implements UseGearmanJobLauncherService, UseOscarConfi
		];
		}
		$filterES['ap'][] = $currentFilter;
		} else if($value2) {

		$currentFilter = [
		"nested" => [
		"path" => "persons",
		"query" => [
		"bool" => [
		"filter" => [
		["term" => ["persons.role_id" => $value2]]
		]
		]
		]
		]
		];
		$filterES['ap'][] = $currentFilter;


		} else {
		$error = "Aucune personne/rôle selectionné";
		}
		break;

		case 'ao':
		if (!array_key_exists('ao', $filterES)) {
		$filterES['ao'] = [];
		}

		if ($value1) {
		try {
		$organization = $this->getOrganizationService()->getOrganization($value1);
		$value1 = $organization->getId();
		$value1Displayed = (string) $organization;
		} catch (OscarException $e) {
		$error = "Impossible de trouver cette organisation";
		}

		$currentFilter = [
		"nested" => [
		"path" => "organizations",
		"query" => [
		"bool" => [
		"filter" => [
		["term" => ["organizations.id" => $value1]]
		]
		]
		]
		]
		];
		if ($value2) {
		$currentFilter['nested']['query']['bool']['filter'][] = [
		"term" => ["organizations.role_id" => $value2],
		];
		}
		$filterES['ao'][] = $currentFilter;
		} else if($value2) {

		$currentFilter = [
		"nested" => [
		"path" => "organizations",
		"query" => [
		"bool" => [
		"filter" => [
		["term" => ["organizations.role_id" => $value2]]
		]
		]
		]
		]
		];
		$filterES['ao'][] = $currentFilter;


		} else {
		$error = "Aucune personne selectionnée";
		$error = "Aucune personne/rôle selectionné";
		}
		break;

		@@ -2308,13 +2379,12 @@ class ProjectGrantService implements UseGearmanJobLauncherService, UseOscarConfi
		foreach ($filterES as $key=>$value) {
		if( $key == 'at' ){
		$value = [
		"terms" => [
		$value['field'] => $value['values']
		]

		[ "terms" => [ $value['field'] => $value['values'] ] ]
		];
		}
		$body['query']['bool']['filter'] = array_merge($body['query']['bool']['filter'], $value);
		foreach ($value as $f) {
		$body['query']['bool']['filter'][] = $f;
		}
		}

		//$body['query']['bool']['filter'] = array_values($filterES);
		@@ -2340,10 +2410,8 @@ class ProjectGrantService implements UseGearmanJobLauncherService, UseOscarConfi

		try {
		if ($params['project'] == '1') {

		$result = $this->getSearchEngineProjectStrategy()->searchRaw($params['q'], $limit = 1000, $body);
		} else {

		$result = $this->getSearchEngineStrategy()->searchRaw($params['q'], $limit = 1000, $body);
		}

		@@ -2395,16 +2463,25 @@ class ProjectGrantService implements UseGearmanJobLauncherService, UseOscarConfi
		return $this->getMilestoneService()->getMilestonesByActivityId($idActivity);
		}

		public function searchIndex_rebuild()
		public function searchIndex_rebuildProject()
		{
		// PROJETS
		$projects = $this->getEntityManager()->getRepository(Project::class)->findAll();

		$this->getLoggerService()->info('[elasic] Reindex ' . count($projects) . ' project(s)');
		$this->getSearchEngineProjectStrategy()->rebuildIndex($projects);
		}

		public function searchIndex_rebuild()
		{

		//$activities = $this->getEntityManager()->getRepository(Activity::class)->findAll();
		$activities = $this->getEntityManager()->getRepository(Activity::class)->findAll();
		/*$activities = $this->getEntityManager()->getRepository(Activity::class)->findBy(
		[],
		null,
		1000
		);*/

		$this->getLoggerService()->info('[elasic] Reindex ' . count($activities) . ' activitie(s)');
		$done = $this->getSearchEngineStrategy()->rebuildIndex($activities);
		@@ -2446,7 +2523,6 @@ class ProjectGrantService implements UseGearmanJobLauncherService, UseOscarConfi
		$class = new \ReflectionClass(ProjectElasticSearch::class);
		$searchStrategy = $class->newInstanceArgs($params);


		$searchStrategy->staticDatas = [
		'typesChainIds' => $this->getActivityTypeService()->getIdsTreeByLast(),
		'typesChainlabels' => $this->getActivityTypeService()->getIdsTreeByLast(true),

module/Oscar/src/Oscar/Strategy/Search/ElasticActivitySearch.php

+35 −101

Original line number	Diff line number	Diff line
		@@ -32,27 +32,19 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear

		protected function indicesCreateBody(): array
		{
		$mapper = new MapperElasticSearch();
		return [
		'settings' => [
		'analysis' => [
		'analyzer' => [
		'folding_analyzer' => [
		'tokenizer' => 'standard',
		'filter' => ['lowercase', 'asciifolding']
		]
		]
		]
		],
		'settings' => $mapper->getMappingAnalysis(),
		'mappings' => [
		'date_detection' => false,
		'numeric_detection' => false,
		'properties' => [
		'id' => ['type' => 'integer'],
		'acronym' => ['type' => 'text'],
		'description' => ['type' => 'text'],
		'acronym' => $mapper->getMappingText(),
		'description' => $mapper->getMappingText(),
		'status' => ['type' => 'keyword'],
		'statusStr' => ['type' => 'keyword'],
		'label' => ['type' => 'text'],
		'label' => $mapper->getMappingText(),
		'type' => ['type' => 'keyword'],
		'type_id' => ['type' => 'integer'],
		'type_chain_id' => ['type' => 'integer'],
		@@ -70,31 +62,15 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear
		'project' => [
		'properties' => [
		"id" => ['type' => 'keyword'],
		"acronym" => ['type' => 'text'],
		"label" => ['type' => 'text'],
		"description" => ['type' => 'text'],
		]
		],
		'jalons' => [
		'type' => 'nested',
		'properties' => [
		'id' => ['type' => 'integer'],
		'label' => ['type' => 'text'],
		'progression' => ['type' => 'keyword'],
		'date' => ['type' => 'date'],
		]
		],
		'persons' => [
		'type' => 'nested',
		'properties' => [
		'id' => ['type' => 'integer'],
		'firstname' => ['type' => 'text'],
		'lastname' => ['type' => 'text'],
		'role_id' => ['type' => 'integer'],
		'role' => ['type' => 'keyword'],
		"acronym" => $mapper->getMappingText(),
		"label" => $mapper->getMappingText(),
		"description" => $mapper->getMappingText(),
		]
		],
		// 'numerotation' => [ 'type' => 'text', 'analyzer' => 'folding_analyzer'],
		'jalons' => $mapper->getMappingMilestones(),
		'persons' => $mapper->getMappingAffectationPerson(),
		'organizations' => $mapper->getMappingAffectationOrganization(),
		// 'numerotation' => [ 'type' => 'text', 'analyzer, => 'french_analyzer 'analyzer' => 'french_analyzer'],
		// 'numbers' => [ 'type' => 'nested' ],

		]
		@@ -116,6 +92,8 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear
		$activity = $object;
		}

		$mapper = new MapperElasticSearch();

		$project_body = "";
		$project_id = null;

		@@ -148,39 +126,23 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear

		$project = null;
		if ($activity->getProject()) {
		$project = [
		'id' => $activity->getProject()->getId(),
		'acronym' => $activity->getProject()->getAcronym(),
		'label' => $activity->getProject()->getLabel(),
		'description' => $activity->getProject()->getDescription(),
		];
		$project = $mapper->mapProjectSimple($activity->getProject());
		}

		$members = [];
		/** @var ActivityPerson $personAffectation */
		foreach ($activity->getPersonsDeep() as $personAffectation) {
		$members[] = [
		'id' => $personAffectation->getPerson()->getId(),
		'firstname' => $personAffectation->getPerson()->getFirstname(),
		'lastname' => $personAffectation->getPerson()->getLastname(),
		'role_id' => $personAffectation->getRoleObj()->getId(),
		'role' => $personAffectation->getRole(),
		'principal' => $personAffectation->isPrincipal(),
		];
		$members[] = $mapper->mapAffectationPerson($personAffectation);
		}

		$partners = [];
		/** @var ActivityOrganization $organizationAffectation */
		foreach ($activity->getOrganizationsDeep() as $organizationAffectation) {
		$partners[] = [
		'id' => $organizationAffectation->getOrganization()->getId(),
		'code' => $organizationAffectation->getOrganization()->getCode(),
		'shortname' => $organizationAffectation->getOrganization()->getShortName(),
		'longname' => $organizationAffectation->getOrganization()->getFullName(),
		'role_id' => $organizationAffectation->getRoleObj() ? $organizationAffectation->getRoleObj()->getId() : "",
		'role' => $organizationAffectation->getRole(),
		'principal' => $organizationAffectation->isPrincipal(),
		];
		if( $organizationAffectation->getRoleObj() ){
		$partners[] = $mapper->mapAffectationOrganization($organizationAffectation);
		} else {
		error_log("Affectation Activity<>Organization error " . $organizationAffectation->getId());
		}
		}

		$jalons = [];
		@@ -188,7 +150,8 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear
		foreach ($activity->getMilestones() as $milestone) {
		$jalons[] = [
		'id' => $milestone->getId(),
		'label' => $milestone->getType()->getLabel(),
		'label' => $mapper->textProcessing($milestone->getType()->getLabel()),
		'comment' => $mapper->textProcessing($milestone->getComment()),
		'finishable' => $milestone->isFinishable(),
		'progression' => $milestone->getFinished(),
		'date' => $milestone->getDateStartStr()
		@@ -197,13 +160,13 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear

		$datas = [
		'id' => $activity->getId(),
		'label' => $activity->getLabel(),
		'label' => $mapper->textProcessing($activity->getLabel()),
		'type' => $type_label,
		'type_id' => $type_id,
		'type_chain_id' => $type_chain_id,
		'type_chain_labels' => $type_chain_label,
		'acronym' => $activity->getAcronym(),
		'description' => $activity->getDescription(),
		'description' => $mapper->textProcessing($activity->getDescription()),
		'eotp' => $activity->getCodeEOTP(),
		'status' => $activity->getStatus(),
		'statusStr' => $activity->getStatusLabel(),
		@@ -212,10 +175,10 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear
		'financialImpact' => $activity->getFinancialImpact(),
		'disciplines' => $activity->getDisciplinesArray(),
		'keywords' => $activity->getMotsclesArray(),
		'dateCreated' => $this->getDateTimeStrOrNull($activity->getDateCreated(), true),
		'dateUpdated' => $this->getDateTimeStrOrNull($activity->getDateUpdated(), true),
		'dateStart' => $this->getDateTimeStrOrNull($activity->getDateStart()),
		'dateEnd' => $this->getDateTimeStrOrNull($activity->getDateEnd()),
		'dateCreated' => $mapper->formatDateTimeNullable($activity->getDateCreated(), true),
		'dateUpdated' => $mapper->formatDateTimeNullable($activity->getDateUpdated(), true),
		'dateStart' => $mapper->formatDateTimeNullable($activity->getDateStart()),
		'dateEnd' => $mapper->formatDateTimeNullable($activity->getDateEnd()),
		'jalons' => $jalons,
		'project' => $project,
		'persons' => $members,
		@@ -254,38 +217,6 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear

		public function getFieldsSearchedWeighted(string $search): array
		{
		$andQuery = null;
		$wordsNbr = 1;

		// // Détection des recherches strictes
		// if (preg_match_all('/^"(.*)"$/', $search, $matches, PREG_SET_ORDER)) {
		// if (count($matches) == 1 && array_key_exists(1, $matches[0])) {
		// $andQuery = $matches[0][0];
		// $wordsNbr = 2;
		// }
		// }
		//
		//// if(preg_match_all('/^query:(.*)$/', $search, $matches, PREG_SET_ORDER)) {
		//// if( count($matches) == 1 && array_key_exists(1, $matches[0])) {
		//// $andQuery = $matches[0][1];
		//// $wordsNbr = 2;
		//// }
		//// }
		//
		// if (preg_match_all('/^(.)\$/', $search, $matches, PREG_SET_ORDER)) {
		// if (count($matches) == 1 && array_key_exists(1, $matches[0])) {
		// $andQuery = $matches[0][0];
		// $wordsNbr = 2;
		// }
		// }
		//
		// if ($andQuery === null) {
		// $words = explode(" ", $search);
		// $wordsNbr = count($words);
		// $andQuery = implode(" AND ", $words);
		// }


		$query = [
		"bool" => [
		"should" => [
		@@ -308,8 +239,9 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear
		$query["bool"]["should"][] = ["prefix" => ["acronym" => ["value" => strtolower($search), "boost" => 5]]];

		// Champ LABEL/PROJECT LABEL
		$query["bool"]["should"][] = ["match" => ["label" => ["query" => $search, "fuzziness" => 1, "boost" => 4]]];
		$query["bool"]["should"][] = ["match" => ["project" => ["query" => $search, "fuzziness" => 1, "boost" => 3]]];
		$query["bool"]["should"][] = ["match" => ["label" => ["query" => $search, "fuzziness" => 1, "boost" => 5]]];
		$query["bool"]["should"][] = ["match" => ["project.label" => ["query" => $search, "fuzziness" => 1, "boost" => 10]]];
		$query["bool"]["should"][] = ["match" => ["project.description" => ["query" => $search, "fuzziness" => 1, "boost" => 3]]];

		// Champ NUMBERS
		$query["bool"]["should"][] = ["match" => ["numbers" => ["query" => $search, "fuzziness" => 1, "boost" => 5]]];
		@@ -322,7 +254,9 @@ class ElasticActivitySearch extends ElasticSearchEngine implements IActivitySear
		$query["bool"]["should"][] = ["match" => ["type" => ["query" => $search, "fuzziness" => 1, "boost" => 3]]];
		$query["bool"]["should"][] = ["match" => ["description" => ["query" => $search, "fuzziness" => 1, "boost" => 2]]];
		$query["bool"]["should"][] = ["match" => ["persons.lastname" => ["query" => $search, "fuzziness" => 1]]];
		$query["bool"]["should"][] = ["match" => ["organization" => ["query" => $search, "fuzziness" => 1]]];
		$query["bool"]["should"][] = ["match" => ["organization.code" => ["query" => $search, "fuzziness" => 1]]];
		$query["bool"]["should"][] = ["match" => ["organization.shortname" => ["query" => $search, "fuzziness" => 1]]];
		$query["bool"]["should"][] = ["match" => ["organization.longname" => ["query" => $search, "fuzziness" => 1]]];


		return $query;