Skip to content
GitLab
Commit 0ff1de35 authored by Bertrand Gauthier's avatar Bertrand Gauthier
Browse files

WIP import/synchro d'une seule ligne

parent abfa8435
Pipeline #15202 passed with stage
in 17 seconds
Hide whitespace changes
Inline Side-by-side
src/UnicaenDbImport/CodeGenerator/CodeGenerator.php
View file @ 0ff1de35
......@@ -240,6 +240,26 @@ abstract class CodeGenerator implements CodeGeneratorInterface
return $query;
}
/**
* @param SourceInterface $source
* @param string $sourceCodeColumn
* @param string $sourceCode
* @return string
*/
public function generateSQLForSelectOneFromSource(SourceInterface $source, string $sourceCodeColumn, string $sourceCode): string
{
if ($source->getSelect()) {
$query = $source->getSelect();
} else {
$sourceTable = $source->getTable();
$query = $this->tableHelper->generateSQLForSelectFromTable($sourceTable);
}
// NB : $source->getWhere() n'est pris en compte.
return sprintf("SELECT * FROM (%s) tmp WHERE %s = '%s'", $query, $sourceCodeColumn, $sourceCode);
}
/**
* @param SourceInterface $source
* @param DestinationInterface $destination
......
src/UnicaenDbImport/CodeGenerator/CodeGeneratorInterface.php
View file @ 0ff1de35
......@@ -66,6 +66,14 @@ interface CodeGeneratorInterface
*/
public function generateSQLForSelectFromSource(SourceInterface $source): string;
/**
* @param SourceInterface $source
* @param string $sourceCodeColumn
* @param string $sourceCode
* @return string
*/
public function generateSQLForSelectOneFromSource(SourceInterface $source, string $sourceCodeColumn, string $sourceCode): string;
/**
* @param SourceInterface $source
* @param DestinationInterface $destination
......
src/UnicaenDbImport/Service/Api/ApiService.php
View file @ 0ff1de35
......@@ -80,15 +80,36 @@ class ApiService
return $this;
}
/**
* Interroge l'API pour obtenir un enregistrement particulier.
*
* @param ApiConnection $connection
* @param SourceInterface $source
* @param string $sourceCode Identifiant de l'enregistrement voulu
* @return stdClass
* @throws \UnicaenDbImport\Service\Exception\ApiServiceException
*/
public function fetchOne(ApiConnection $connection, SourceInterface $source, string $sourceCode): stdclass
{
$this->setConfig($connection->toArray());
$url = $this->buildUrl($connection, $source, $sourceCode);
return $this->getOne($url);
}
/**
* Interroge l'API pour obtenir tous les enregistrements.
*
* @param ApiConnection $connection
* @param SourceInterface $source
* @param array $params
*
* @return stdClass[]
*
* @throws \UnicaenDbImport\Service\Exception\ApiServiceException
*/
public function fetchAll(ApiConnection $connection, SourceInterface $source): array
public function fetch(ApiConnection $connection, SourceInterface $source, array $params = []): array
{
$this->setConfig($connection->toArray());
......@@ -99,7 +120,7 @@ class ApiService
$pageParam = $connection->getPageParam();
$pageSizeParam = $connection->getPageSizeParam();
return $this->callApi($url, $pageSize, 0, $pageSizeParam, $pageParam);
return $this->get($url, $pageSize, 0, $pageSizeParam, $pageParam, $params);
}
/**
......@@ -110,7 +131,7 @@ class ApiService
* @return stdClass[]
* @throws \UnicaenDbImport\Service\Exception\ApiServiceException
*/
public function fetchFirstRow(ApiConnection $connection, SourceInterface $source): array
public function fetchFirst(ApiConnection $connection, SourceInterface $source): array
{
$this->setConfig($connection->toArray());
......@@ -120,11 +141,11 @@ class ApiService
$pageParam = $connection->getPageParam();
$pageSizeParam = $connection->getPageSizeParam();
return $this->callApi($url, $pageSize, $page, $pageSizeParam, $pageParam);
return $this->get($url, $pageSize, $page, $pageSizeParam, $pageParam);
}
/**
* Interrogation de l'API.
* Interrogation de l'API retournant plusieurs enregistrements.
*
* NB : Dès lors que $pageSize > 0, l'interrogation se fait page par page.
*
......@@ -133,36 +154,42 @@ class ApiService
* @param int $page
* @param string $pageSizeParam
* @param string $pageParam
* @param array $params
*
* @return stdClass[]
*
* @throws \UnicaenDbImport\Service\Exception\ApiServiceException
*/
protected function callApi(
protected function get(
string $url,
int $pageSize,
int $page = 0,
string $pageSizeParam = 'page_size',
string $pageParam = 'page'): array
string $pageParam = 'page',
array $params = []): array
{
$paginated = $pageSize > 0;
$onePageOnly = $page <> 0;
if ($paginated) {
$params[$pageSizeParam] = $pageSize;
}
$rows = [];
$p = 1;
$knownPageCount = 0;
do {
$finalUrl = $url;
$params = [];
if ($paginated) {
$params[$pageParam] = $p;
$params[$pageSizeParam] = $pageSize;
}
if (!empty($params)) {
$sep = strpos($finalUrl, '?') === FALSE ? '?' : '&';
$finalUrl .= $sep . http_build_query($params);
}
$responseData = $this->get($finalUrl);
$responseData = $this->sendRequest($finalUrl);
// si l'API retourne des données au format HAL, le nombre de pages total est fourni, on prend !
if (isset($responseData->page_count)) {
......@@ -170,6 +197,9 @@ class ApiService
}
$data = $this->normalizeResponseData($responseData);
if (! is_array($data)) {
return [$data];
}
$rows = array_merge($rows, $data);
$p++;
......@@ -191,33 +221,57 @@ class ApiService
return $rows;
}
/**
* Interrogation de l'API retournant plusieurs enregistrements.
*
* NB : Dès lors que $pageSize > 0, l'interrogation se fait page par page.
*
* @param string $url
* @return stdClass
* @throws \UnicaenDbImport\Service\Exception\ApiServiceException
*/
protected function getOne(string $url): stdClass
{
$responseData = $this->sendRequest($url);
return $this->normalizeResponseData($responseData);
}
/**
* @param array|stdClass $responseData
* @return array
* @return array|stdClass
*/
private function normalizeResponseData($responseData)
{
if (isset($responseData->page_count) && isset($responseData->{'_embedded'})) {
// HAL
$toArray = get_object_vars($responseData->{'_embedded'});
$data = (array) reset($toArray);
array_walk($data, function(stdClass &$obj) {
unset($obj->_links);
});
return $data;
if ($responseData instanceof stdClass) {
// plusieurs enregistrements au format HAL
if (isset($responseData->page_count) && isset($responseData->{'_embedded'})) {
$toArray = get_object_vars($responseData->{'_embedded'});
$data = (array) reset($toArray);
array_walk($data, function(stdClass &$obj) {
unset($obj->_links);
});
return $data;
}
// 1 enregistrement au format HAL
elseif (isset($responseData->_links)) {
unset($responseData->_links);
}
}
return $responseData;
}
/**
* @param ApiConnection $connection
* @param ApiConnection $connection
* @param SourceInterface $source
* @param string|null $sourceCode
* @return string
*/
private function buildUrl(ApiConnection $connection, SourceInterface $source): string
private function buildUrl(ApiConnection $connection, SourceInterface $source, string $sourceCode = null): string
{
$url = $connection->getUrl();
......@@ -228,19 +282,12 @@ class ApiService
$url = $url . '/' . $select;
}
return $url;
}
if ($sourceCode !== null) {
$url = rtrim($url, '/');
$url .= '/' . $sourceCode; // NB : il faut que l'API supporte ce format d'URL
}
/**
* Envoie une requête quelconque au web service, ex: 'version/current'.
*
* @param string $uri
* @return stdClass|stdClass[]
* @throws \UnicaenDbImport\Service\Exception\ApiServiceException
*/
private function get(string $uri)
{
return $this->sendRequest($uri);
return $url;
}
/**
......
src/UnicaenDbImport/Service/Database/DatabaseService.php
View file @ 0ff1de35
......@@ -342,7 +342,7 @@ class DatabaseService
/**
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function truncateDestinationTable()
private function truncateDestinationTable()
{
// détermination de l'id de la source des données à insérer, pour n'effacer que les données de cette source
// (si aucune donnée à insérer, basta)
......@@ -363,11 +363,27 @@ class DatabaseService
}
}
/**
* @param string $sourceCode
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
private function deleteFromDestinationTable(string $sourceCode)
{
$sourceCodeColumn = $this->destination->getSourceCodeColumn();
$sql = $this->codeGenerator->generateSQLForClearTable($table = $this->destination->getTable());
$sql .= " WHERE $sourceCodeColumn = '$sourceCode'";
try {
$result = $this->queryExecutor->exec($sql, $this->destination->getConnection());
} catch (Exception $e) {
throw DatabaseServiceException::error("Erreur lors du vidage de la table destination '$table' pour le code '$sourceCode'", $e);
}
}
/**
* @return int
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function populateDestinationTableFromSource(): int
public function populateDestinationTable(): int
{
$destinationTable = $this->destination->getTable();
$idColumnSequenceName = $this->destination->getIdColumnSequence();
......@@ -377,7 +393,34 @@ class DatabaseService
$this->truncateDestinationTable();
$result = -1;
try {
$result = $this->populateTableFromSource($destinationTable, $idColumnSequenceName);
$result = $this->populateTable($destinationTable, $idColumnSequenceName);
$connection->commit();
} catch (DatabaseServiceException $e) {
$this->rollBack($connection);
throw $e;
} catch (Exception $e) {
$this->rollBack($connection);
}
return $result;
}
/**
* @param string $sourceCode
* @return int
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function populateDestinationTableForSourceCode(string $sourceCode): int
{
$destinationTable = $this->destination->getTable();
$idColumnSequenceName = $this->destination->getIdColumnSequence();
$connection = $this->destination->getConnection();
$connection->beginTransaction();
$this->deleteFromDestinationTable($sourceCode);
$result = -1;
try {
$result = $this->populateTable($destinationTable, $idColumnSequenceName);
$connection->commit();
} catch (DatabaseServiceException $e) {
$this->rollBack($connection);
......@@ -407,7 +450,7 @@ class DatabaseService
* @return int
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function populateTableFromSource(string $tableName, $idColumnSequence = null): int
public function populateTable(string $tableName, $idColumnSequence = null): int
{
$sourceCode = $this->source->getCode();
$data = $this->source->getData();
......@@ -618,12 +661,12 @@ class DatabaseService
}
/**
* Requête la Source pour obtenir les données sources.
* Requête la Source pour obtenir toutes les données sources.
*
* @return array
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function fetchSource(): array
public function fetch(): array
{
$selectSql = $this->codeGenerator->generateSQLForSelectFromSource($this->source);
......@@ -634,13 +677,35 @@ class DatabaseService
}
}
/**
* Requête la Source pour obtenir un enregistrement particulier des données sources.
*
* @param string $sourceCode
* @return array|null
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function fetchOne(string $sourceCode): ?array
{
$selectSql = $this->codeGenerator->generateSQLForSelectOneFromSource(
$this->source,
$this->destination->getSourceCodeColumn(),
$sourceCode
);
try {
return $this->queryExecutor->fetch($selectSql, $this->source->getConnection());
} catch (Exception $e) {
throw DatabaseServiceException::error("Erreur lors du fetch", $e);
}
}
/**
* Requête la Source pour obtenir le 1er enregistrement des données sources.
*
* @return array|null
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function fetchSourceFirstRow(): ?array
public function fetchFirst(): ?array
{
$selectSql = $this->codeGenerator->generateSQLForSelectFromSource($this->source);
......
src/UnicaenDbImport/Service/Facade/AbstractFacadeService.php
View file @ 0ff1de35
......@@ -113,10 +113,60 @@ abstract class AbstractFacadeService
* Requête la Source pour obtenir les données sources.
* Ces données sont ensuite disponibles dans la Source en question.
*
* @param array $params
*
* @throws \Exception
*
* @see SourceInterface::getData()
*/
public function fetchSource(array $params = [])
{
if (count($this->source->getData()) === 0) {
/** @var ApiConnection|DbConnection $connection */
$connection = $this->source->getConnection();
$this->logger->info("Interrogation de la source " . $this->source);
switch (true) {
case $connection instanceof ApiConnection:
if ($this->logger) {
$this->apiService->setLogger($this->logger);
}
try {
$data = $this->apiService->fetch($connection, $this->source, $params);
} catch (ApiServiceException $e) {
throw new Exception("Erreur rencontrée lors de l'obtention des données sources (API)", null, $e);
}
break;
case $connection instanceof DbConnection:
try {
$data = $this->databaseService->fetch();
} catch (DatabaseServiceException $e) {
throw new DatabaseServiceException("Erreur rencontrée lors de l'obtention des données sources (DB)", null, $e);
}
break;
default:
throw new RuntimeException("Type de connexion source inattendue");
}
$this->source->setData($data);
}
}
/**
* Requête la Source pour obtenir 1 enregistrement particulier.
* Ces données sont ensuite disponibles dans la Source en question.
*
* @param string $sourceCode
*
* @throws \Exception
*
* @see SourceInterface::getData()
*/
public function fetchSource()
public function fetchSourceSingle(string $sourceCode)
{
if (count($this->source->getData()) === 0) {
......@@ -131,7 +181,8 @@ abstract class AbstractFacadeService
$this->apiService->setLogger($this->logger);
}
try {
$data = $this->apiService->fetchAll($connection, $this->source);
$result = $this->apiService->fetchOne($connection, $this->source, $sourceCode);
$data = $result ? [$result] : [];
} catch (ApiServiceException $e) {
throw new Exception("Erreur rencontrée lors de l'obtention des données sources (API)", null, $e);
}
......@@ -139,7 +190,7 @@ abstract class AbstractFacadeService
case $connection instanceof DbConnection:
try {
$data = $this->databaseService->fetchSource();
$data = $this->databaseService->fetchOne($sourceCode);
} catch (DatabaseServiceException $e) {
throw new DatabaseServiceException("Erreur rencontrée lors de l'obtention des données sources (DB)", null, $e);
}
......@@ -175,7 +226,7 @@ abstract class AbstractFacadeService
$this->apiService->setLogger($this->logger);
}
try {
$data = $this->apiService->fetchFirstRow($connection, $this->source);
$data = $this->apiService->fetchFirst($connection, $this->source);
} catch (ApiServiceException $e) {
throw new Exception("Erreur rencontrée lors de l'obtention de la 1ere ligne des données sources (API)", null, $e);
}
......@@ -183,7 +234,7 @@ abstract class AbstractFacadeService
case $connection instanceof DbConnection:
try {
$firstRow = $this->databaseService->fetchSourceFirstRow();
$firstRow = $this->databaseService->fetchFirst();
} catch (DatabaseServiceException $e) {
throw new Exception("Erreur rencontrée lors de l'obtention de la 1ere ligne des données sources (DB)", null, $e);
}
......
src/UnicaenDbImport/Service/Facade/ImportFacadeService.php
View file @ 0ff1de35
......@@ -36,7 +36,54 @@ class ImportFacadeService extends AbstractFacadeService
switch (true) {
case $connection instanceof DbConnection:
$count = $this->databaseService->populateDestinationTableFromSource();
$count = $this->databaseService->populateDestinationTable();
$result = new ImportResult();
$result->setDestinationTablePopulateResult($count);
return $result;
default:
throw ConnectionException::unexpected($connection);
}
}
/**
* @param string $sourceCode
* @return ImportResult
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function importOneToDestination(string $sourceCode): ImportResult
{
$connection = $this->destination->getConnection();
$this->logger->info(sprintf("Mise à jour de la destination %s pour le code %s", $this->destination, $sourceCode));
switch (true) {
case $connection instanceof DbConnection:
$count = $this->databaseService->populateDestinationTableForSourceCode($sourceCode);
$result = new ImportResult();
$result->setDestinationTablePopulateResult($count);
return $result;
default:
throw ConnectionException::unexpected($connection);
}
}
/**
* @return ImportResult
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function importPartialToDestination(): ImportResult
{
$connection = $this->destination->getConnection();
$this->logger->info(sprintf("Mise à jour partielle de la destination %s", $this->destination));
switch (true) {
case $connection instanceof DbConnection:
$count = $this->databaseService->populateDestinationTablePartial();
$result = new ImportResult();
$result->setDestinationTablePopulateResult($count);
......
src/UnicaenDbImport/Service/Facade/SynchroFacadeService.php
View file @ 0ff1de35
......@@ -103,7 +103,7 @@ class SynchroFacadeService extends AbstractFacadeService
$this->logger->info("Création de la table intermédiaire $intermediateTable");
$this->databaseService->createIntermediateTable($intermediateTable);
$this->logger->info("Peuplement de la table intermédiaire $intermediateTable");
$this->databaseService->populateTableFromSource($intermediateTable, false);
$this->databaseService->populateTable($intermediateTable, false);
}
$this->recreateDiffView();
......
src/UnicaenDbImport/Service/ImportService.php
View file @ 0ff1de35
......@@ -152,4 +152,85 @@ class ImportService
return $result;
}
/**
* Lance l'import spécifié (import brut) d'un seul enregistrement.
*
* @param ImportInterface $import
* @param string $sourceCode Code unique de l'enregistrement à importer
*
* @return ImportResult Résultat de l'exécution de l'import
*
* @throws \UnicaenDbImport\Service\Exception\DatabaseServiceException
*/
public function runImportOne(ImportInterface $import, string $sourceCode): ImportResult
{
$this->logger->info(sprintf("Lancement de l'import %s[%s]", $import, $sourceCode));
$this->importFacadeService->setImport($import);
$this->logFacadeService->setDestination($import->getDestination());
$startDate = date_create();
try {
$this->logFacadeService->createImportLog();
$this->importFacadeService->validateDestination();
$this->importFacadeService->validateSource();
$this->importFacadeService->fetchSourceSingle($sourceCode);
$this->importFacadeService->discoverSourceColumns();
$result = $this->importFacadeService->importOneToDestination($sourceCode);
} catch (Exception $e) {
$result = new ImportResult();
$result->setFailureException($e);
}