From 2f1628887e7425fa14f0673f5f5786c433239914 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vial Date: Tue, 15 Jul 2025 19:06:07 -0400 Subject: [PATCH 1/7] Permisos entrypoint --- cli/entrypoint | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cli/entrypoint diff --git a/cli/entrypoint b/cli/entrypoint old mode 100644 new mode 100755 From 8b3cf4776266002cc2989c75d9b33bb65aded312 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vial Date: Tue, 15 Jul 2025 22:18:45 -0400 Subject: [PATCH 2/7] Pending now in queue --- cli/src/Command/{Job => Queue}/Pending.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename cli/src/Command/{Job => Queue}/Pending.php (79%) diff --git a/cli/src/Command/Job/Pending.php b/cli/src/Command/Queue/Pending.php similarity index 79% rename from cli/src/Command/Job/Pending.php rename to cli/src/Command/Queue/Pending.php index 718b375..e85d6cd 100644 --- a/cli/src/Command/Job/Pending.php +++ b/cli/src/Command/Queue/Pending.php @@ -1,10 +1,10 @@ Date: Tue, 15 Jul 2025 22:19:07 -0400 Subject: [PATCH 3/7] Push can now push json file --- cli/src/Command/Queue/Push.php | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/cli/src/Command/Queue/Push.php b/cli/src/Command/Queue/Push.php index 48f32f3..d0f2fd6 100644 --- a/cli/src/Command/Queue/Push.php +++ b/cli/src/Command/Queue/Push.php @@ -15,7 +15,8 @@ class Push extends Console\Command\Command protected function configure(): void { - $this->addOption('configurations', 'c', Console\Input\InputOption::VALUE_REQUIRED | Console\Input\InputOption::VALUE_IS_ARRAY, 'Job configuration, must be in valid JSON format'); + $this->addOption('configurations', 'c', Console\Input\InputOption::VALUE_REQUIRED | Console\Input\InputOption::VALUE_IS_ARRAY, 'Job configuration options array, each job configuration must be in valid JSON format'); + $this->addOption('file', 'f', Console\Input\InputOption::VALUE_REQUIRED, 'Path to jobs configuration file with JSON array'); } protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output): int @@ -23,8 +24,8 @@ class Push extends Console\Command\Command $io = new Console\Style\SymfonyStyle($input, $output); $io->title("Pushing job"); - $configurations = $input->getOption('configurations'); - if ($configurations === null) { + $configurations = $this->getConfigurations($input); + if (count($configurations) === 0) { $io->error('Missing configurations'); return self::FAILURE; } @@ -46,4 +47,21 @@ class Push extends Console\Command\Command } return $result; } + + protected function getConfigurations(Console\Input\InputInterface $input): array + { + $configurations = []; + $filePath = $input->getOption('file'); + if ($filePath !== null and file_exists($filePath)) { + $json = file_get_contents($filePath); + if (json_validate($json)) { + $configurations = array_map(fn($configArray) => json_encode($configArray), json_decode($json, true)); + } + } + $configOptions = $input->getOption('configurations'); + if ($configOptions !== null) { + $configurations = array_merge($configurations, array_filter($configOptions, fn($config) => json_validate($config))); + } + return $configurations; + } } From c512d7a79acedaec2dfcb6f2240f3cb9827ca58f Mon Sep 17 00:00:00 2001 From: Juan Pablo Vial Date: Tue, 15 Jul 2025 22:56:48 -0400 Subject: [PATCH 4/7] FIX: sacar configuracion de json --- cli/src/Command/Queue/Push.php | 76 +++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/cli/src/Command/Queue/Push.php b/cli/src/Command/Queue/Push.php index d0f2fd6..d0bfa63 100644 --- a/cli/src/Command/Queue/Push.php +++ b/cli/src/Command/Queue/Push.php @@ -2,13 +2,14 @@ namespace Incoviba\Command\Queue; use Throwable; +use Psr\Log\LoggerInterface; use Symfony\Component\Console; use Incoviba\Service; #[Console\Attribute\AsCommand(name: 'queue:push', description: 'Push a job to the queue')] class Push extends Console\Command\Command { - public function __construct(protected Service\Job $jobService, ?string $name = null) + public function __construct(protected LoggerInterface $logger, protected Service\Job $jobService, ?string $name = null) { parent::__construct($name); } @@ -16,7 +17,7 @@ class Push extends Console\Command\Command protected function configure(): void { $this->addOption('configurations', 'c', Console\Input\InputOption::VALUE_REQUIRED | Console\Input\InputOption::VALUE_IS_ARRAY, 'Job configuration options array, each job configuration must be in valid JSON format'); - $this->addOption('file', 'f', Console\Input\InputOption::VALUE_REQUIRED, 'Path to jobs configuration file with JSON array'); + $this->addOption('files', 'f', Console\Input\InputOption::VALUE_REQUIRED | Console\Input\InputOption::VALUE_IS_ARRAY, 'Paths to jobs configurations files with JSON array content'); } protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output): int @@ -49,19 +50,72 @@ class Push extends Console\Command\Command } protected function getConfigurations(Console\Input\InputInterface $input): array + { + return [ + ...$this->getFilesConfigurations($input), + ...$this->getOptionConfigurations($input), + ]; + } + protected function getFilesConfigurations(Console\Input\InputInterface $input): array { $configurations = []; - $filePath = $input->getOption('file'); - if ($filePath !== null and file_exists($filePath)) { - $json = file_get_contents($filePath); - if (json_validate($json)) { - $configurations = array_map(fn($configArray) => json_encode($configArray), json_decode($json, true)); - } + $files = $input->getOption('files'); + if ($files === null) { + return $configurations; } - $configOptions = $input->getOption('configurations'); - if ($configOptions !== null) { - $configurations = array_merge($configurations, array_filter($configOptions, fn($config) => json_validate($config))); + foreach ($files as $filePath) { + if (!file_exists($filePath)) { + continue; + } + $configurations = array_merge($configurations, $this->getFileConfigurations($filePath)); } return $configurations; } + protected function getFileConfigurations(string $filePath): array + { + $configurations = []; + if (!file_exists($filePath)) { + return $configurations; + } + $json = file_get_contents($filePath); + if (!json_validate($json)) { + return $configurations; + } + $tmp = json_decode($json, true); + foreach ($tmp as $config) { + try { + $configurations []= $this->processConfiguration(json_encode($config)); + } catch (Throwable $exception) { + $this->logger->warning($exception->getMessage(), ['exception' => $exception, 'config' => $config]); + } + } + return $configurations; + } + protected function getOptionConfigurations(Console\Input\InputInterface $input): array + { + $configurations = []; + $configOptions = $input->getOption('configurations'); + if ($configOptions === null) { + return $configurations; + } + foreach ($configOptions as $config) { + try { + $configurations []= $this->processConfiguration($config); + } catch (Throwable $exception) { + $this->logger->warning($exception->getMessage(), ['exception' => $exception, 'config' => $config]); + } + } + return $configurations; + } + protected function processConfiguration(string $configuration): string + { + $json = json_decode($configuration, true); + if (!array_key_exists('type', $json) and !array_key_exists('configuration', $json)) { + throw new Console\Exception\InvalidArgumentException('Missing type or configuration key in JSON'); + } + if (array_key_exists('type', $json)) { + return json_encode($json); + } + return json_encode($json['configuration']); + } } From d3d69408425ee7a5f3353f07afb11353ded91a9b Mon Sep 17 00:00:00 2001 From: Juan Pablo Vial Date: Tue, 15 Jul 2025 23:07:32 -0400 Subject: [PATCH 5/7] Add retries when job execute failed --- app/src/Service/Queue.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/src/Service/Queue.php b/app/src/Service/Queue.php index 755e6dc..3d082a7 100644 --- a/app/src/Service/Queue.php +++ b/app/src/Service/Queue.php @@ -59,6 +59,8 @@ class Queue extends Ideal\Service try { if (!$worker->execute($job)) { $this->logger->debug("Could not execute job {$job->id}"); + $job->retries++; + $this->jobService->update($job); return false; } if (!$this->jobService->execute($job)) { @@ -67,6 +69,12 @@ class Queue extends Ideal\Service } } catch (Exception $exception) { $this->logger->warning("Could not run job {$job->id}", ['exception' => $exception]); + $job->retries++; + try { + $this->jobService->update($job); + } catch (Update $exception) { + $this->logger->error($exception->getMessage(), ['job' => $job, 'exception' => $exception]); + } return false; } return true; From fd134804a206db97306cab073dafca7f749cbbf0 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vial Date: Tue, 15 Jul 2025 23:18:09 -0400 Subject: [PATCH 6/7] Queue runs batch of jobs --- cli/src/Command/Queue.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/src/Command/Queue.php b/cli/src/Command/Queue.php index 302ee0c..38df4d4 100644 --- a/cli/src/Command/Queue.php +++ b/cli/src/Command/Queue.php @@ -42,7 +42,14 @@ class Queue extends Command } $io->title("[{$now->format('Y-m-d H:i:s e')}] Running Queue..."); - return $this->runJob(); + $results = []; + for ($i = 0; $i < $this->batchSize; $i++) { + if ($this->jobService->getPending() === 0) { + break; + } + $results []= $this->runJob(); + } + return count(array_filter($results, fn ($result) => $result === self::FAILURE)) === 0 ? self::SUCCESS : self::FAILURE; } protected array $sections; From b211af47d0c32d63fa8c0e9a7ef476f44eb8084c Mon Sep 17 00:00:00 2001 From: Juan Pablo Vial Date: Tue, 15 Jul 2025 23:30:53 -0400 Subject: [PATCH 7/7] Permissions --- app/bin/console | 0 app/bin/integration_tests | 0 app/bin/performance_tests | 0 app/bin/unit_tests | 0 cli/start_command | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 app/bin/console mode change 100644 => 100755 app/bin/integration_tests mode change 100644 => 100755 app/bin/performance_tests mode change 100644 => 100755 app/bin/unit_tests mode change 100644 => 100755 cli/start_command diff --git a/app/bin/console b/app/bin/console old mode 100644 new mode 100755 diff --git a/app/bin/integration_tests b/app/bin/integration_tests old mode 100644 new mode 100755 diff --git a/app/bin/performance_tests b/app/bin/performance_tests old mode 100644 new mode 100755 diff --git a/app/bin/unit_tests b/app/bin/unit_tests old mode 100644 new mode 100755 diff --git a/cli/start_command b/cli/start_command old mode 100644 new mode 100755