From 88f91c4bd52b8b945b7b6f3266fbf59d1e62549f Mon Sep 17 00:00:00 2001 From: Aldarien Date: Mon, 12 Jun 2023 21:14:07 -0400 Subject: [PATCH] Jobs setup --- api/common/Controller/Attachments.php | 64 +++---- api/common/Controller/Base.php | 5 +- api/common/Controller/Jobs.php | 66 +++++-- api/common/Controller/Messages.php | 56 +++--- .../Exception/Request/Auth/Forbidden.php | 15 ++ .../Exception/Request/Auth/Unauthorized.php | 15 ++ api/common/Middleware/Auth.php | 35 ++-- api/common/Middleware/Messages.php | 24 --- api/common/Service/Attachments.php | 2 +- api/common/Service/Auth.php | 33 ++++ api/common/Service/Jobs.php | 65 ++++--- api/common/Service/Messages.php | 16 +- api/docker-compose.yml | 2 +- api/nginx.conf | 15 +- api/resources/routes/02_attachments.php | 5 +- api/resources/routes/02_messages.php | 6 +- api/resources/routes/03_jobs.php | 15 ++ api/setup/app.php | 7 +- api/setup/middleware/04_db.php | 1 - api/setup/setups/02_services.php | 3 + api/setup/setups/03_factories.php | 4 +- api/setup/setups/97_auth.php | 2 +- api/setup/setups/98_log.php | 3 +- api/src/Model/Attachment.php | 4 +- api/src/Model/Job.php | 24 +-- api/src/Model/State/Job.php | 2 +- api/src/Repository/Job.php | 19 +- api/src/Repository/State/Job.php | 2 +- cli/common/Command/Jobs/Check.php | 35 +--- cli/common/Command/Jobs/Execute.php | 41 +++++ cli/common/Command/Mailboxes/Check.php | 31 +--- cli/common/Command/Messages/Grab.php | 31 +--- .../Exception/Response/EmptyResponse.php | 16 ++ .../Exception/Response/MissingResponse.php | 15 ++ cli/common/Middleware/Logging.php | 38 ---- cli/common/Service/Attachments.php | 114 ++++++++++++ cli/common/Service/Communicator.php | 21 +-- cli/common/Service/Jobs.php | 71 ++++++++ cli/common/Service/Mailboxes.php | 61 +++++++ cli/common/Service/Messages.php | 43 +++++ cli/common/Wrapper/Application.php | 1 - cli/crontab | 10 +- cli/docker-compose.yml | 1 + cli/resources/commands/03_jobs.php | 1 + cli/setup/middleware/98_log.php | 2 - cli/setup/settings/01_env.php | 2 + cli/setup/setups/04_commands.php | 17 +- cli/setup/setups/98_log.php | 3 +- emails-2022-11-30-03-59-49.sql | 94 ---------- ui/common/Controller/Jobs.php | 14 ++ ui/common/Middleware/Logging.php | 2 +- ui/nginx.conf | 5 +- ui/public/index.php | 8 +- ui/resources/routes/03_jobs.php | 4 + ui/resources/views/emails/messages.blade.php | 43 +++-- ui/resources/views/home.blade.php | 6 +- ui/resources/views/jobs/base.blade.php | 14 ++ ui/resources/views/jobs/list.blade.php | 165 ++++++++++++++++++ .../views/layout/body/header/navbar.blade.php | 1 + ui/setup/setups/98_log.php | 40 +++-- 60 files changed, 965 insertions(+), 495 deletions(-) create mode 100644 api/common/Exception/Request/Auth/Forbidden.php create mode 100644 api/common/Exception/Request/Auth/Unauthorized.php delete mode 100644 api/common/Middleware/Messages.php create mode 100644 api/common/Service/Auth.php create mode 100644 api/resources/routes/03_jobs.php create mode 100644 cli/common/Command/Jobs/Execute.php create mode 100644 cli/common/Exception/Response/EmptyResponse.php create mode 100644 cli/common/Exception/Response/MissingResponse.php delete mode 100644 cli/common/Middleware/Logging.php create mode 100644 cli/common/Service/Attachments.php create mode 100644 cli/common/Service/Jobs.php create mode 100644 cli/common/Service/Mailboxes.php create mode 100644 cli/common/Service/Messages.php delete mode 100644 cli/setup/middleware/98_log.php delete mode 100644 emails-2022-11-30-03-59-49.sql create mode 100644 ui/common/Controller/Jobs.php create mode 100644 ui/resources/routes/03_jobs.php create mode 100644 ui/resources/views/jobs/base.blade.php create mode 100644 ui/resources/views/jobs/list.blade.php diff --git a/api/common/Controller/Attachments.php b/api/common/Controller/Attachments.php index 68d3dd6..d95d028 100644 --- a/api/common/Controller/Attachments.php +++ b/api/common/Controller/Attachments.php @@ -1,19 +1,20 @@ toArray(); @@ -24,35 +25,7 @@ class Attachments ]; return $this->withJson($response, $output); } - public function grab(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Jobs $jobsService): ResponseInterface - { - $body = $request->getBody(); - $json = \Safe\json_decode($body->getContents()); - if (!isset($json->messages)) { - throw new MissingArgument('messages', 'array', 'message UIDs'); - } - $output = [ - 'messages' => $json->messages, - 'total' => count($json->messages), - 'saved' => [ - 'attachments' => [], - 'total' => 0 - ] - ]; - foreach ($json->messages as $message_id) { - if (!$jobsService->isPending($message_id)) { - continue; - } - if ($service->grab($message_id)) { - $job = $jobsService->find($message_id); - $jobsService->execute($job->getId()); - $output['saved']['attachments'] []= $job->toArray(); - $output['saved']['total'] ++; - } - } - return $this->withJson($response, $output); - } - public function get(ServerRequestInterface $request, ResponseInterface $response, Service $service, LoggerInterface $logger, int $attachment_id): ResponseInterface + public function get(ServerRequestInterface $request, ResponseInterface $response, Service\Attachments $service, LoggerInterface $logger, int $attachment_id): ResponseInterface { $attachment = $service->getRepository()->fetchById($attachment_id); @@ -61,4 +34,25 @@ class Attachments $response->getBody()->write($service->getFile($attachment_id)); return $response; } -} \ No newline at end of file + public function grab(ServerRequestInterface $request, ResponseInterface $response, Service\Attachments $service, Service\Messages $messagesService): ResponseInterface + { + $body = $request->getBody(); + $json = json_decode($body->getContents()); + if (!isset($json->messages)) { + throw new MissingArgument('messages', 'array', 'Messages UIDs'); + } + $output = [ + 'messages' => $json->messages, + 'attachments' => [], + 'total' => 0 + ]; + foreach ($json->messages as $message_uid) { + $message = $messagesService->getLocalMessage($message_uid); + $attachments = $service->grab($message->getId()); + $output['attachments'] = array_merge($output['attachments'], $attachments); + $output['total'] += count($attachments); + } + + return $this->withJson($response, $output); + } +} diff --git a/api/common/Controller/Base.php b/api/common/Controller/Base.php index 8611788..4b8013f 100644 --- a/api/common/Controller/Base.php +++ b/api/common/Controller/Base.php @@ -12,7 +12,8 @@ class Base public function __invoke(ServerRequestInterface $request, ResponseInterface $response): ResponseInterface { return $this->withJson($response, [ - 'version' => '1.0.0' + 'version' => '1.0.0', + 'app' => 'emails' ]); } -} \ No newline at end of file +} diff --git a/api/common/Controller/Jobs.php b/api/common/Controller/Jobs.php index 437555f..5979fdc 100644 --- a/api/common/Controller/Jobs.php +++ b/api/common/Controller/Jobs.php @@ -1,7 +1,6 @@ getRepository()->fetchAll(); + return $this->withJson($response, compact('jobs')); + } + public function schedule(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface { $body = $request->getBody(); $json = json_decode($body->getContents()); - if (!isset($json->messages)) { - throw new MissingArgument('messages', 'array', 'messages ids'); + if (!isset($json->jobs)) { + throw new MissingArgument('jobs', 'array', 'job commands with arguments'); } $output = [ - 'messages' => $json->messages, - 'total' => count($json->messages), + 'jobs' => $json->jobs, + 'total' => count($json->jobs), 'scheduled' => 0 ]; - foreach ($json->messages as $message_id) { - if ($service->schedule($message_id)) { - $message = $messagesService->getRepository()->fetchById($message_id); - $message->doesHaveScheduledDownloads(); - $messagesService->getRepository()->save($message); + foreach ($json->jobs as $job) { + if ($service->queue($job->command, $job->arguments)) { $output['scheduled'] ++; } } @@ -37,12 +38,47 @@ class Jobs } public function pending(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface { - $pending = array_map(function(Job $job) { - return $job->toArray(); - }, $service->getPending()); + $pending = $service->getPending(); $output = [ 'total' => count($pending), - 'pending' => $pending + 'jobs' => $pending + ]; + return $this->withJson($response, $output); + } + public function pendingCommands(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface + { + $body = $response->getBody(); + $json = json_decode($body->getContents()); + if (!isset($json->commands)) { + throw new MissingArgument('commands', 'array', 'job commands'); + } + $output = [ + 'commands' => $json->commands, + 'total' => count($json->commands), + 'pending' => [] + ]; + foreach ($json->commands as $command) { + $pending = $service->getPendingByCommand($command); + if (count($pending) === 0) { + continue; + } + $output['pending'][$command] = $pending; + } + return $this->withJson($response, $output); + } + public function finish(ServerRequestInterface $request, ResponseInterface $response, Service $service, $job_id): ResponseInterface + { + $output = [ + 'job_id' => $job_id, + 'status' => $service->finish($job_id) + ]; + return $this->withJson($response, $output); + } + public function failed(ServerRequestInterface $request, ResponseInterface $response, Service $service, $job_id): ResponseInterface + { + $output = [ + 'job_id' => $job_id, + 'status' => $service->failed($job_id) ]; return $this->withJson($response, $output); } diff --git a/api/common/Controller/Messages.php b/api/common/Controller/Messages.php index db86073..7b0d387 100644 --- a/api/common/Controller/Messages.php +++ b/api/common/Controller/Messages.php @@ -1,11 +1,11 @@ getMailboxes()->get($mailbox_id); $messages = array_map(function(Message $message) { @@ -37,7 +37,7 @@ class Messages ]; return $this->withJson($response, $output); } - public function valid(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Attachments $attachments, int $mailbox_id): ResponseInterface + public function valid(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, Service\Attachments $attachments, int $mailbox_id): ResponseInterface { $mailbox = $service->getMailboxes()->get($mailbox_id); $messages = array_values(array_filter(array_map(function(Message $message) use ($service, $attachments) { @@ -63,32 +63,48 @@ class Messages ]; return $this->withJson($response, $output); } - public function get(ServerRequestInterface $request, ResponseInterface $response, Service $service, int $message_id): ResponseInterface + public function get(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, int $message_id): ResponseInterface { $message = $service->getRepository()->fetchById($message_id); return $this->withJson($response, ['message' => $message->toArray()]); } - public function grab(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Attachments $attachmentsService): ResponseInterface + public function grab(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, \ProVM\Common\Service\Attachments $attachmentsService, $mailbox_id): ResponseInterface + { + $output = [ + 'mailbox_id' => $mailbox_id, + 'messages' => [], + 'count' => 0 + ]; + $mailbox = $service->getMailboxes()->get($mailbox_id); + $messages = $service->grab($mailbox->getName()); + foreach ($messages as $message_id) { + $message = $service->getLocalMessage($message_id); + if ($message->hasValidAttachments()) { + $attachmentsService->create($message->getId()); + } + } + $output['messages'] = $messages; + $output['count'] = count($messages); + return $this->withJson($response, $output); + } + public function schedule(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, Service\Jobs $jobsService): ResponseInterface { $body = $request->getBody(); - $json = json_decode($body->getContents()); - if (!isset($json->mailboxes)) { - throw new MissingArgument('mailboxes', 'array', 'mailboxes names'); + $json = json_decode($body); + if (!isset($json->messages)) { + throw new MissingArgument('messages', 'array', 'messages IDs'); } $output = [ - 'mailboxes' => $json->mailboxes, - 'messages' => [], - 'message_count' => 0 + 'messages' => $json->messages, + 'scheduled' => 0 ]; - foreach ($json->mailboxes as $mailbox_name) { - $messages = $service->grab($mailbox_name); - foreach ($messages as $message) { - if ($message->hasValidAttachments()) { - $attachmentsService->create($message); - } + foreach ($json->messages as $message_id) { + if ($jobsService->queue('attachments:grab', [$message_id])) { + $message = $service->getRepository()->fetchById($message_id); + $message->doesHaveScheduledDownloads(); + $service->getRepository()->save($message); + $output['scheduled'] ++; } - $output['messages'] = array_merge($output['messages'], $messages); - $output['message_count'] += count($messages); } return $this->withJson($response, $output); } diff --git a/api/common/Exception/Request/Auth/Forbidden.php b/api/common/Exception/Request/Auth/Forbidden.php new file mode 100644 index 0000000..9c24549 --- /dev/null +++ b/api/common/Exception/Request/Auth/Forbidden.php @@ -0,0 +1,15 @@ +setResponseFactory($factory); $this->setLogger($logger); - $this->setAPIKey($api_key); } protected ResponseFactoryInterface $factory; protected LoggerInterface $logger; - protected string $api_key; public function getResponseFactory(): ResponseFactoryInterface { @@ -28,10 +28,6 @@ class Auth { return $this->logger; } - public function getAPIKey(): string - { - return $this->api_key; - } public function setResponseFactory(ResponseFactoryInterface $factory): Auth { @@ -43,30 +39,23 @@ class Auth $this->logger = $logger; return $this; } - public function setAPIKey(string $key): Auth - { - $this->api_key = $key; - return $this; - } public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface { if ($request->getMethod() === 'OPTIONS') { return $handler->handle($request); } - $auths = $request->getHeader('Authorization'); - foreach ($auths as $auth) { - if (str_contains($auth, 'Bearer')) { - $key = str_replace('Bearer ', '', $auth); - if (sha1($this->getAPIKey()) === $key) { - return $handler->handle($request); - } + try { + if ($this->service->validate($request)) { + return $handler->handle($request); } + } catch (Unauthorized $e) { + $response = $this->getResponseFactory()->createResponse($e->getCode()); + $response->getBody()->write(json_encode(['error' => $e->getCode(), 'message' => $e->getMessage()])); } - $this->getLogger()->debug(sha1($this->getAPIKey())); - $response = $this->getResponseFactory()->createResponse(401); - $response->getBody()->write(\Safe\json_encode(['error' => 401, 'message' => 'Incorrect token'])); + $response = $this->getResponseFactory()->createResponse(413); + $response->getBody()->write(\Safe\json_encode(['error' => 413, 'message' => 'Incorrect token'])); return $response ->withHeader('Content-Type', 'application/json'); } -} \ No newline at end of file +} diff --git a/api/common/Middleware/Messages.php b/api/common/Middleware/Messages.php deleted file mode 100644 index 13c13cb..0000000 --- a/api/common/Middleware/Messages.php +++ /dev/null @@ -1,24 +0,0 @@ -service->checkSchedule(); - } catch (BlankResult $e) { - $this->logger->notice($e); - } - return $handler->handle($request); - } -} diff --git a/api/common/Service/Attachments.php b/api/common/Service/Attachments.php index 1487b77..c5b6db9 100644 --- a/api/common/Service/Attachments.php +++ b/api/common/Service/Attachments.php @@ -117,7 +117,7 @@ class Attachments extends Base continue; } $name = $file->getBasename(".{$file->getExtension()}"); - list($subject, $date, $filename) = explode(' - ', $name); + list($date, $subject, $filename) = explode(' - ', $name); try { $message = $this->getMessages()->find($subject, $date)[0]; $filename = "{$filename}.{$file->getExtension()}"; diff --git a/api/common/Service/Auth.php b/api/common/Service/Auth.php new file mode 100644 index 0000000..7442981 --- /dev/null +++ b/api/common/Service/Auth.php @@ -0,0 +1,33 @@ +getHeaderKey($request); + if (sha1($this->api_key) === $key) { + return true; + } + return false; + } + + protected function getHeaderKey(ServerRequestInterface $request): string + { + if (!$request->hasHeader('Authorization')) { + throw new Unauthorized(); + } + $auths = $request->getHeader('Authorization'); + foreach ($auths as $auth) { + if (str_contains($auth, 'Bearer')) { + return str_replace('Bearer ', '', $auth); + } + } + throw new Unauthorized(); + } +} diff --git a/api/common/Service/Jobs.php b/api/common/Service/Jobs.php index 13ef3d8..f9822c4 100644 --- a/api/common/Service/Jobs.php +++ b/api/common/Service/Jobs.php @@ -1,40 +1,49 @@ setRepository($repository); } - protected Job $repository; + protected Repository\Job $repository; - public function getRepository(): Job + public function getRepository(): Repository\Job { return $this->repository; } - public function setRepository(Job $repository): Jobs + public function setRepository(Repository\Job $repository): Jobs { $this->repository = $repository; return $this; } - public function schedule(int $message_id): bool + public function queue(string $command, ?array $arguments = null): bool { $data = [ - 'message_id' => $message_id, - 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s') + 'command' => $command, + 'arguments' => implode(' ', $arguments) ]; try { $job = $this->getRepository()->create($data); $this->getRepository()->save($job); + $data = [ + 'job_id' => $job->getId(), + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Pending + ]; + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); return true; } catch (PDOException $e) { return false; @@ -44,28 +53,42 @@ class Jobs extends Base { return $this->getRepository()->fetchAllPending(); } - public function isPending(int $message_id): bool + public function getPendingByCommand(string $command): array { try { - $this->getRepository()->fetchPendingByMessage($message_id); - return true; + return $this->getRepository()->fetchAllPendingByCommand($command); } catch (BlankResult $e) { - return false; + return []; } } - public function find(int $message_id): \ProVM\Emails\Model\Job - { - return $this->getRepository()->fetchPendingByMessage($message_id); - } - public function execute(int $job_id): bool + public function finish(int $job_id): bool { + $data = [ + 'job_id' => $job_id, + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Executed + ]; try { - $job = $this->getRepository()->fetchById($job_id); - $job->wasExecuted(); - $this->getRepository()->save($job); + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); return true; } catch (PDOException $e) { return false; } } -} \ No newline at end of file + public function failed(int $job_id): bool + { + $data = [ + 'job_id' => $job_id, + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Failure + ]; + try { + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); + return true; + } catch (PDOException $e) { + return false; + } + } +} diff --git a/api/common/Service/Messages.php b/api/common/Service/Messages.php index a0cd5e0..ac52463 100644 --- a/api/common/Service/Messages.php +++ b/api/common/Service/Messages.php @@ -154,7 +154,6 @@ class Messages extends Base $message->doesHaveValidAttachments(); } } - error_log(json_encode(compact('message')).PHP_EOL,3,'/logs/debug'); $this->getRepository()->save($message); return true; } catch (PDOException $e) { @@ -196,20 +195,7 @@ class Messages extends Base $registered = $this->getMailboxes()->getRegistered(); foreach ($registered as $mailbox) { if (!$this->getMailboxes()->isUpdated($mailbox)) { - $this->logger->info("Updating messages from {$mailbox->getName()}"); - $this->grab($mailbox->getName()); - } - } - } - public function checkSchedule(): void - { - $messages = $this->getRepository()->fetchAll(); - foreach ($messages as $message) { - if ($message->hasAttachments() and $message->hasValidAttachments() and !$message->hasDownloadedAttachments() and !$message->hasScheduledDownloads()) { - if ($this->getJobsService()->schedule($message->getId())) { - $message->doesHaveDownloadedAttachments(); - $this->getRepository()->save($message); - } + $this->getJobsService()->queue('messages:grab', [$mailbox->getId()]); } } } diff --git a/api/docker-compose.yml b/api/docker-compose.yml index 173b1be..0f71511 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -7,7 +7,7 @@ services: - ${API_PATH:-.}:/app/api - ${API_PATH:-.}/nginx.conf:/etc/nginx/conf.d/api.conf ports: - - "${API_PORT:-8080}:8080" + - "${API_PORT:-8080}:81" api: profiles: - api diff --git a/api/nginx.conf b/api/nginx.conf index 6d22e8e..6a50944 100644 --- a/api/nginx.conf +++ b/api/nginx.conf @@ -1,15 +1,15 @@ server { - listen 0.0.0.0:8080; + listen 81; root /app/api/public; index index.php index.html index.htm; - access_log /var/logs/nginx/access.log; - error_log /var/logs/nginx/error.log; + access_log /var/logs/nginx/api.access.log; + error_log /var/logs/nginx/api.error.log; location / { - try_files $uri $uri/ /index.php?$query_string; + try_files $uri /index.php$is_args$args; } - location ~ \.php$ { + location ~ \.php { if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Headers' 'Authorization,Accept,Origin,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; @@ -26,11 +26,12 @@ server { try_files $uri =404; fastcgi_split_path_info ^(.+\.php)(/.+)$; + include fastcgi_params; fastcgi_pass api:9000; fastcgi_index index.php; - include fastcgi_params; fastcgi_param REQUEST_URI $request_uri; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param SCRIPT_NAME $fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; } -} \ No newline at end of file +} diff --git a/api/resources/routes/02_attachments.php b/api/resources/routes/02_attachments.php index 9b45b9d..8c53514 100644 --- a/api/resources/routes/02_attachments.php +++ b/api/resources/routes/02_attachments.php @@ -2,9 +2,8 @@ use ProVM\Common\Controller\Attachments; $app->group('/attachments', function($app) { - $app->put('/grab', [Attachments::class, 'grab']); - $app->get('/pending', [Attachments::class, 'pending']); - $app->post('/decrypt', [Attachments::class, 'decrypt']); + $app->put('/grab[/]', [Attachments::class, 'grab']); + $app->get('/pending[/]', [Attachments::class, 'pending']); $app->get('[/]', Attachments::class); }); $app->group('/attachment/{attachment_id}', function($app) { diff --git a/api/resources/routes/02_messages.php b/api/resources/routes/02_messages.php index ed080d5..4d10259 100644 --- a/api/resources/routes/02_messages.php +++ b/api/resources/routes/02_messages.php @@ -4,9 +4,9 @@ use ProVM\Common\Controller\Jobs; $app->group('/messages', function($app) { $app->put('/grab', [Messages::class, 'grab']); - $app->put('/schedule', [Jobs::class, 'schedule']); - $app->get('/pending', [Jobs::class, 'pending']); + $app->put('/schedule', [Messages::class, 'schedule']); + //$app->get('/pending', [Jobs::class, 'pending']); }); $app->group('/message/{message_id}', function($app) { $app->get('[/]', [Messages::class, 'get']); -}); \ No newline at end of file +}); diff --git a/api/resources/routes/03_jobs.php b/api/resources/routes/03_jobs.php new file mode 100644 index 0000000..1753d96 --- /dev/null +++ b/api/resources/routes/03_jobs.php @@ -0,0 +1,15 @@ +group('/jobs', function($app) { + $app->group('/pending', function($app) { + $app->put('/command[/]', [Jobs::class, 'pendingCommands']); + $app->get('[/]', [Jobs::class, 'pending']); + }); + $app->post('/schedule[/]', [Jobs::class, 'schedule']); + $app->get('[/]', Jobs::class); +}); +$app->group('/job/{job_id}', function($app) { + $app->get('/finish[/]', [Jobs::class, 'finish']); + $app->get('/failed[/]', [Jobs::class, 'failed']); +}); diff --git a/api/setup/app.php b/api/setup/app.php index 7367296..0ae6f29 100644 --- a/api/setup/app.php +++ b/api/setup/app.php @@ -1,7 +1,10 @@ addDefinitions($file->getRealPath()); } } -$app = \DI\Bridge\Slim\Bridge::create($builder->build()); +$app = Bridge::create($builder->build()); $folder = implode(DIRECTORY_SEPARATOR, [ __DIR__, diff --git a/api/setup/middleware/04_db.php b/api/setup/middleware/04_db.php index 9d831e8..b71467b 100644 --- a/api/setup/middleware/04_db.php +++ b/api/setup/middleware/04_db.php @@ -1,5 +1,4 @@ add($app->getContainer()->get(ProVM\Common\Middleware\Attachments::class)); -$app->add($app->getContainer()->get(ProVM\Common\Middleware\Messages::class)); $app->add($app->getContainer()->get(ProVM\Common\Middleware\Mailboxes::class)); $app->add($app->getContainer()->get(ProVM\Common\Middleware\Install::class)); diff --git a/api/setup/setups/02_services.php b/api/setup/setups/02_services.php index 2a09318..4348151 100644 --- a/api/setup/setups/02_services.php +++ b/api/setup/setups/02_services.php @@ -33,5 +33,8 @@ return [ $container->get(ProVM\Common\Factory\Model::class), $container->get('model_list') ); + }, + ProVM\Common\Service\Auth::class => function(ContainerInterface $container) { + return new ProVM\Common\Service\Auth($container->get('api_key')); } ]; diff --git a/api/setup/setups/03_factories.php b/api/setup/setups/03_factories.php index 281eafa..49b605e 100644 --- a/api/setup/setups/03_factories.php +++ b/api/setup/setups/03_factories.php @@ -8,9 +8,11 @@ return [ 'Mailbox' => ProVM\Emails\Repository\Mailbox::class, 'Message' => ProVM\Emails\Repository\Message::class, 'Attachment' => ProVM\Emails\Repository\Attachment::class, + 'Job' => ProVM\Emails\Repository\Job::class, "State\\Mailbox" => ProVM\Emails\Repository\State\Mailbox::class, "State\\Message" => ProVM\Emails\Repository\State\Message::class, - "State\\Attachment" => ProVM\Emails\Repository\State\Attachment::class + "State\\Attachment" => ProVM\Emails\Repository\State\Attachment::class, + "State\\Job" => ProVM\Emails\Repository\State\Job::class, ]); } ]; diff --git a/api/setup/setups/97_auth.php b/api/setup/setups/97_auth.php index 03fa0f3..9476f57 100644 --- a/api/setup/setups/97_auth.php +++ b/api/setup/setups/97_auth.php @@ -6,7 +6,7 @@ return [ return new ProVM\Common\Middleware\Auth( $container->get(Nyholm\Psr7\Factory\Psr17Factory::class), $container->get(Psr\Log\LoggerInterface::class), - $container->get('api_key') + $container->get(ProVM\Common\Service\Auth::class) ); } ]; diff --git a/api/setup/setups/98_log.php b/api/setup/setups/98_log.php index 0ebc4cb..39788a4 100644 --- a/api/setup/setups/98_log.php +++ b/api/setup/setups/98_log.php @@ -11,8 +11,7 @@ return [ ]; }, 'request_log_handler' => function(ContainerInterface $container) { - return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) - ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))); }, 'request_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/api/src/Model/Attachment.php b/api/src/Model/Attachment.php index 912a2e8..9cc8683 100644 --- a/api/src/Model/Attachment.php +++ b/api/src/Model/Attachment.php @@ -95,9 +95,9 @@ class Attachment implements Model public function getFullFilename(): string { return implode(' - ', [ - $this->getMessage()->getSubject(), $this->getMessage()->getDateTime()->format('Y-m-d His'), - $this->getFilename() + $this->getMessage()->getSubject(), + $this->getFilename(), ]); } diff --git a/api/src/Model/Job.php b/api/src/Model/Job.php index 5aa26ee..7b69d93 100644 --- a/api/src/Model/Job.php +++ b/api/src/Model/Job.php @@ -25,24 +25,20 @@ class Job implements Model { return $this->arguments ?? ''; } - public function isExecuted(): bool - { - return $this->lastState()->getStatus() === State\Job::Executed; - } public function setId(int $id): Job { $this->id = $id; return $this; } - public function setCommand(string $message): Job + public function setCommand(string $command): Job { - $this->message = $message; + $this->command = $command; return $this; } - public function setArguments(string $dateTime): Job + public function setArguments(string $arguments): Job { - $this->dateTime = $dateTime; + $this->arguments = $arguments; return $this; } @@ -82,10 +78,14 @@ class Job implements Model return $this; } + public function isExecuted(): bool + { + return $this->lastState()->getStatus() === State\Job::Executed; + } public function lastState(): State\Job { if (count($this->getStates()) === 0) { - throw new Stateless($this); + throw new Stateless($this->getId()); } return $this->getStates()[array_key_last($this->getStates())]; } @@ -94,9 +94,9 @@ class Job implements Model { return [ 'id' => $this->getId(), - 'message' => $this->getMessage()->toArray(), - 'date_time' => $this->getDateTime()->format('Y-m-d H:i:s'), - 'executed' => $this->isExecuted() + 'command' => $this->getCommand(), + 'arguments' => $this->getArguments(), + 'states' => $this->getStates() ]; } public function jsonSerialize(): mixed diff --git a/api/src/Model/State/Job.php b/api/src/Model/State/Job.php index 2826908..1d1e9af 100644 --- a/api/src/Model/State/Job.php +++ b/api/src/Model/State/Job.php @@ -57,7 +57,7 @@ class Job implements Model { return [ 'id' => $this->getId(), - 'job' => $this->getJob(), + 'job_id' => $this->getJob()->getId(), 'date' => $this->getDateTime(), 'status' => $this->getStatus() ]; diff --git a/api/src/Repository/Job.php b/api/src/Repository/Job.php index e8f6be9..17c39df 100644 --- a/api/src/Repository/Job.php +++ b/api/src/Repository/Job.php @@ -15,7 +15,7 @@ class Job extends Repository { parent::__construct($connection, $logger); $this->setFactory($factory) - ->setTable('attachments_jobs'); + ->setTable('jobs'); } protected Factory\Model $factory; @@ -98,19 +98,22 @@ CREATE TABLE {$this->getTable()} ( public function fetchAllPending(): array { $query = "SELECT a.* -FROM {$this->getTable()} a - JOIN `jobs_states` b ON b.job_id = a.id +FROM `{$this->getTable()}` a + JOIN (SELECT s1.* FROM `jobs_states` s1 JOIN (SELECT MAX(id) AS id, job_id FROM `jobs_states` GROUP BY job_id) s2 ON s2.id = s1.id) b ON b.`job_id` = a.`id` WHERE b.`status` = ?"; - return $this->fetchMany($query); + return $this->fetchMany($query, [Emails\Model\State\Job::Pending]); } - public function fetchByCommandAndArguments(string $command, string $arguments): \ProVM\Emails\Model\Job + public function fetchByCommandAndArguments(string $command, string $arguments): Emails\Model\Job { $query = "SELECT * FROM {$this->getTable()} WHERE `command` = ? AND `arguments` = ?"; return $this->fetchOne($query, [$command, $arguments]); } - public function fetchPendingByMessage(int $message_id): \ProVM\Emails\Model\Job + public function fetchAllPendingByCommand(string $command): array { - $query = "SELECT * FROM {$this->getTable()} WHERE `message_id` = ? AND `executed` = 0"; - return $this->fetchOne($query, [$message_id]); + $query = "SELECT a.* +FROM `{$this->getTable()}` a + JOIN (SELECT s1.* FROM `jobs_states` s1 JOIN (SELECT MAX(id) AS id, job_id FROM `jobs_states` GROUP BY job_id) s2 ON s2.id = s1.id) b ON b.`job_id` = a.`id` +WHERE a.`command` = ? AND b.`status` = ?"; + return $this->fetchMany($query, [$command, Emails\Model\State\Job::Pending]); } } diff --git a/api/src/Repository/State/Job.php b/api/src/Repository/State/Job.php index e7f9d57..11a6699 100644 --- a/api/src/Repository/State/Job.php +++ b/api/src/Repository/State/Job.php @@ -57,7 +57,7 @@ CREATE TABLE {$this->getTable()} ( $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ?"; return $this->fetchMany($query, [$job_id]); } - public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\Job + public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\State\Job { $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ? AND `status` = ?"; return $this->fetchOne($query, [$job_id, $status]); diff --git a/cli/common/Command/Jobs/Check.php b/cli/common/Command/Jobs/Check.php index d3c9b23..5a5ecc6 100644 --- a/cli/common/Command/Jobs/Check.php +++ b/cli/common/Command/Jobs/Check.php @@ -1,14 +1,12 @@ logger->notice('Grabbing pending jobs.'); - $response = $this->communicator->get('/jobs/pending'); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return []; - } - return json_decode($body)->jobs; - } - protected function runJob($job): bool - { - $base_command = '/app/bin/emails'; - $cmd = [$base_command, $job->command]; - if ($job->arguments !== '') { - $cmd []= $job->arguments; - } - $cmd = implode(' ', $cmd); - $this->logger->notice("Running '{$cmd}'"); - $response = shell_exec($cmd); - $this->logger->info("Result: {$response}"); - return $response !== false; - } - public function execute(InputInterface $input, OutputInterface $output) { $section1 = $output->section(); @@ -53,17 +27,16 @@ class Check extends Command $io1 = new SymfonyStyle($input, $section1); $io2 = new SymfonyStyle($input, $section2); $io1->title('Checking Pending Jobs'); - $pending_jobs = $this->getPendingJobs(); + $pending_jobs = $this->service->getPending(); $notice = 'Found ' . count($pending_jobs) . ' jobs'; $io1->text($notice); - $this->logger->info($notice); if (count($pending_jobs) > 0) { $io1->section('Running Jobs'); $io1->progressStart(count($pending_jobs)); foreach ($pending_jobs as $job) { $section2->clear(); $io2->text("Running {$job->command}"); - if ($this->runJob($job)) { + if ($this->service->run($job)) { $io2->success('Success'); } else { $io2->error('Failure'); diff --git a/cli/common/Command/Jobs/Execute.php b/cli/common/Command/Jobs/Execute.php new file mode 100644 index 0000000..0c41a4e --- /dev/null +++ b/cli/common/Command/Jobs/Execute.php @@ -0,0 +1,41 @@ +addArgument('job_id', InputArgument::REQUIRED, 'Job ID to be executed'); + } + + public function execute(InputInterface $input, OutputInterface $output) + { + $io = new SymfonyStyle($input, $output); + $job_id = $input->getArgument('job_id'); + $job = $this->service->get($job_id); + if ($this->service->run($job)) { + $io->success('Success'); + } else { + $io->error('Failed'); + } + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Mailboxes/Check.php b/cli/common/Command/Mailboxes/Check.php index c0fa6eb..eb2f94c 100644 --- a/cli/common/Command/Mailboxes/Check.php +++ b/cli/common/Command/Mailboxes/Check.php @@ -5,9 +5,8 @@ use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -use ProVM\Service\Communicator; use Symfony\Component\Console\Style\SymfonyStyle; -use function Safe\json_decode; +use ProVM\Service\Mailboxes; #[AsCommand( name: 'mailboxes:check', @@ -16,33 +15,11 @@ use function Safe\json_decode; )] class Check extends Command { - public function __construct(protected Communicator $communicator, protected int $min_check_days, string $name = null) + public function __construct(protected Mailboxes $service, string $name = null) { parent::__construct($name); } - protected function getMailboxes(): array - { - $response = $this->communicator->get('/mailboxes/registered'); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return []; - } - return json_decode($body)->mailboxes; - } - protected function checkMailbox($mailbox): bool - { - if ((new \DateTimeImmutable())->diff(new \DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { - return true; - } - $response = $this->communicator->get("/mailbox/{$mailbox->id}/check"); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return true; - } - return json_decode($body)->status; - } - public function execute(InputInterface $input, OutputInterface $output): int { $section1 = $output->section(); @@ -50,7 +27,7 @@ class Check extends Command $io1 = new SymfonyStyle($input, $section1); $io2 = new SymfonyStyle($input, $section2); $io1->title('Checking for New Messages'); - $mailboxes = $this->getMailboxes(); + $mailboxes = $this->service->getAll(); $notice = 'Found ' . count($mailboxes) . ' mailboxes'; $io1->text($notice); if (count($mailboxes) > 0) { @@ -59,7 +36,7 @@ class Check extends Command foreach ($mailboxes as $mailbox) { $section2->clear(); $io2->text("Checking {$mailbox->name}"); - if ($this->checkMailbox($mailbox)) { + if ($this->service->check($mailbox)) { $io2->success("Found new emails in {$mailbox->name}"); } else { $io2->info("No new emails in {$mailbox->name}"); diff --git a/cli/common/Command/Messages/Grab.php b/cli/common/Command/Messages/Grab.php index 89a85f9..1be0fd0 100644 --- a/cli/common/Command/Messages/Grab.php +++ b/cli/common/Command/Messages/Grab.php @@ -7,19 +7,17 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; -use ProVM\Service\Communicator; -use function Safe\json_decode; +use ProVM\Service\Mailboxes; #[AsCommand( name: 'messages:grab', - description: 'Run grab messages job for registered mailboxes', + description: 'Run grab messages job for mailbox', hidden: false )] class Grab extends Command { - public function __construct(Communicator $communicator, string $name = null) + public function __construct(protected Mailboxes $service, string $name = null) { - $this->setCommunicator($communicator); parent::__construct($name); } protected function configure() @@ -27,27 +25,6 @@ class Grab extends Command $this->addArgument('mailbox_id', InputArgument::REQUIRED, 'Mailbox ID to grab emails'); } - protected Communicator $communicator; - public function getCommunicator(): Communicator - { - return $this->communicator; - } - public function setCommunicator(Communicator $communicator): Grab - { - $this->communicator = $communicator; - return $this; - } - - protected function grabMessages(int $mailbox_id): int - { - $response = $this->getCommunicator()->get("/mailbox/{$mailbox_id}/grab"); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return 0; - } - return json_decode($body)->messages->count; - } - public function execute(InputInterface $input, OutputInterface $output) { $io = new SymfonyStyle($input, $output); @@ -55,7 +32,7 @@ class Grab extends Command $mailbox_id = $input->getArgument('mailbox_id'); $io->title("Grabbing Messages for Mailbox ID {$mailbox_id}"); $io->section('Grabbing Messages'); - $count = $this->grabMessages($mailbox_id); + $count = $this->service->grabMessages($mailbox_id); $io->info("Found {$count} messages"); $io->success('Done.'); diff --git a/cli/common/Exception/Response/EmptyResponse.php b/cli/common/Exception/Response/EmptyResponse.php new file mode 100644 index 0000000..5fff71f --- /dev/null +++ b/cli/common/Exception/Response/EmptyResponse.php @@ -0,0 +1,16 @@ +setLogger($logger); - } - - protected LoggerInterface $logger; - - public function getLogger(): LoggerInterface - { - return $this->logger; - } - - public function setLogger(LoggerInterface $logger): Logging - { - $this->logger = $logger; - return $this; - } - - public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface - { - $response = $handler->handle($request); - $output = [ - 'uri' => var_export($request->getUri(), true), - 'body' => $request->getBody()->getContents() - ]; - $this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); - return $response; - } -} diff --git a/cli/common/Service/Attachments.php b/cli/common/Service/Attachments.php new file mode 100644 index 0000000..681b291 --- /dev/null +++ b/cli/common/Service/Attachments.php @@ -0,0 +1,114 @@ +logger->info('Finding all downloaded attachment files'); + $folder = '/attachments'; + $files = new \FilesystemIterator($folder); + foreach ($files as $file) { + if ($file->isDir()) { + continue; + } + yield $file->getRealPath(); + } + } + public function getAll(): array + { + if (!isset($this->attachments)) { + $this->logger->info('Grabbing all attachments'); + $response = $this->communicator->get('/attachments'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + $this->attachments = []; + return $this->attachments; + } + $this->attachments = json_decode($body)->attachments; + } + return $this->attachments; + } + public function get(int $attachment_id): object + { + $this->logger->info("Getting attachment {$attachment_id}"); + $uri = "/attachment/{$attachment_id}"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->attachment)) { + throw new MissingResponse('attachment'); + } + return $json->attachment; + } + public function find(string $filename): int + { + $this->logger->info("Finding attachment {$filename}"); + foreach ($this->getAll() as $attachment) { + if ($attachment->fullfilename === $filename) { + return $attachment->id; + } + } + throw new \Exception("{$filename} is not in the database"); + } + public function isEncrypted(string $filename): bool + { + if (!file_exists($filename)) { + throw new \InvalidArgumentException("File not found {$filename}"); + } + $escaped_filename = escapeshellarg($filename); + $cmd = "{$this->base_command} --is-encrypted {$escaped_filename}"; + exec($cmd, $output, $retcode); + return $retcode == 0; + } + public function scheduleDecrypt(int $attachment_id): bool + { + $this->logger->info("Scheduling decryption of attachment {$attachment_id}"); + $uri = "/attachment/{$attachment_id}/decrypt"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->status)) { + throw new MissingResponse('status'); + } + return $json->status; + } + public function decrypt(string $basename): bool + { + $this->logger->info("Decrypting {$basename}"); + $in_filename = implode('/', ['attachments', $basename]); + $out_filename = implode('/', ['attachments', 'decrypted', $basename]); + if (file_exists($out_filename)) { + throw new \Exception("{$basename} already decrypted"); + } + foreach ($this->passwords as $password) { + $cmd = $this->base_command . ' -password=' . escapeshellarg($password) . ' -decrypt ' . escapeshellarg($in_filename) . ' ' . escapeshellarg($out_filename); + exec($cmd, $output, $retcode); + $success = $retcode == 0; + if ($success) { + return true; + } + if (file_exists($out_filename)) { + unlink($out_filename); + } + unset($output); + } + return false; + } + +} diff --git a/cli/common/Service/Communicator.php b/cli/common/Service/Communicator.php index b22589c..c655079 100644 --- a/cli/common/Service/Communicator.php +++ b/cli/common/Service/Communicator.php @@ -4,29 +4,12 @@ namespace ProVM\Service; use HttpResponseException; use Psr\Http\Client\ClientInterface; use Psr\Http\Message\ResponseInterface; -use Psr\Log\LoggerInterface; use Safe\Exceptions\JsonException; use function Safe\json_encode; class Communicator { - public function __construct(ClientInterface $client) - { - $this->setClient($client); - } - - protected ClientInterface $client; - - public function getClient(): ClientInterface - { - return $this->client; - } - - public function setClient(ClientInterface $client): Communicator - { - $this->client = $client; - return $this; - } + public function __construct(protected ClientInterface $client) {} /** * @throws HttpResponseException @@ -52,7 +35,7 @@ class Communicator ]; $options['body'] = json_encode($body); } - return $this->handleResponse($this->getClient()->request($method, $uri, $options)); + return $this->handleResponse($this->client->request($method, $uri, $options)); } /** diff --git a/cli/common/Service/Jobs.php b/cli/common/Service/Jobs.php new file mode 100644 index 0000000..d3543ef --- /dev/null +++ b/cli/common/Service/Jobs.php @@ -0,0 +1,71 @@ +logger->info('Getting pending jobs'); + $response = $this->communicator->get('/jobs/pending'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + $json = json_decode($body); + if (!isset($json->jobs)) { + return []; + } + return $json->jobs; + } + public function get(int $job_id): object + { + $this->logger->info("Getting Job {$job_id}"); + $uri = "/job/{$job_id}"; + return $this->send($uri, 'job'); + } + public function run(object $job): bool + { + $this->logger->debug("Running Job {$job->id}"); + $base_command = '/app/bin/emails'; + $cmd = [$base_command, $job->command]; + if ($job->arguments !== '') { + $cmd []= $job->arguments; + } + $cmd = implode(' ', $cmd); + $response = shell_exec($cmd); + if ($response !== false) { + return $this->finished($job->id); + } + return $this->failure($job->id); + } + + protected function finished(int $job_id): bool + { + $uri = "/job/{$job_id}/finish"; + return $this->send($uri, 'status'); + } + protected function failure(int $job_id): bool + { + $uri = "/job/{$job_id}/failed"; + return $this->send($uri, 'status'); + } + protected function send(string $uri, string $param): mixed + { + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->{$param})) { + throw new MissingResponse($param); + } + return $json->{$param}; + } +} diff --git a/cli/common/Service/Mailboxes.php b/cli/common/Service/Mailboxes.php new file mode 100644 index 0000000..4d8eabf --- /dev/null +++ b/cli/common/Service/Mailboxes.php @@ -0,0 +1,61 @@ +logger->info('Getting all registered mailboxes'); + $response = $this->communicator->get('/mailboxes/registered'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + $json = json_decode($body); + if (!isset($json->mailboxes)) { + return []; + } + return $json->mailboxes; + } + public function check(object $mailbox): bool + { + $this->logger->info("Checking mailbox {$mailbox->id}"); + if ((new DateTimeImmutable())->diff(new DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { + return true; + } + $uri = "/mailbox/{$mailbox->id}/check"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->status)) { + throw new MissingResponse('status'); + } + return $json->status; + } + public function grabMessages(int $mailbox_id): int + { + $this->logger->info("Grabbing messages for {$mailbox_id}"); + $uri = "/mailbox/{$mailbox_id}/messages/grab"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + $json = json_decode($body); + if (!isset($json->count)) { + return 0; + } + return $json->count; + } +} diff --git a/cli/common/Service/Messages.php b/cli/common/Service/Messages.php new file mode 100644 index 0000000..3854dbd --- /dev/null +++ b/cli/common/Service/Messages.php @@ -0,0 +1,43 @@ +logger->info("Getting message {$message_id}"); + $uri = "/message/{$message_id}"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->message)) { + throw new MissingResponse('message'); + } + return $json->message; + } + public function grabAttachments(string $message_uid): int + { + $this->logger->info("Grabbing attachments for message UID {$message_uid}"); + $uri = '/attachments/grab'; + $response = $this->communicator->put($uri, ['messages' => [$message_uid]]); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + $json = json_decode($body); + if (!isset($json->total)) { + return 0; + } + return $json->total; + } +} diff --git a/cli/common/Wrapper/Application.php b/cli/common/Wrapper/Application.php index c27dbf4..7ffac08 100644 --- a/cli/common/Wrapper/Application.php +++ b/cli/common/Wrapper/Application.php @@ -3,7 +3,6 @@ namespace ProVM\Wrapper; use Psr\Container\ContainerInterface; use Symfony\Component\Console\Application as Base; - class Application extends Base { public function __construct(ContainerInterface $container, string $name = 'UNKNOWN', string $version = 'UNKNOWN') diff --git a/cli/crontab b/cli/crontab index fc8ab81..b567b58 100644 --- a/cli/crontab +++ b/cli/crontab @@ -1,10 +1,10 @@ # minutes hour day_of_month month day_of_week command -#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log -#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log +#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log +#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log # Pending jobs every minute -1 * * * * /app/bin/emails jobs:pending >> /logs/jobs.log +* * * * * /app/bin/emails jobs:check >> /logs/jobs.log # Check mailboxes for new emails every weekday -0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log +0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log # Check attachments every weekday -0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log +0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log diff --git a/cli/docker-compose.yml b/cli/docker-compose.yml index ea1ec32..e86b638 100644 --- a/cli/docker-compose.yml +++ b/cli/docker-compose.yml @@ -12,5 +12,6 @@ services: - .key.env volumes: - ${CLI_PATH:-.}/:/app + - ${CLI_PATH}/crontab:/var/spool/cron/crontabs/root - ${LOGS_PATH}/cli:/logs - ${ATT_PATH}:/attachments diff --git a/cli/resources/commands/03_jobs.php b/cli/resources/commands/03_jobs.php index bd6eed1..ea36967 100644 --- a/cli/resources/commands/03_jobs.php +++ b/cli/resources/commands/03_jobs.php @@ -1,2 +1,3 @@ add($app->getContainer()->get(ProVM\Command\Jobs\Check::class)); +$app->add($app->getContainer()->get(ProVM\Command\Jobs\Execute::class)); diff --git a/cli/setup/middleware/98_log.php b/cli/setup/middleware/98_log.php deleted file mode 100644 index 5628448..0000000 --- a/cli/setup/middleware/98_log.php +++ /dev/null @@ -1,2 +0,0 @@ -add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); diff --git a/cli/setup/settings/01_env.php b/cli/setup/settings/01_env.php index 107718a..76b4051 100644 --- a/cli/setup/settings/01_env.php +++ b/cli/setup/settings/01_env.php @@ -2,7 +2,9 @@ return [ 'api_uri' => $_ENV['API_URI'], 'api_key' => sha1($_ENV['API_KEY']), + 'base_command' => 'qpdf', 'passwords' => function() { return explode($_ENV['PASSWORDS_SEPARATOR'] ?? ',', $_ENV['PASSWORDS'] ?? ''); }, + 'min_check_days' => 1 ]; diff --git a/cli/setup/setups/04_commands.php b/cli/setup/setups/04_commands.php index 50ae9d1..23fd362 100644 --- a/cli/setup/setups/04_commands.php +++ b/cli/setup/setups/04_commands.php @@ -3,18 +3,19 @@ use Psr\Container\ContainerInterface; return [ - ProVM\Command\Attachments\DecryptPdf::class => function(ContainerInterface $container) { - return new ProVM\Command\Attachments\DecryptPdf( + ProVM\Service\Mailboxes::class => function(ContainerInterface $container) { + return new ProVM\Service\Mailboxes( $container->get(ProVM\Service\Communicator::class), $container->get(Psr\Log\LoggerInterface::class), - 'qpdf', - $container->get('passwords') + $container->get('min_check_days') ); }, - ProVM\Command\Mailboxes\Check::class => function(ContainerInterface $container) { - return new ProVM\Command\Mailboxes\Check( + ProVM\Service\Attachments::class => function(ContainerInterface $container) { + return new ProVM\Service\Attachments( $container->get(ProVM\Service\Communicator::class), - 1 + $container->get(Psr\Log\LoggerInterface::class), + $container->get('passwords'), + $container->get('base_command') ); - } + }, ]; diff --git a/cli/setup/setups/98_log.php b/cli/setup/setups/98_log.php index 7f6e30b..ef4a697 100644 --- a/cli/setup/setups/98_log.php +++ b/cli/setup/setups/98_log.php @@ -10,8 +10,7 @@ return [ ]; }, 'request_log_handler' => function(ContainerInterface $container) { - return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) - ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))); }, 'request_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/emails-2022-11-30-03-59-49.sql b/emails-2022-11-30-03-59-49.sql deleted file mode 100644 index 3fd22d9..0000000 --- a/emails-2022-11-30-03-59-49.sql +++ /dev/null @@ -1,94 +0,0 @@ --- Adminer 4.8.1 MySQL 5.5.5-10.9.3-MariaDB-1:10.9.3+maria~ubu2204 dump - -SET NAMES utf8; -SET time_zone = '+00:00'; -SET foreign_key_checks = 0; -SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; - -SET NAMES utf8mb4; - -DROP TABLE IF EXISTS `attachments`; -CREATE TABLE `attachments` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `filename` varchar(255) NOT NULL, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `attachments_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `attachments_jobs`; -CREATE TABLE `attachments_jobs` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `date_time` datetime NOT NULL, - `executed` int(1) DEFAULT 0, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `attachments_jobs_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `attachments_states`; -CREATE TABLE `attachments_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `attachment_id` int(10) unsigned NOT NULL, - `name` varchar(100) NOT NULL, - `value` int(1) unsigned NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `attachment_id` (`attachment_id`), - CONSTRAINT `attachments_states_ibfk_1` FOREIGN KEY (`attachment_id`) REFERENCES `attachments` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `mailboxes`; -CREATE TABLE `mailboxes` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(100) NOT NULL, - `validity` int(11) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `mailboxes_states`; -CREATE TABLE `mailboxes_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `mailbox_id` int(10) unsigned NOT NULL, - `date_time` datetime NOT NULL, - `count` int(10) unsigned NOT NULL, - `uids` text NOT NULL, - PRIMARY KEY (`id`), - KEY `mailbox_id` (`mailbox_id`), - CONSTRAINT `mailboxes_states_ibfk_2` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `messages`; -CREATE TABLE `messages` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `mailbox_id` int(10) unsigned NOT NULL, - `position` int(10) unsigned NOT NULL, - `uid` varchar(255) NOT NULL, - `subject` varchar(255) NOT NULL, - `from` varchar(100) NOT NULL, - `date_time` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `mailbox_id` (`mailbox_id`), - CONSTRAINT `messages_ibfk_1` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `messages_states`; -CREATE TABLE `messages_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `name` varchar(100) NOT NULL, - `value` int(1) unsigned NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `messages_states_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - --- 2022-11-30 03:59:49 diff --git a/ui/common/Controller/Jobs.php b/ui/common/Controller/Jobs.php new file mode 100644 index 0000000..091539f --- /dev/null +++ b/ui/common/Controller/Jobs.php @@ -0,0 +1,14 @@ +render($response, 'jobs.list'); + } +} diff --git a/ui/common/Middleware/Logging.php b/ui/common/Middleware/Logging.php index c60465d..a1e97a0 100644 --- a/ui/common/Middleware/Logging.php +++ b/ui/common/Middleware/Logging.php @@ -29,7 +29,7 @@ class Logging { $response = $handler->handle($request); $output = [ - 'uri' => var_export($request->getUri(), true), + 'uri' => print_r($request->getUri(), true), 'body' => $request->getParsedBody() ]; $this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); diff --git a/ui/nginx.conf b/ui/nginx.conf index 893e62a..43895cc 100644 --- a/ui/nginx.conf +++ b/ui/nginx.conf @@ -3,6 +3,9 @@ server { root /app/ui/public; index index.php index.html index.htm; + access_log /var/logs/nginx/ui.access.log; + error_log /var/logs/nginx/ui.error.log; + location / { try_files $uri $uri/ /index.php?$query_string; } @@ -16,4 +19,4 @@ server { fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; } -} \ No newline at end of file +} diff --git a/ui/public/index.php b/ui/public/index.php index 9e2bc0e..3408a32 100644 --- a/ui/public/index.php +++ b/ui/public/index.php @@ -8,11 +8,5 @@ Monolog\ErrorHandler::register($app->getContainer()->get(Psr\Log\LoggerInterface try { $app->run(); } catch (Error | Exception $e) { - $logger = $app->getContainer()->get(Psr\Log\LoggerInterface::class); - if (isset($_REQUEST)) { - $logger->debug(Safe\json_encode(compact('_REQUEST'))); - } - $logger->debug(Safe\json_encode(compact('_SERVER'))); - $logger->error($e); - throw $e; + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->error($e); } diff --git a/ui/resources/routes/03_jobs.php b/ui/resources/routes/03_jobs.php new file mode 100644 index 0000000..f9d8f0a --- /dev/null +++ b/ui/resources/routes/03_jobs.php @@ -0,0 +1,4 @@ +get('/jobs', Jobs::class); diff --git a/ui/resources/views/emails/messages.blade.php b/ui/resources/views/emails/messages.blade.php index 4af8bb1..089eaee 100644 --- a/ui/resources/views/emails/messages.blade.php +++ b/ui/resources/views/emails/messages.blade.php @@ -5,7 +5,12 @@
@endsection +@push('page_styles') + +@endpush + @push('page_scripts') + + +@endpush diff --git a/ui/resources/views/layout/body/header/navbar.blade.php b/ui/resources/views/layout/body/header/navbar.blade.php index 92fa22b..a0a622f 100644 --- a/ui/resources/views/layout/body/header/navbar.blade.php +++ b/ui/resources/views/layout/body/header/navbar.blade.php @@ -1,5 +1,6 @@
diff --git a/ui/setup/setups/98_log.php b/ui/setup/setups/98_log.php index ef8b9d6..f12fc26 100644 --- a/ui/setup/setups/98_log.php +++ b/ui/setup/setups/98_log.php @@ -2,8 +2,13 @@ use Psr\Container\ContainerInterface; return [ - Monolog\Handler\DeduplicationHandler::class => function(ContainerInterface $container) { - return new Monolog\Handler\DeduplicationHandler($container->get(Monolog\Handler\RotatingFileHandler::class)); + 'log_processors' => function(ContainerInterface $container) { + return [ + $container->get(Monolog\Processor\PsrLogMessageProcessor::class), + $container->get(Monolog\Processor\WebProcessor::class), + $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class) + ]; }, Monolog\Handler\RotatingFileHandler::class => function(ContainerInterface $container) { $handler = new Monolog\Handler\RotatingFileHandler($container->get('log_file')); @@ -11,22 +16,23 @@ return [ return $handler; }, 'request_logger' => function(ContainerInterface $container) { - $logger = new Monolog\Logger('request_logger'); - $handler = new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'requests.log'])); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - $dedupHandler = new Monolog\Handler\DeduplicationHandler($handler, null, Monolog\Level::Info); - $logger->pushHandler($dedupHandler); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger('request_logger', [ + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'requests.log']))), + ], $container->get('log_processors')); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - $logger = new Monolog\Logger('file_logger'); - $logger->pushHandler($container->get(Monolog\Handler\DeduplicationHandler::class)); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger('file', [ + new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler($container->get('log_file'))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Error + ), + new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'debug.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Debug, + Monolog\Level::Warning + ) + ], $container->get('log_processors')); }, ];