From 9307ba330cbb29f8d2733fbede0102a1d29bc073 Mon Sep 17 00:00:00 2001 From: Aldarien Date: Thu, 8 Jun 2023 20:49:27 -0400 Subject: [PATCH 1/3] Various updates --- NOTES.md | 12 ++- api/common/Controller/Jobs.php | 3 +- api/common/Controller/Messages.php | 28 +++++- api/common/Define/Model.php | 6 +- api/common/Define/Repository.php | 28 ++++++ api/common/Factory/Model.php | 2 +- api/common/Implement/Repository.php | 118 +++++++++++++----------- api/common/Middleware/Attachments.php | 25 +++++ api/common/Middleware/Install.php | 20 ++++ api/common/Middleware/Logging.php | 23 +---- api/common/Middleware/Mailboxes.php | 24 +++++ api/common/Middleware/Messages.php | 24 +++++ api/common/Service/Attachments.php | 92 ++++++++++++++++-- api/common/Service/Install.php | 40 ++++++++ api/common/Service/Mailboxes.php | 15 ++- api/common/Service/Messages.php | 43 ++++++++- api/public/index.php | 12 +-- api/setup/middleware/04_db.php | 5 + api/setup/settings/01_env.php | 5 +- api/setup/settings/04_db.php | 33 +++++++ api/setup/settings/98_log.php | 5 +- api/setup/setups/01_emails.php | 8 +- api/setup/setups/02_services.php | 15 +++ api/setup/setups/03_factories.php | 23 +++-- api/setup/setups/04_middlewares.php | 2 +- api/setup/setups/97_auth.php | 8 +- api/setup/setups/98_log.php | 72 ++++++++++----- api/src/Model/Attachment.php | 6 +- api/src/Model/Job.php | 6 +- api/src/Model/Mailbox.php | 9 +- api/src/Model/Message.php | 12 ++- api/src/Model/State/Attachment.php | 16 +++- api/src/Model/State/Mailbox.php | 19 +++- api/src/Model/State/Message.php | 16 +++- api/src/Repository/Attachment.php | 23 ++++- api/src/Repository/Job.php | 17 +++- api/src/Repository/Mailbox.php | 14 ++- api/src/Repository/Message.php | 47 +++++++--- api/src/Repository/State/Attachment.php | 18 +++- api/src/Repository/State/Mailbox.php | 19 +++- api/src/Repository/State/Message.php | 18 +++- cli/common/Command/DecryptPdf.php | 9 +- cli/common/Command/GrabAttachments.php | 9 +- cli/common/Command/Messages.php | 9 +- emails-2022-11-30-03-59-49.sql | 94 +++++++++++++++++++ 45 files changed, 864 insertions(+), 188 deletions(-) create mode 100644 api/common/Define/Repository.php create mode 100644 api/common/Middleware/Attachments.php create mode 100644 api/common/Middleware/Install.php create mode 100644 api/common/Middleware/Mailboxes.php create mode 100644 api/common/Middleware/Messages.php create mode 100644 api/common/Service/Install.php create mode 100644 api/setup/middleware/04_db.php create mode 100644 api/setup/settings/04_db.php create mode 100644 emails-2022-11-30-03-59-49.sql diff --git a/NOTES.md b/NOTES.md index d20957b..ec9035b 100644 --- a/NOTES.md +++ b/NOTES.md @@ -25,4 +25,14 @@ -> **[API]** Register selected `mailboxes` and get `messages` for recently registered. * **[Cron]** Get registered `mailboxes` -> **[API]** Get `messages` * **[User]** Check messages found -> **[API]** Schedule `attachments` -* **[Cron]** Get `attachment download` jobs -> **[API]** grab `attachments` \ No newline at end of file +* **[Cron]** Get `attachment download` jobs -> **[API]** grab `attachments` + +### Jobs +#### Automatic +* [ ] Check registered `mailboxes` for new `messages`, logging last check. +* [ ] Check if `attachments` are `encrypted`. +* [ ] Check for new scheduled jobs. +#### Scheduled +* [ ] Grab `messages`. +* [ ] Grab `attachments`. +* [ ] Decrypt `attachments`. diff --git a/api/common/Controller/Jobs.php b/api/common/Controller/Jobs.php index 9320eae..437555f 100644 --- a/api/common/Controller/Jobs.php +++ b/api/common/Controller/Jobs.php @@ -7,6 +7,7 @@ use Psr\Http\Message\ServerRequestInterface; use ProVM\Common\Exception\Request\MissingArgument; use ProVM\Common\Implement\Controller\Json; use ProVM\Common\Service\Jobs as Service; +use function Safe\json_decode; class Jobs { @@ -15,7 +16,7 @@ class Jobs public function schedule(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Messages $messagesService): ResponseInterface { $body = $request->getBody(); - $json = \Safe\json_decode($body->getContents()); + $json = json_decode($body->getContents()); if (!isset($json->messages)) { throw new MissingArgument('messages', 'array', 'messages ids'); } diff --git a/api/common/Controller/Messages.php b/api/common/Controller/Messages.php index 47425ef..db86073 100644 --- a/api/common/Controller/Messages.php +++ b/api/common/Controller/Messages.php @@ -1,13 +1,13 @@ toArray(); }, $service->getAll($mailbox->getName())); + usort($messages, function($a, $b) { + $d = $a['date_time'] - $b['date_time']; + if ($d->days === 0) { + $f = strcmp($a['from'], $b['from']); + if ($f === 0) { + return strcmp($a['subject'], $b['subject']); + } + return $f; + } + return $d->format('%r%a'); + }); $output = [ 'mailbox' => $mailbox->toArray(), 'total' => count($messages), @@ -34,6 +45,17 @@ class Messages }, $service->getValid($mailbox->getName())), function($message) { return $message !== null; })); + usort($messages, function($a, $b) { + $d = strcmp($a['date_time'], $b['date_time']); + if ($d === 0) { + $f = strcmp($a['from'], $b['from']); + if ($f === 0) { + return strcmp($a['subject'], $b['subject']); + } + return $f; + } + return $d; + }); $output = [ 'mailbox' => $mailbox->toArray(), 'total' => count($messages), @@ -49,7 +71,7 @@ class Messages public function grab(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Attachments $attachmentsService): ResponseInterface { $body = $request->getBody(); - $json = \Safe\json_decode($body->getContents()); + $json = json_decode($body->getContents()); if (!isset($json->mailboxes)) { throw new MissingArgument('mailboxes', 'array', 'mailboxes names'); } @@ -70,4 +92,4 @@ class Messages } return $this->withJson($response, $output); } -} \ No newline at end of file +} diff --git a/api/common/Define/Model.php b/api/common/Define/Model.php index 28a9b45..9af5971 100644 --- a/api/common/Define/Model.php +++ b/api/common/Define/Model.php @@ -1,6 +1,8 @@ getContainer()->get($repository_class); } -} \ No newline at end of file +} diff --git a/api/common/Implement/Repository.php b/api/common/Implement/Repository.php index f8a7b7a..7cbe22d 100644 --- a/api/common/Implement/Repository.php +++ b/api/common/Implement/Repository.php @@ -3,11 +3,11 @@ namespace ProVM\Common\Implement; use PDO; use PDOException; -use ProVM\Common\Exception\Database\BlankResult; use Psr\Log\LoggerInterface; -use ProVM\Common\Define\Model as ModelInterface; +use ProVM\Common\Exception\Database\BlankResult; +use ProVM\Common\Define; -abstract class Repository +abstract class Repository implements Define\Repository { public function __construct(PDO $connection, LoggerInterface $logger) { @@ -32,33 +32,32 @@ abstract class Repository return $this->logger; } - public function setConnection(PDO $pdo): Repository + public function setConnection(PDO $pdo): Define\Repository { $this->connection = $pdo; return $this; } - public function setTable(string $table): Repository + public function setTable(string $table): Define\Repository { $this->table = $table; return $this; } - public function setLogger(LoggerInterface $logger): Repository + public function setLogger(LoggerInterface $logger): Define\Repository { $this->logger = $logger; return $this; } - abstract protected function fieldsForUpdate(): array; - abstract protected function valuesForUpdate(ModelInterface $model): array; - protected function idProperty(): string + public function isInstalled(): bool { - return 'getId'; + $query = "SHOW TABLES LIKE '{$this->getTable()}'"; + $st = $this->getConnection()->query($query); + if ($st === false) { + throw new PDOException("Could not run query {$query}"); + } + return $st->rowCount() > 0; } - protected function idField(): string - { - return 'id'; - } - public function update(ModelInterface $model, ModelInterface $old): void + public function update(Define\Model $model, Define\Model $old): void { $query = "UPDATE `{$this->getTable()}` SET "; $model_values = $this->valuesForUpdate($model); @@ -79,22 +78,7 @@ abstract class Repository $st = $this->getConnection()->prepare($query); $st->execute($values); } - abstract protected function fieldsForInsert(): array; - abstract protected function valuesForInsert(ModelInterface $model): array; - protected function insert(ModelInterface $model): void - { - $fields = $this->fieldsForInsert(); - $fields_string = implode(', ', array_map(function($field) { - return "`{$field}`"; - }, $fields)); - $fields_questions = implode(', ', array_fill(0, count($fields), '?')); - $query = "INSERT INTO `{$this->getTable()}` ({$fields_string}) VALUES ({$fields_questions})"; - $values = $this->valuesForInsert($model); - $st = $this->getConnection()->prepare($query); - $st->execute($values); - } - abstract protected function defaultFind(ModelInterface $model): ModelInterface; - public function save(ModelInterface &$model): void + public function save(Define\Model &$model): void { try { $old = $this->defaultFind($model); @@ -107,12 +91,7 @@ abstract class Repository throw $e; } } - abstract public function load(array $row): ModelInterface; - - abstract protected function fieldsForCreate(): array; - abstract protected function valuesForCreate(array $data): array; - abstract protected function defaultSearch(array $data): ModelInterface; - public function create(array $data): ModelInterface + public function create(array $data): Define\Model { try { return $this->defaultSearch($data); @@ -121,11 +100,6 @@ abstract class Repository return $this->load($data); } } - - protected function getId(ModelInterface $model): int - { - return $model->getId(); - } public function resetIndex(): void { $query = "ALTER TABLE `{$this->getTable()}` AUTO_INCREMENT = 1"; @@ -136,7 +110,7 @@ abstract class Repository $query = "OPTIMIZE TABLE `{$this->getTable()}`"; $this->getConnection()->query($query); } - public function delete(ModelInterface $model): void + public function delete(Define\Model $model): void { $query = "DELETE FROM `{$this->getTable()}` WHERE `{$this->idField()}` = ?"; $st = $this->getConnection()->prepare($query); @@ -144,8 +118,42 @@ abstract class Repository $this->resetIndex(); $this->optimize(); } + public function fetchAll(): array + { + $query = "SELECT * FROM `{$this->getTable()}`"; + return $this->fetchMany($query); + } + public function fetchById(int $id): Define\Model + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `{$this->idField()}` = ?"; + return $this->fetchOne($query, [$id]); + } - protected function fetchOne(string $query, ?array $values = null): ModelInterface + protected function idProperty(): string + { + return 'getId'; + } + protected function idField(): string + { + return 'id'; + } + protected function insert(Define\Model $model): void + { + $fields = $this->fieldsForInsert(); + $fields_string = implode(', ', array_map(function($field) { + return "`{$field}`"; + }, $fields)); + $fields_questions = implode(', ', array_fill(0, count($fields), '?')); + $query = "INSERT INTO `{$this->getTable()}` ({$fields_string}) VALUES ({$fields_questions})"; + $values = $this->valuesForInsert($model); + $st = $this->getConnection()->prepare($query); + $st->execute($values); + } + protected function getId(Define\Model $model): int + { + return $model->getId(); + } + protected function fetchOne(string $query, ?array $values = null): Define\Model { if ($values !== null) { $st = $this->getConnection()->prepare($query); @@ -174,14 +182,14 @@ abstract class Repository return array_map([$this, 'load'], $rows); } - public function fetchAll(): array - { - $query = "SELECT * FROM `{$this->getTable()}`"; - return $this->fetchMany($query); - } - public function fetchById(int $id): ModelInterface - { - $query = "SELECT * FROM `{$this->getTable()}` WHERE `{$this->idField()}` = ?"; - return $this->fetchOne($query, [$id]); - } -} \ No newline at end of file + abstract public function install(): void; + abstract public function load(array $row): Define\Model; + abstract protected function fieldsForUpdate(): array; + abstract protected function valuesForUpdate(Define\Model $model): array; + abstract protected function fieldsForInsert(): array; + abstract protected function valuesForInsert(Define\Model $model): array; + abstract protected function defaultFind(Define\Model $model): Define\Model; + abstract protected function fieldsForCreate(): array; + abstract protected function valuesForCreate(array $data): array; + abstract protected function defaultSearch(array $data): Define\Model; +} diff --git a/api/common/Middleware/Attachments.php b/api/common/Middleware/Attachments.php new file mode 100644 index 0000000..1baabca --- /dev/null +++ b/api/common/Middleware/Attachments.php @@ -0,0 +1,25 @@ +service->checkDownloaded(); + $this->service->checkEncryption(); + } catch (BlankResult $e) { + $this->logger->notice($e); + } + return $handler->handle($request); + } +} diff --git a/api/common/Middleware/Install.php b/api/common/Middleware/Install.php new file mode 100644 index 0000000..b9a8053 --- /dev/null +++ b/api/common/Middleware/Install.php @@ -0,0 +1,20 @@ +service->check()) { + $this->service->install(); + } + return $handler->handle($request); + } +} diff --git a/api/common/Middleware/Logging.php b/api/common/Middleware/Logging.php index a95c37b..2e91a6d 100644 --- a/api/common/Middleware/Logging.php +++ b/api/common/Middleware/Logging.php @@ -5,34 +5,21 @@ use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Server\RequestHandlerInterface; use Psr\Log\LoggerInterface; +use function Safe\json_encode; class Logging { - public function __construct(LoggerInterface $logger) { - $this->setLogger($logger); - } - - protected LoggerInterface $logger; - - public function getLogger(): LoggerInterface - { - return $this->logger; - } - - public function setLogger(LoggerInterface $logger): Logging - { - $this->logger = $logger; - return $this; - } + public function __construct(protected LoggerInterface $logger) {} public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface { $response = $handler->handle($request); $output = [ 'uri' => var_export($request->getUri(), true), - 'body' => $request->getBody()->getContents() + 'body' => $request->getBody()->getContents(), + 'response' => (clone $response)->getBody()->getContents() ]; - $this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); + $this->logger->info(json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); return $response; } } diff --git a/api/common/Middleware/Mailboxes.php b/api/common/Middleware/Mailboxes.php new file mode 100644 index 0000000..0808f7f --- /dev/null +++ b/api/common/Middleware/Mailboxes.php @@ -0,0 +1,24 @@ +service->checkUpdate(); + } catch (BlankResult $e) { + $this->logger->notice($e); + } + return $handler->handle($request); + } +} diff --git a/api/common/Middleware/Messages.php b/api/common/Middleware/Messages.php new file mode 100644 index 0000000..13c13cb --- /dev/null +++ b/api/common/Middleware/Messages.php @@ -0,0 +1,24 @@ +service->checkSchedule(); + } catch (BlankResult $e) { + $this->logger->notice($e); + } + return $handler->handle($request); + } +} diff --git a/api/common/Service/Attachments.php b/api/common/Service/Attachments.php index 6c92915..1487b77 100644 --- a/api/common/Service/Attachments.php +++ b/api/common/Service/Attachments.php @@ -1,15 +1,15 @@ getFullFilename() ]); } - return \Safe\file_get_contents($filename); + return file_get_contents($filename); } public function getAll(): array { return $this->getRepository()->fetchAll(); } + public function getDownloadedFiles(): array + { + $downloaded = []; + $folder = $this->getFolder(); + $files = new \FilesystemIterator($folder); + foreach ($files as $file) { + if ($file->isDir()) { + continue; + } + $name = $file->getBasename(".{$file->getExtension()}"); + list($subject, $date, $filename) = explode(' - ', $name); + try { + $message = $this->getMessages()->find($subject, $date)[0]; + $filename = "{$filename}.{$file->getExtension()}"; + $downloaded []= compact('message', 'filename'); + } catch (BlankResult $e) { + } + } + return $downloaded; + } public function create(int $message_id): array { $message = $this->getMessages()->getRepository()->fetchById($message_id); @@ -183,7 +203,7 @@ class Attachments extends Base $attachment->getFullFilename() ]); try { - \Safe\file_put_contents($destination, $remote_attachment->getDecodedContent()); + file_put_contents($destination, $remote_attachment->getDecodedContent()); return true; } catch (FilesystemException $e) { $this->getLogger()->error($e); @@ -236,6 +256,9 @@ class Attachments extends Base if (!$message->hasValidAttachments()) { return false; } + if ($message->hasDownloadedAttachments()) { + return true; + } foreach ($remote_message->getAttachments() as $attachment) { if (!str_contains($attachment->getFilename(), '.pdf')) { continue; @@ -247,4 +270,59 @@ class Attachments extends Base } return true; } -} \ No newline at end of file + public function find(Message $message, string $filename): \ProVM\Emails\Model\Attachment + { + return $this->getRepository()->fetchByMessageAndFilename($message->getId(), $filename); + } + public function exists(Message $message, string $filename): bool + { + try { + $this->find($message, $filename); + return true; + } catch (BlankResult $e) { + return false; + } + } + public function add(Message $message, string $filename): bool + { + $data = [ + 'message_id' => $message->getId(), + 'filename' => $filename + ]; + try { + $attachment = $this->getRepository()->create($data); + $attachment->itIsDownloaded(); + $this->getRepository()->save($attachment); + $message->doesHaveDownloadedAttachments(); + $this->getMessages()->getRepository()->save($message); + return true; + } catch (PDOException $e) { + $this->getLogger()->error($e); + return false; + } + } + public function checkDownloaded(): void + { + $data = $this->getDownloadedFiles(); + foreach ($data as $info) { + if (!$this->exists($info['message'], $info['filename'])) { + $this->logger->info("Updating attachment {$info['filename']} for message {$info['message']->getSubject()}"); + $this->add($info['message'], $info['filename']); + } + } + } + public function getDownloaded(): array + { + return $this->getRepository()->fetchDownloaded(); + } + public function checkEncryption(): void + { + $attachments = $this->getDownloaded(); + foreach ($attachments as $attachment) { + if ($attachment->isEncrypted() and !$attachment->isDecrypted()) { + $this->logger->notice("Schedule decrypt for {$attachment->getFullFilename()}"); + $this->decrypt($attachment->getMessage(), $attachment->getFilename()); + } + } + } +} diff --git a/api/common/Service/Install.php b/api/common/Service/Install.php new file mode 100644 index 0000000..2ae9574 --- /dev/null +++ b/api/common/Service/Install.php @@ -0,0 +1,40 @@ +model_list as $model_class) { + $repository = $this->factory->find($model_class); + if (!$repository->isInstalled()) { + return false; + } + } + return true; + } + public function install(): void + { + $check = true; + $repository = null; + foreach ($this->model_list as $model_class) { + $repository = $this->factory->find($model_class); + if ($check) { + $query = "SET FOREIGN_KEY_CHECKS = 0"; + $repository->getConnection()->query($query); + $check = false; + } + if (!$repository->isInstalled()) { + $repository->install(); + } + } + if (!$check) { + $query = "SET FOREIGN_KEY_CHECKS = 1"; + $repository->getConnection()->query($query); + } + } +} diff --git a/api/common/Service/Mailboxes.php b/api/common/Service/Mailboxes.php index 908cf00..3eda6e6 100644 --- a/api/common/Service/Mailboxes.php +++ b/api/common/Service/Mailboxes.php @@ -11,7 +11,7 @@ use ProVM\Emails\Repository\State; class Mailboxes extends Base { - public function __construct(Mailbox $repository, Remote\Mailboxes $remoteService, State\Mailbox $states, LoggerInterface $logger) + public function __construct(Mailbox $repository, Remote\Mailboxes $remoteService, State\Mailbox $states, LoggerInterface $logger, protected int $max_update_days) { $this->setRepository($repository) ->setRemoteService($remoteService) @@ -35,7 +35,6 @@ class Mailboxes extends Base { return $this->statesRepository; } - public function setRepository(Mailbox $repository): Mailboxes { $this->repository = $repository; @@ -110,6 +109,7 @@ class Mailboxes extends Base $this->getStatesRepository()->save($state); return true; } catch (PDOException $e) { + $this->getLogger()->error($e); return false; } } @@ -141,4 +141,13 @@ class Mailboxes extends Base } return true; } -} \ No newline at end of file + public function isUpdated(\ProVM\Emails\Model\Mailbox $mailbox): bool + { + $states = $mailbox->getStates(); + if (count($states) === 0) { + return false; + } + $last = $states[count($states) - 1]; + return abs((int) $last->getDateTime()->diff(new \DateTimeImmutable())->format('%r%a')) < $this->max_update_days; + } +} diff --git a/api/common/Service/Messages.php b/api/common/Service/Messages.php index c5d0f05..a0cd5e0 100644 --- a/api/common/Service/Messages.php +++ b/api/common/Service/Messages.php @@ -4,6 +4,7 @@ namespace ProVM\Common\Service; use Ddeboer\Imap\Exception\MessageDoesNotExistException; use Ddeboer\Imap\MailboxInterface; use PDOException; +use ProVM\Common\Exception\Database\BlankResult; use ProVM\Common\Exception\Mailbox\Stateless; use Psr\Log\LoggerInterface; use Ddeboer\Imap\MessageInterface; @@ -14,17 +15,19 @@ use Safe\DateTimeImmutable; class Messages extends Base { - public function __construct(Mailboxes $mailboxes, Message $repository, Remote\Messages $remoteService, LoggerInterface $logger) + public function __construct(Mailboxes $mailboxes, Message $repository, Remote\Messages $remoteService, Jobs $jobsService, LoggerInterface $logger) { $this->setMailboxes($mailboxes) ->setRepository($repository) ->setRemoteService($remoteService) + ->setJobsService($jobsService) ->setLogger($logger); } protected Mailboxes $mailboxes; protected Message $repository; protected Remote\Messages $remoteService; + protected Jobs $jobsService; public function getMailboxes(): Mailboxes { @@ -38,6 +41,10 @@ class Messages extends Base { return $this->remoteService; } + public function getJobsService(): Jobs + { + return $this->jobsService; + } public function setMailboxes(Mailboxes $mailboxes): Messages { @@ -54,6 +61,11 @@ class Messages extends Base $this->remoteService = $service; return $this; } + public function setJobsService(Jobs $service): Messages + { + $this->jobsService = $service; + return $this; + } public function getLocalMessage(string $message_uid): \ProVM\Emails\Model\Message { @@ -142,6 +154,7 @@ class Messages extends Base $message->doesHaveValidAttachments(); } } + error_log(json_encode(compact('message')).PHP_EOL,3,'/logs/debug'); $this->getRepository()->save($message); return true; } catch (PDOException $e) { @@ -174,4 +187,30 @@ class Messages extends Base } return false; } -} \ No newline at end of file + public function find(string $subject, string $date): array + { + return $this->repository->fetchAllBySubjectAndDate($subject, new DateTimeImmutable($date)); + } + public function checkUpdate(): void + { + $registered = $this->getMailboxes()->getRegistered(); + foreach ($registered as $mailbox) { + if (!$this->getMailboxes()->isUpdated($mailbox)) { + $this->logger->info("Updating messages from {$mailbox->getName()}"); + $this->grab($mailbox->getName()); + } + } + } + public function checkSchedule(): void + { + $messages = $this->getRepository()->fetchAll(); + foreach ($messages as $message) { + if ($message->hasAttachments() and $message->hasValidAttachments() and !$message->hasDownloadedAttachments() and !$message->hasScheduledDownloads()) { + if ($this->getJobsService()->schedule($message->getId())) { + $message->doesHaveDownloadedAttachments(); + $this->getRepository()->save($message); + } + } + } + } +} diff --git a/api/public/index.php b/api/public/index.php index 9e2bc0e..2e6a761 100644 --- a/api/public/index.php +++ b/api/public/index.php @@ -7,12 +7,8 @@ $app = require_once implode(DIRECTORY_SEPARATOR, [ Monolog\ErrorHandler::register($app->getContainer()->get(Psr\Log\LoggerInterface::class)); try { $app->run(); -} catch (Error | Exception $e) { - $logger = $app->getContainer()->get(Psr\Log\LoggerInterface::class); - if (isset($_REQUEST)) { - $logger->debug(Safe\json_encode(compact('_REQUEST'))); - } - $logger->debug(Safe\json_encode(compact('_SERVER'))); - $logger->error($e); - throw $e; +} catch (Error $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->error($e); +} catch (Exception $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->debug($e); } diff --git a/api/setup/middleware/04_db.php b/api/setup/middleware/04_db.php new file mode 100644 index 0000000..9d831e8 --- /dev/null +++ b/api/setup/middleware/04_db.php @@ -0,0 +1,5 @@ +add($app->getContainer()->get(ProVM\Common\Middleware\Attachments::class)); +$app->add($app->getContainer()->get(ProVM\Common\Middleware\Messages::class)); +$app->add($app->getContainer()->get(ProVM\Common\Middleware\Mailboxes::class)); +$app->add($app->getContainer()->get(ProVM\Common\Middleware\Install::class)); diff --git a/api/setup/settings/01_env.php b/api/setup/settings/01_env.php index 6e61213..89deb37 100644 --- a/api/setup/settings/01_env.php +++ b/api/setup/settings/01_env.php @@ -5,7 +5,7 @@ return [ 'host' => $_ENV['EMAIL_HOST'], 'username' => $_ENV['EMAIL_USERNAME'], 'password' => $_ENV['EMAIL_PASSWORD'], - 'folder' => $_ENV['EMAIL_FOLDER'], + //'folder' => $_ENV['EMAIL_FOLDER'], ]; if (isset($_ENV['EMAIL_PORT'])) { $data['port'] = $_ENV['EMAIL_PORT']; @@ -27,5 +27,6 @@ return [ $arr['port'] = $_ENV['MYSQL_PORT']; } return (object) $arr; - } + }, + 'max_update_days' => 7 ]; diff --git a/api/setup/settings/04_db.php b/api/setup/settings/04_db.php new file mode 100644 index 0000000..62415ba --- /dev/null +++ b/api/setup/settings/04_db.php @@ -0,0 +1,33 @@ + function() { + function getClassesFromFolder(string $folder): array { + $classes = []; + $files = new FilesystemIterator($folder); + foreach ($files as $file) { + if ($file->isDir()) { + $classes = array_merge($classes, getClassesFromFolder($file->getRealPath())); + continue; + } + $classes []= ltrim(str_replace("\\\\", "\\", implode("\\", [ + 'ProVM', + 'Emails', + 'Model', + str_replace([implode(DIRECTORY_SEPARATOR, [ + dirname(__FILE__, 3), + 'src', + 'Model' + ]), '/'], ['', "\\"], $folder), + $file->getBasename(".{$file->getExtension()}") + ])), "\\"); + } + return $classes; + } + $folder = implode(DIRECTORY_SEPARATOR, [ + dirname(__FILE__, 3), + 'src', + 'Model' + ]); + return getClassesFromFolder($folder); + } +]; diff --git a/api/setup/settings/98_log.php b/api/setup/settings/98_log.php index 55328ea..3c1e3f0 100644 --- a/api/setup/settings/98_log.php +++ b/api/setup/settings/98_log.php @@ -1,4 +1,5 @@ '/logs/php.log' -]; \ No newline at end of file + 'log_file' => '/logs/php.log', + 'logstash_socket' => 'localhost:50000' +]; diff --git a/api/setup/setups/01_emails.php b/api/setup/setups/01_emails.php index 23c62b3..1428bae 100644 --- a/api/setup/setups/01_emails.php +++ b/api/setup/setups/01_emails.php @@ -5,13 +5,13 @@ return [ Ddeboer\Imap\ServerInterface::class => function(ContainerInterface $container) { $emails = $container->get('emails'); if (isset($emails->port)) { - return new \Ddeboer\Imap\Server($emails->host, $emails->port); + return new Ddeboer\Imap\Server($emails->host, $emails->port); } - return new \Ddeboer\Imap\Server($emails->host); + return new Ddeboer\Imap\Server($emails->host); }, - \Ddeboer\Imap\ConnectionInterface::class => function(ContainerInterface $container) { + Ddeboer\Imap\ConnectionInterface::class => function(ContainerInterface $container) { $emails = $container->get('emails'); - $server = $container->get(\Ddeboer\Imap\ServerInterface::class); + $server = $container->get(Ddeboer\Imap\ServerInterface::class); return $server->authenticate($emails->username, $emails->password); }, PDO::class => function(ContainerInterface $container) { diff --git a/api/setup/setups/02_services.php b/api/setup/setups/02_services.php index 8fd0f0a..2a09318 100644 --- a/api/setup/setups/02_services.php +++ b/api/setup/setups/02_services.php @@ -18,5 +18,20 @@ return [ $container->get('attachments_folder'), $container->get(Psr\Log\LoggerInterface::class) ); + }, + ProVM\Common\Service\Mailboxes::class => function(ContainerInterface $container) { + return new ProVM\Common\Service\Mailboxes( + $container->get(ProVM\Emails\Repository\Mailbox::class), + $container->get(ProVM\Common\Service\Remote\Mailboxes::class), + $container->get(ProVM\Emails\Repository\State\Mailbox::class), + $container->get(Psr\Log\LoggerInterface::class), + $container->get('max_update_days') + ); + }, + ProVM\Common\Service\Install::class => function(ContainerInterface $container) { + return new ProVM\Common\Service\Install( + $container->get(ProVM\Common\Factory\Model::class), + $container->get('model_list') + ); } ]; diff --git a/api/setup/setups/03_factories.php b/api/setup/setups/03_factories.php index 654ca74..281eafa 100644 --- a/api/setup/setups/03_factories.php +++ b/api/setup/setups/03_factories.php @@ -2,16 +2,15 @@ use Psr\Container\ContainerInterface; return [ - \ProVM\Common\Factory\Model::class => function(ContainerInterface $container) { - $factory = new \ProVM\Common\Factory\Model($container); - $repositories = [ - 'Mailbox' => \ProVM\Emails\Repository\Mailbox::class, - 'Message' => \ProVM\Emails\Repository\Message::class, - 'Attachment' => \ProVM\Emails\Repository\Attachment::class, - "State\\Mailbox" => \ProVM\Emails\Repository\State\Mailbox::class, - "State\\Message" => \ProVM\Emails\Repository\State\Message::class, - "State\\Attachment" => \ProVM\Emails\Repository\State\Attachment::class - ]; - return $factory->setRepositories($repositories); + ProVM\Common\Factory\Model::class => function(ContainerInterface $container) { + $factory = new ProVM\Common\Factory\Model($container); + return $factory->setRepositories([ + 'Mailbox' => ProVM\Emails\Repository\Mailbox::class, + 'Message' => ProVM\Emails\Repository\Message::class, + 'Attachment' => ProVM\Emails\Repository\Attachment::class, + "State\\Mailbox" => ProVM\Emails\Repository\State\Mailbox::class, + "State\\Message" => ProVM\Emails\Repository\State\Message::class, + "State\\Attachment" => ProVM\Emails\Repository\State\Attachment::class + ]); } -]; \ No newline at end of file +]; diff --git a/api/setup/setups/04_middlewares.php b/api/setup/setups/04_middlewares.php index 1ec3b45..10775dd 100644 --- a/api/setup/setups/04_middlewares.php +++ b/api/setup/setups/04_middlewares.php @@ -10,5 +10,5 @@ return [ }, ProVM\Common\Middleware\Logging::class => function(ContainerInterface $container) { return new ProVM\Common\Middleware\Logging($container->get('request_logger')); - } + }, ]; diff --git a/api/setup/setups/97_auth.php b/api/setup/setups/97_auth.php index 17e45ac..03fa0f3 100644 --- a/api/setup/setups/97_auth.php +++ b/api/setup/setups/97_auth.php @@ -2,10 +2,10 @@ use Psr\Container\ContainerInterface; return [ - \ProVM\Common\Middleware\Auth::class => function(ContainerInterface $container) { - return new \ProVM\Common\Middleware\Auth( - $container->get(\Nyholm\Psr7\Factory\Psr17Factory::class), - $container->get(\Psr\Log\LoggerInterface::class), + ProVM\Common\Middleware\Auth::class => function(ContainerInterface $container) { + return new ProVM\Common\Middleware\Auth( + $container->get(Nyholm\Psr7\Factory\Psr17Factory::class), + $container->get(Psr\Log\LoggerInterface::class), $container->get('api_key') ); } diff --git a/api/setup/setups/98_log.php b/api/setup/setups/98_log.php index ec2c9a7..4c6a66e 100644 --- a/api/setup/setups/98_log.php +++ b/api/setup/setups/98_log.php @@ -2,31 +2,61 @@ use Psr\Container\ContainerInterface; return [ - Monolog\Handler\DeduplicationHandler::class => function(ContainerInterface $container) { - return new Monolog\Handler\DeduplicationHandler($container->get(Monolog\Handler\RotatingFileHandler::class)); + 'log_processors' => function(ContainerInterface $container) { + return [ + $container->get(Monolog\Processor\PsrLogMessageProcessor::class), + $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class), + ]; }, - Monolog\Handler\RotatingFileHandler::class => function(ContainerInterface $container) { - $handler = new Monolog\Handler\RotatingFileHandler($container->get('log_file')); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - return $handler; + 'request_log_handler' => function(ContainerInterface $container) { + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); }, 'request_logger' => function(ContainerInterface $container) { - $logger = new Monolog\Logger('request_logger'); - $handler = new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log'])); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - $dedupHandler = new Monolog\Handler\DeduplicationHandler($handler, null, Monolog\Level::Info); - $logger->pushHandler($dedupHandler); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger( + 'request_logger', + [$container->get('request_log_handler')], + $container->get('log_processors') + ); + }, + 'file_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler($container->get('log_file'))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Error + ); + }, + 'debug_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'debug.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Debug, + Monolog\Level::Warning + ); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - $logger = new Monolog\Logger('file_logger'); - $logger->pushHandler($container->get(Monolog\Handler\DeduplicationHandler::class)); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return $container->get('elk_logger'); + }, + 'file_logger' => function(ContainerInterface $container) { + return new Monolog\Logger( + 'file', + [ + $container->get('file_log_handler'), + $container->get('debug_log_handler') + ], + $container->get('log_processors') + ); + }, + 'elk_logger' => function(ContainerInterface $container) { + return new Monolog\Logger('elk', [ + (new Monolog\Handler\SocketHandler($container->get('logstash_socket'))) + ->setFormatter(new Monolog\Formatter\LogstashFormatter('emails', 'docker')) + ], [ + new Monolog\Processor\PsrLogMessageProcessor(), + new Monolog\Processor\WebProcessor(), + new Monolog\Processor\IntrospectionProcessor(), + new Monolog\Processor\MemoryPeakUsageProcessor() + ]); } ]; diff --git a/api/src/Model/Attachment.php b/api/src/Model/Attachment.php index 49bf820..912a2e8 100644 --- a/api/src/Model/Attachment.php +++ b/api/src/Model/Attachment.php @@ -169,4 +169,8 @@ class Attachment implements Model 'decrypted' => $this->isDecrypted() ]; } -} \ No newline at end of file + public function jsonSerialize(): mixed + { + return $this->toArray(); + } +} diff --git a/api/src/Model/Job.php b/api/src/Model/Job.php index 421f79a..b4a9cc5 100644 --- a/api/src/Model/Job.php +++ b/api/src/Model/Job.php @@ -58,4 +58,8 @@ class Job implements Model 'executed' => $this->isExecuted() ]; } -} \ No newline at end of file + public function jsonSerialize(): mixed + { + return $this->toArray(); + } +} diff --git a/api/src/Model/Mailbox.php b/api/src/Model/Mailbox.php index 0a4221a..770f715 100644 --- a/api/src/Model/Mailbox.php +++ b/api/src/Model/Mailbox.php @@ -104,6 +104,9 @@ class Mailbox implements Model public function lastPosition(): int { $state = $this->lastState()->getUIDs(); + if (count($state) === 0) { + return 0; + } return array_key_last($state); } @@ -119,4 +122,8 @@ class Mailbox implements Model ] ]; } -} \ No newline at end of file + public function jsonSerialize(): mixed + { + return $this->toArray(); + } +} diff --git a/api/src/Model/Message.php b/api/src/Model/Message.php index d9a4373..5992956 100644 --- a/api/src/Model/Message.php +++ b/api/src/Model/Message.php @@ -99,14 +99,14 @@ class Message implements Model { if (!isset($this->states)) { try { - $this->setStates($this->getFactory()->find(\ProVM\Emails\Model\State\Message::class)->fetchByMessage($this->getId())); + $this->setStates($this->getFactory()->find(State\Message::class)->fetchByMessage($this->getId())); } catch (BlankResult $e) { return []; } } return $this->states; } - public function getState(string $name): \ProVM\Emails\Model\State\Message + public function getState(string $name): State\Message { try { return $this->getStates()[$name]; @@ -115,7 +115,7 @@ class Message implements Model return $this->getStates()[$name]; } } - public function addState(\ProVM\Emails\Model\State\Message $state): Message + public function addState(State\Message $state): Message { $this->states[$state->getName()] = $state; return $this; @@ -129,7 +129,7 @@ class Message implements Model } protected function newState(string $name): Message { - $this->addState((new \ProVM\Emails\Model\State\Message()) + $this->addState((new State\Message()) ->setName($name) ->setMessage($this) ->setValue(false) @@ -219,4 +219,8 @@ class Message implements Model }, $this->getAttachments()) : [] ]; } + public function jsonSerialize(): mixed + { + return $this->toArray(); + } } diff --git a/api/src/Model/State/Attachment.php b/api/src/Model/State/Attachment.php index a50961b..edb5e86 100644 --- a/api/src/Model/State/Attachment.php +++ b/api/src/Model/State/Attachment.php @@ -47,4 +47,18 @@ class Attachment implements Model $this->value = $value; return $this; } -} \ No newline at end of file + + public function toArray(): array + { + return [ + 'id' => $this->getId(), + 'attachment_id' => $this->getAttachment()->getId(), + 'name' => $this->getName(), + 'value' => $this->getValue() + ]; + } + public function jsonSerialize(): mixed + { + return $this->toArray(); + } +} diff --git a/api/src/Model/State/Mailbox.php b/api/src/Model/State/Mailbox.php index 35d5df6..35e66d3 100644 --- a/api/src/Model/State/Mailbox.php +++ b/api/src/Model/State/Mailbox.php @@ -30,7 +30,7 @@ class Mailbox implements Model } public function getUIDs(): array { - return $this->uids; + return $this->uids ?? []; } public function setId(int $id): Mailbox @@ -65,4 +65,19 @@ class Mailbox implements Model } return $this; } -} \ No newline at end of file + + public function toArray(): array + { + return [ + 'id' => $this->getId(), + 'mailbox_id' => $this->getMailbox()->getId(), + 'date_time' => $this->getDateTime()->format('Y-m-d H:i:s'), + 'count' => $this->getCount(), + 'uids' => $this->getUIDs() + ]; + } + public function jsonSerialize(): mixed + { + return $this->toArray(); + } +} diff --git a/api/src/Model/State/Message.php b/api/src/Model/State/Message.php index c1fab5c..304cdd6 100644 --- a/api/src/Model/State/Message.php +++ b/api/src/Model/State/Message.php @@ -47,4 +47,18 @@ class Message implements Model $this->value = $value; return $this; } -} \ No newline at end of file + + public function toArray(): array + { + return [ + 'id' => $this->getId(), + 'message_id' => $this->getMessage()->getId(), + 'name' => $this->getName(), + 'value' => $this->getValue() + ]; + } + public function jsonSerialize(): mixed + { + return $this->toArray(); + } +} diff --git a/api/src/Repository/Attachment.php b/api/src/Repository/Attachment.php index 568d4b6..d27f59c 100644 --- a/api/src/Repository/Attachment.php +++ b/api/src/Repository/Attachment.php @@ -28,6 +28,20 @@ class Attachment extends Repository return $this; } + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `message_id` INT UNSIGNED NOT NULL, + `filename` VARCHAR(255) NOT NULL, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_messages_{$this->getTable()}` (`message_id`) + REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +)"; + $this->getConnection()->query($query); + } + protected function fieldsForInsert(): array { return [ @@ -115,4 +129,11 @@ class Attachment extends Repository $query = "SELECT * FROM {$this->getTable()} WHERE message_id = ?"; return $this->fetchMany($query, [$message_id]); } -} \ No newline at end of file + public function fetchDownloaded(): array + { + $query = "SELECT a.* +FROM `{$this->getTable()}` a JOIN `attachments_states` `as` ON `as`.attachment_id = a.id +WHERE `as`.name = 'downloaded' AND `as`.value = 1"; + return $this->fetchMany($query); + } +} diff --git a/api/src/Repository/Job.php b/api/src/Repository/Job.php index 164d900..0f47ee3 100644 --- a/api/src/Repository/Job.php +++ b/api/src/Repository/Job.php @@ -29,6 +29,21 @@ class Job extends Repository return $this; } + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `message_id` INT UNSIGNED NOT NULL, + `date_time` DATETIME NOT NULL, + `executed` INT(1) UNSIGNED DEFAULT 0, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_messages_{$this->getTable()}` (`message_id`) + REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +)"; + $this->getConnection()->query($query); + } + protected function fieldsForUpdate(): array { return $this->fieldsForInsert(); @@ -105,4 +120,4 @@ class Job extends Repository $query = "SELECT * FROM {$this->getTable()} WHERE `message_id` = ? AND `executed` = 0"; return $this->fetchOne($query, [$message_id]); } -} \ No newline at end of file +} diff --git a/api/src/Repository/Mailbox.php b/api/src/Repository/Mailbox.php index 50a0b92..212355d 100644 --- a/api/src/Repository/Mailbox.php +++ b/api/src/Repository/Mailbox.php @@ -28,6 +28,18 @@ class Mailbox extends Repository return $this; } + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `name` VARCHAR(100) NOT NULL, + `validity` INT UNSIGNED NOT NULL, + PRIMARY KEY (`id`) +)"; + $this->getConnection()->query($query); + } + protected function fieldsForUpdate(): array { return $this->fieldsForInsert(); @@ -86,4 +98,4 @@ class Mailbox extends Repository $query = "SELECT * FROM `{$this->getTable()}` WHERE `name` = ?"; return $this->fetchOne($query, [$name]); } -} \ No newline at end of file +} diff --git a/api/src/Repository/Message.php b/api/src/Repository/Message.php index 45d192f..11ed659 100644 --- a/api/src/Repository/Message.php +++ b/api/src/Repository/Message.php @@ -3,34 +3,50 @@ namespace ProVM\Emails\Repository; use DateTimeInterface; use PDO; -use PDOException; -use Exception; -use ProVM\Common\Define\Model; use Psr\Log\LoggerInterface; -use ProVM\Common\Implement\Repository; use Safe\DateTimeImmutable; -use Safe\Exceptions\ErrorfuncException; +use ProVM\Common\Define\Model; +use ProVM\Common\Implement\Repository; +use ProVM\Common\Factory; class Message extends Repository { - public function __construct(PDO $connection, LoggerInterface $logger, \ProVM\Common\Factory\Model $factory) + public function __construct(PDO $connection, LoggerInterface $logger, Factory\Model $factory) { parent::__construct($connection, $logger); $this->setTable('messages') ->setFactory($factory); } - protected \ProVM\Common\Factory\Model $factory; - public function getFactory(): \ProVM\Common\Factory\Model + protected Factory\Model $factory; + public function getFactory(): Factory\Model { return $this->factory; } - public function setFactory(\ProVM\Common\Factory\Model $factory): Message + public function setFactory(Factory\Model $factory): Message { $this->factory = $factory; return $this; } + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `uid` VARCHAR(255) NOT NULL, + `mailbox_id` INT UNSIGNED NOT NULL, + `position` INT UNSIGNED NOT NULL, + `subject` VARCHAR(255) NOT NULL, + `from` VARCHAR(255) NOT NULL, + `date_time` DATETIME NOT NULL, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_mailboxes_{$this->getTable()}` (`mailbox_id`) + REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +)"; + $this->getConnection()->query($query); + } + protected function fieldsForUpdate(): array { return $this->fieldsForInsert(); @@ -114,21 +130,23 @@ class Message extends Repository 'downloaded_attachments', 'scheduled_downloads' ]; + $stateRepository = $this->getFactory()->find(\ProVM\Emails\Model\State\Message::class); foreach ($valid_states as $state_name) { try { $model->getState($state_name); } catch (\Exception $e) { + $this->getLogger()->warning($e); $data = [ 'message_id' => $model->getId(), 'name' => $state_name ]; - $state = $this->getFactory()->find(\ProVM\Emails\Model\State\Message::class)->create($data); + $state = $stateRepository->create($data); $model->addState($state); } } foreach ($model->getStates() as $state) { - $state->setMessage($model); - $this->getFactory()->find(\ProVM\Emails\Model\State\Message::class)->save($state); + //$state->setMessage($model); + $stateRepository->save($state); } } @@ -161,4 +179,9 @@ class Message extends Repository WHERE `mailbox_id` = ? `subject` = ? AND `from` = ? AND `date_time` = ?"; return $this->fetchOne($query, [$mailbox_id, $subject, $from, $dateTime->format('Y-m-d H:i:s')]); } + public function fetchAllBySubjectAndDate(string $subject, DateTimeInterface $dateTime): array + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `subject` = ? AND `date_time` BETWEEN ? AND ?"; + return $this->fetchMany($query, [$subject, $dateTime->format('Y-m-d 00:00:00'), $dateTime->format('Y-m-d 23:59:59')]); + } } diff --git a/api/src/Repository/State/Attachment.php b/api/src/Repository/State/Attachment.php index 7da6b9b..2240eda 100644 --- a/api/src/Repository/State/Attachment.php +++ b/api/src/Repository/State/Attachment.php @@ -26,6 +26,22 @@ class Attachment extends Repository return $this; } + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `attachment_id` INT UNSIGNED NOT NULL, + `name` VARCHAR(100) NOT NULL, + `value` INT(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_attachments_{$this->getTable()}` (`attachment_id`) + REFERENCES `attachments` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) +"; + $this->getConnection()->query($query); + } + protected function fieldsForInsert(): array { return [ @@ -90,4 +106,4 @@ class Attachment extends Repository $query = "SELECT * FROM `{$this->getTable()}` WHERE `attachment_id` = ? AND `name` = ?"; return $this->fetchOne($query, [$attachment_id, $name]); } -} \ No newline at end of file +} diff --git a/api/src/Repository/State/Mailbox.php b/api/src/Repository/State/Mailbox.php index 7d498d0..e6e03a5 100644 --- a/api/src/Repository/State/Mailbox.php +++ b/api/src/Repository/State/Mailbox.php @@ -28,6 +28,23 @@ class Mailbox extends Repository return $this; } + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `mailbox_id` INT UNSIGNED NOT NULL, + `date_time` DATETIME NOT NULL, + `count` INT UNSIGNED NOT NULL, + `uids` TEXT NOT NULL, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_mailboxes_{$this->getTable()}` (`mailbox_id`) + REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) +"; + $this->getConnection()->query($query); + } + protected function fieldsForUpdate(): array { return $this->fieldsForInsert(); @@ -99,4 +116,4 @@ class Mailbox extends Repository $query = "SELECT * FROM `{$this->getTable()}` WHERE `mailbox_id` = ? AND `date_time` = ?"; return $this->fetchOne($query, [$mailbox_id, $date_time]); } -} \ No newline at end of file +} diff --git a/api/src/Repository/State/Message.php b/api/src/Repository/State/Message.php index 74acb5b..8859eb8 100644 --- a/api/src/Repository/State/Message.php +++ b/api/src/Repository/State/Message.php @@ -26,6 +26,22 @@ class Message extends Repository return $this; } + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `message_id` INT UNSIGNED NOT NULL, + `name` VARCHAR(100) NOT NULL, + `value` INT(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_messages_{$this->getTable()}` (`message_id`) + REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) +"; + $this->getConnection()->query($query); + } + protected function fieldsForUpdate(): array { return $this->fieldsForInsert(); @@ -90,4 +106,4 @@ class Message extends Repository $query = "SELECT * FROM `{$this->getTable()}` WHERE `message_id` = ? AND `name` = ?"; return $this->fetchOne($query, [$message_id, $name]); } -} \ No newline at end of file +} diff --git a/cli/common/Command/DecryptPdf.php b/cli/common/Command/DecryptPdf.php index 92027b6..e868843 100644 --- a/cli/common/Command/DecryptPdf.php +++ b/cli/common/Command/DecryptPdf.php @@ -1,12 +1,13 @@ getCommunicator()->get('/attachments/pending'); - return \Safe\json_decode($response->getBody()->getContents())->attachments; + return json_decode($response->getBody()->getContents())->attachments; } protected function decrypt(string $attachment): bool { $response = $this->getCommunicator()->put('/attachments/decrypt', ['attachments' => [$attachment]]); - return \Safe\json_decode($response->getBody()->getContents())->status; + return json_decode($response->getBody()->getContents())->status; } public function execute(InputInterface $input, OutputInterface $output) @@ -64,4 +65,4 @@ class DecryptPdf extends Command return Command::SUCCESS; } -} \ No newline at end of file +} diff --git a/cli/common/Command/GrabAttachments.php b/cli/common/Command/GrabAttachments.php index 1055a8c..23295e1 100644 --- a/cli/common/Command/GrabAttachments.php +++ b/cli/common/Command/GrabAttachments.php @@ -1,12 +1,13 @@ getCommunicator()->get('/messages/pending'); - return \Safe\json_decode($response->getBody()->getContents())->messages; + return json_decode($response->getBody()->getContents())->messages; } protected function grabAttachments(int $message_uid): int { $response = $this->getCommunicator()->put('/attachments/grab', ['messages' => [$message_uid]]); - return \Safe\json_decode($response->getBody()->getContents())->attachment_count; + return json_decode($response->getBody()->getContents())->attachment_count; } public function execute(InputInterface $input, OutputInterface $output): int @@ -62,4 +63,4 @@ class GrabAttachments extends Command return Command::SUCCESS; } -} \ No newline at end of file +} diff --git a/cli/common/Command/Messages.php b/cli/common/Command/Messages.php index a09b493..848436e 100644 --- a/cli/common/Command/Messages.php +++ b/cli/common/Command/Messages.php @@ -1,12 +1,13 @@ getCommunicator()->get('/mailboxes/registered'); - return \Safe\json_decode($response->getBody()->getContents())->mailboxes; + return json_decode($response->getBody()->getContents())->mailboxes; } protected function grabMessages(string $mailbox): int { $response = $this->getCommunicator()->put('/messages/grab', ['mailboxes' => [$mailbox]]); - $body = \Safe\json_decode($response->getBody()->getContents()); + $body = json_decode($response->getBody()->getContents()); return $body->message_count; } @@ -63,4 +64,4 @@ class Messages extends Command return Command::SUCCESS; } -} \ No newline at end of file +} diff --git a/emails-2022-11-30-03-59-49.sql b/emails-2022-11-30-03-59-49.sql new file mode 100644 index 0000000..3fd22d9 --- /dev/null +++ b/emails-2022-11-30-03-59-49.sql @@ -0,0 +1,94 @@ +-- Adminer 4.8.1 MySQL 5.5.5-10.9.3-MariaDB-1:10.9.3+maria~ubu2204 dump + +SET NAMES utf8; +SET time_zone = '+00:00'; +SET foreign_key_checks = 0; +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; + +SET NAMES utf8mb4; + +DROP TABLE IF EXISTS `attachments`; +CREATE TABLE `attachments` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `message_id` int(10) unsigned NOT NULL, + `filename` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + KEY `message_id` (`message_id`), + CONSTRAINT `attachments_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +DROP TABLE IF EXISTS `attachments_jobs`; +CREATE TABLE `attachments_jobs` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `message_id` int(10) unsigned NOT NULL, + `date_time` datetime NOT NULL, + `executed` int(1) DEFAULT 0, + PRIMARY KEY (`id`), + KEY `message_id` (`message_id`), + CONSTRAINT `attachments_jobs_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +DROP TABLE IF EXISTS `attachments_states`; +CREATE TABLE `attachments_states` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `attachment_id` int(10) unsigned NOT NULL, + `name` varchar(100) NOT NULL, + `value` int(1) unsigned NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + KEY `attachment_id` (`attachment_id`), + CONSTRAINT `attachments_states_ibfk_1` FOREIGN KEY (`attachment_id`) REFERENCES `attachments` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +DROP TABLE IF EXISTS `mailboxes`; +CREATE TABLE `mailboxes` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `name` varchar(100) NOT NULL, + `validity` int(11) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +DROP TABLE IF EXISTS `mailboxes_states`; +CREATE TABLE `mailboxes_states` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `mailbox_id` int(10) unsigned NOT NULL, + `date_time` datetime NOT NULL, + `count` int(10) unsigned NOT NULL, + `uids` text NOT NULL, + PRIMARY KEY (`id`), + KEY `mailbox_id` (`mailbox_id`), + CONSTRAINT `mailboxes_states_ibfk_2` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +DROP TABLE IF EXISTS `messages`; +CREATE TABLE `messages` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `mailbox_id` int(10) unsigned NOT NULL, + `position` int(10) unsigned NOT NULL, + `uid` varchar(255) NOT NULL, + `subject` varchar(255) NOT NULL, + `from` varchar(100) NOT NULL, + `date_time` datetime NOT NULL, + PRIMARY KEY (`id`), + KEY `mailbox_id` (`mailbox_id`), + CONSTRAINT `messages_ibfk_1` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +DROP TABLE IF EXISTS `messages_states`; +CREATE TABLE `messages_states` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `message_id` int(10) unsigned NOT NULL, + `name` varchar(100) NOT NULL, + `value` int(1) unsigned NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + KEY `message_id` (`message_id`), + CONSTRAINT `messages_states_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +-- 2022-11-30 03:59:49 From 03c1dac2f2bfb107b41a92c9f02cc18e41a4c191 Mon Sep 17 00:00:00 2001 From: Aldarien Date: Fri, 9 Jun 2023 00:54:34 -0400 Subject: [PATCH 2/3] Cleanup of cli --- NOTES.md | 33 ++--- api/common/Exception/Job/Stateless.php | 15 +++ api/common/Exception/Mailbox/EmptyMailbox.php | 5 +- api/common/Exception/Mailbox/Invalid.php | 3 +- api/common/Exception/Mailbox/Stateless.php | 3 +- api/resources/routes/01_mailboxes.php | 1 + api/resources/routes/02_attachments.php | 3 +- api/setup/setups/98_log.php | 3 +- api/src/Model/Job.php | 65 ++++++++-- api/src/Model/State/Job.php | 65 ++++++++++ api/src/Repository/Job.php | 67 +++++------ api/src/Repository/State/Job.php | 113 ++++++++++++++++++ cli/Dockerfile | 2 +- cli/common/Command/DecryptPdf.php | 68 ----------- cli/common/Command/GrabAttachments.php | 66 ---------- cli/common/Command/Jobs/Check.php | 79 ++++++++++++ cli/common/Command/Mailboxes/Check.php | 76 ++++++++++++ .../{Messages.php => Messages/Grab.php} | 43 ++++--- cli/common/Middleware/Logging.php | 2 +- cli/common/Service/Communicator.php | 33 ++++- cli/common/Wrapper/Application.php | 4 +- cli/composer.json | 2 +- cli/crontab | 11 +- cli/docker-compose.yml | 3 +- cli/public/index.php | 9 +- cli/resources/commands/01_mailboxes.php | 2 + cli/resources/commands/01_messages.php | 2 +- cli/resources/commands/02_attachments.php | 5 +- cli/resources/commands/03_jobs.php | 2 + cli/setup/app.php | 4 +- cli/setup/middleware/98_log.php | 2 +- cli/setup/settings/01_env.php | 5 +- cli/setup/setups/02_api.php | 6 +- cli/setup/setups/03_middleware.php | 4 +- cli/setup/setups/04_commands.php | 20 ++++ cli/setup/setups/98_log.php | 60 +++++++--- 36 files changed, 614 insertions(+), 272 deletions(-) create mode 100644 api/common/Exception/Job/Stateless.php create mode 100644 api/src/Model/State/Job.php create mode 100644 api/src/Repository/State/Job.php delete mode 100644 cli/common/Command/DecryptPdf.php delete mode 100644 cli/common/Command/GrabAttachments.php create mode 100644 cli/common/Command/Jobs/Check.php create mode 100644 cli/common/Command/Mailboxes/Check.php rename cli/common/Command/{Messages.php => Messages/Grab.php} (54%) create mode 100644 cli/resources/commands/01_mailboxes.php create mode 100644 cli/resources/commands/03_jobs.php create mode 100644 cli/setup/setups/04_commands.php diff --git a/NOTES.md b/NOTES.md index ec9035b..b00fae6 100644 --- a/NOTES.md +++ b/NOTES.md @@ -7,8 +7,14 @@ * [ ] Download `attachments` (*encrypted* & *decrypted*). ## CLI -* [x] Get `mailboxes` from **[API]** then run `grab messages` job in **[API]** for each one. -* [x] Get `pending attachments` jobs from **[API]** and run. +#### Automatic +* [x] `mailboxes:check`: Get *registered* `mailboxes` and schedule `messages:grab` for `mailbox_id`. +* [x] `attachments:check`: Check *saved* `attachments` and schedule `attachments:decrypt` for `attachment_id`. +* [x] `jobs:check`: Get *pending* `jobs` and run them. +#### Scheduled +* [x] `messages:grab`: Grab `messages` for `mailbox`. Arguments: `mailbox_id`. +* [x] `attachments:grab`: Grab `attachments` for `message`. Arguments: `message_id`. +* [x] `attachments:decrypt`: Decrypt `attachment`. Arguments: `attachment_id`. ## API * [x] Grab all `mailboxes` from `Email Provider`, identifying those that are registered. @@ -21,18 +27,17 @@ ## Workflow -* **[User]** Choose `mailboxes` to register or unregister - -> **[API]** Register selected `mailboxes` and get `messages` for recently registered. -* **[Cron]** Get registered `mailboxes` -> **[API]** Get `messages` -* **[User]** Check messages found -> **[API]** Schedule `attachments` -* **[Cron]** Get `attachment download` jobs -> **[API]** grab `attachments` +* **[User]** Choose `mailboxes` to register or unregister. + -> **[API]** Register selected `mailboxes`, register new `messages:grab` job. +* **[Cron]** Get `jobs`, run `jobs`. +* **[User]** Check messages found -> **[API]** Schedule `attachments`. -### Jobs +## Jobs #### Automatic -* [ ] Check registered `mailboxes` for new `messages`, logging last check. -* [ ] Check if `attachments` are `encrypted`. -* [ ] Check for new scheduled jobs. +* [x] Check *registered* `mailboxes` for new `messages`. Every weekday. +* [x] Check if `attachments` are *encrypted*. Every weekday. +* [x] Check for new *scheduled* `jobs`. Every minute. #### Scheduled -* [ ] Grab `messages`. -* [ ] Grab `attachments`. -* [ ] Decrypt `attachments`. +* [ ] Grab `messages` for `mailbox` id. +* [ ] Grab `attachments` for `message` id. +* [ ] Decrypt `attachment`. diff --git a/api/common/Exception/Job/Stateless.php b/api/common/Exception/Job/Stateless.php new file mode 100644 index 0000000..7f2d7f5 --- /dev/null +++ b/api/common/Exception/Job/Stateless.php @@ -0,0 +1,15 @@ +group('/mailboxes', function($app) { $app->group('/mailbox/{mailbox_id}', function($app) { $app->group('/messages', function($app) { + $app->get('/grab[/]', [Messages::class, 'grab']); $app->get('/valid[/]', [Messages::class, 'valid']); $app->get('[/]', Messages::class); }); diff --git a/api/resources/routes/02_attachments.php b/api/resources/routes/02_attachments.php index 62969b7..9b45b9d 100644 --- a/api/resources/routes/02_attachments.php +++ b/api/resources/routes/02_attachments.php @@ -3,9 +3,10 @@ use ProVM\Common\Controller\Attachments; $app->group('/attachments', function($app) { $app->put('/grab', [Attachments::class, 'grab']); + $app->get('/pending', [Attachments::class, 'pending']); $app->post('/decrypt', [Attachments::class, 'decrypt']); $app->get('[/]', Attachments::class); }); $app->group('/attachment/{attachment_id}', function($app) { $app->get('[/]', [Attachments::class, 'get']); -}); \ No newline at end of file +}); diff --git a/api/setup/setups/98_log.php b/api/setup/setups/98_log.php index 4c6a66e..0ebc4cb 100644 --- a/api/setup/setups/98_log.php +++ b/api/setup/setups/98_log.php @@ -6,6 +6,7 @@ return [ return [ $container->get(Monolog\Processor\PsrLogMessageProcessor::class), $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\WebProcessor::class), $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class), ]; }, @@ -36,7 +37,7 @@ return [ ); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - return $container->get('elk_logger'); + return $container->get('file_logger'); }, 'file_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/api/src/Model/Job.php b/api/src/Model/Job.php index b4a9cc5..5aa26ee 100644 --- a/api/src/Model/Job.php +++ b/api/src/Model/Job.php @@ -3,29 +3,31 @@ namespace ProVM\Emails\Model; use DateTimeInterface; use ProVM\Common\Define\Model; +use ProVM\Common\Exception\Database\BlankResult; +use ProVM\Common\Exception\Job\Stateless; +use ProVM\Emails; class Job implements Model { protected int $id; - protected Message $message; - protected DateTimeInterface $dateTime; - protected bool $executed; + protected string $command; + protected string $arguments; public function getId(): int { return $this->id; } - public function getMessage(): Message + public function getCommand(): string { - return $this->message; + return $this->command; } - public function getDateTime(): DateTimeInterface + public function getArguments(): string { - return $this->dateTime; + return $this->arguments ?? ''; } public function isExecuted(): bool { - return $this->executed ?? false; + return $this->lastState()->getStatus() === State\Job::Executed; } public function setId(int $id): Job @@ -33,22 +35,61 @@ class Job implements Model $this->id = $id; return $this; } - public function setMessage(Message $message): Job + public function setCommand(string $message): Job { $this->message = $message; return $this; } - public function setDateTime(DateTimeInterface $dateTime): Job + public function setArguments(string $dateTime): Job { $this->dateTime = $dateTime; return $this; } - public function wasExecuted(): Job + + protected Emails\Repository\State\Job $stateRepository; + public function getStateRepository(): Emails\Repository\State\Job { - $this->executed = true; + return $this->stateRepository; + } + public function setStateRepository(Emails\Repository\State\Job $repository): Job + { + $this->stateRepository = $repository; return $this; } + protected array $states; + public function getStates(): array + { + if (!isset($this->states)) { + try { + $this->setStates($this->getStateRepository()->fetchByJob($this->getId())); + } catch (BlankResult $e) { + return []; + } + } + return $this->states; + } + public function addState(State\Job $state): Job + { + $this->states []= $state; + return $this; + } + public function setStates(array $states): Job + { + foreach ($states as $state) { + $this->addState($state); + } + return $this; + } + + public function lastState(): State\Job + { + if (count($this->getStates()) === 0) { + throw new Stateless($this); + } + return $this->getStates()[array_key_last($this->getStates())]; + } + public function toArray(): array { return [ diff --git a/api/src/Model/State/Job.php b/api/src/Model/State/Job.php new file mode 100644 index 0000000..2826908 --- /dev/null +++ b/api/src/Model/State/Job.php @@ -0,0 +1,65 @@ +id; + } + public function getJob(): Emails\Model\Job + { + return $this->job; + } + public function getDateTime(): \DateTimeInterface + { + return $this->dateTime; + } + public function getStatus(): int + { + return $this->status; + } + + public function setId(int $id): Job + { + $this->id = $id; + return $this; + } + public function setJob(Emails\Model\Job $job): Job + { + $this->job = $job; + return $this; + } + public function setDateTime(\DateTimeInterface $dateTime): Job + { + $this->dateTime = $dateTime; + return $this; + } + public function setStatus(int $status): Job + { + $this->status = $status; + return $this; + } + + public function jsonSerialize(): mixed + { + return [ + 'id' => $this->getId(), + 'job' => $this->getJob(), + 'date' => $this->getDateTime(), + 'status' => $this->getStatus() + ]; + } +} diff --git a/api/src/Repository/Job.php b/api/src/Repository/Job.php index 0f47ee3..e8f6be9 100644 --- a/api/src/Repository/Job.php +++ b/api/src/Repository/Job.php @@ -2,28 +2,28 @@ namespace ProVM\Emails\Repository; use PDO; -use ProVM\Common\Factory\Model; +use ProVM\Common\Factory; use Psr\Log\LoggerInterface; -use Safe\DateTimeImmutable; use ProVM\Common\Define\Model as ModelInterface; use ProVM\Common\Implement\Repository; use ProVM\Emails\Model\Job as BaseModel; +use ProVM\Emails; class Job extends Repository { - public function __construct(PDO $connection, LoggerInterface $logger, Model $factory) + public function __construct(PDO $connection, LoggerInterface $logger, Factory\Model $factory) { parent::__construct($connection, $logger); $this->setFactory($factory) ->setTable('attachments_jobs'); } - protected \ProVM\Common\Factory\Model $factory; - public function getFactory(): \ProVM\Common\Factory\Model + protected Factory\Model $factory; + public function getFactory(): Factory\Model { return $this->factory; } - public function setFactory(\ProVM\Common\Factory\Model $factory): Job + public function setFactory(Factory\Model $factory): Job { $this->factory = $factory; return $this; @@ -34,12 +34,9 @@ class Job extends Repository $query = " CREATE TABLE {$this->getTable()} ( `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, - `message_id` INT UNSIGNED NOT NULL, - `date_time` DATETIME NOT NULL, - `executed` INT(1) UNSIGNED DEFAULT 0, - PRIMARY KEY (`id`), - FOREIGN KEY `fk_messages_{$this->getTable()}` (`message_id`) - REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE + `command` VARCHAR(100) NOT NULL, + `arguments` TEXT NOT NULL, + PRIMARY KEY (`id`) )"; $this->getConnection()->query($query); } @@ -55,65 +52,61 @@ CREATE TABLE {$this->getTable()} ( protected function fieldsForInsert(): array { return [ - 'message_id', - 'date_time', - 'executed' + 'command', + 'arguments', ]; } protected function valuesForInsert(ModelInterface $model): array { return [ - $model->getMessage()->getId(), - $model->getDateTime()->format('Y-m-d H:i:s'), - $model->isExecuted() ? 1 : 0 + $model->getCommand(), + $model->getArguments(), ]; } protected function defaultFind(ModelInterface $model): ModelInterface { - return $this->fetchByMessageAndDate($model->getMessage()->getId(), $model->getDateTime()->format('Y-m-d H:i:s')); + return $this->fetchByCommandAndArguments($model->getCommand(), $model->getArguments()); } protected function fieldsForCreate(): array { return [ - 'message_id', - 'date_time', - 'executed' + 'command', + 'arguments', ]; } protected function valuesForCreate(array $data): array { return [ - $data['message_id'], - $data['date_time'], - $data['executed'] ?? 0 + $data['command'], + $data['arguments'], ]; } protected function defaultSearch(array $data): ModelInterface { - return $this->fetchByMessageAndDate($data['message_id'], $data['date_time']); + return $this->fetchByCommandAndArguments($data['command'], $data['arguments']); } public function load(array $row): ModelInterface { - $model = (new BaseModel()) + return (new BaseModel()) ->setId($row['id']) - ->setMessage($this->getFactory()->find(\ProVM\Emails\Model\Message::class)->fetchById($row['message_id'])) - ->setDateTime(new DateTimeImmutable($row['date_time'])); - if ($row['executed'] ?? 0 === 1) { - $model->wasExecuted(); - } - return $model; + ->setCommand($row['command']) + ->setArguments($row['arguments']) + ->setStateRepository($this->getFactory()->find(Emails\Model\State\Job::class)); } public function fetchAllPending(): array { - $query = "SELECT * FROM {$this->getTable()} WHERE `executed` = 0"; + $query = "SELECT a.* +FROM {$this->getTable()} a + JOIN `jobs_states` b ON b.job_id = a.id +WHERE b.`status` = ?"; return $this->fetchMany($query); } - public function fetchByMessageAndDate(int $message_id, string $date_time): \ProVM\Emails\Model\Job + public function fetchByCommandAndArguments(string $command, string $arguments): \ProVM\Emails\Model\Job { - $query = "SELECT * FROM {$this->getTable()} WHERE `message_id` = ? AND `date_time` = ?"; - return $this->fetchOne($query, [$message_id, $date_time]); + $query = "SELECT * FROM {$this->getTable()} WHERE `command` = ? AND `arguments` = ?"; + return $this->fetchOne($query, [$command, $arguments]); } public function fetchPendingByMessage(int $message_id): \ProVM\Emails\Model\Job { diff --git a/api/src/Repository/State/Job.php b/api/src/Repository/State/Job.php new file mode 100644 index 0000000..e7f9d57 --- /dev/null +++ b/api/src/Repository/State/Job.php @@ -0,0 +1,113 @@ +setTable('jobs_states') + ->setFactory($factory); + } + + protected Factory\Model $factory; + public function getFactory(): Factory\Model + { + return $this->factory; + } + public function setFactory(Factory\Model $factory): Job + { + $this->factory = $factory; + return $this; + } + + public function install(): void + { + $query = " +CREATE TABLE {$this->getTable()} ( + `id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `job_id` INT UNSIGNED NOT NULL, + `date_time` DATETIME NOT NULL, + `status` INT NOT NULL DEFAULT 1, + PRIMARY KEY (`id`), + FOREIGN KEY `fk_jobs_{$this->getTable()}` (`job_id`) + REFERENCES `jobs` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +)"; + $this->getConnection()->query($query); + } + + public function load(array $row): Define\Model + { + return (new Emails\Model\State\Job()) + ->setId($row['id']) + ->setJob($this->getFactory()->find(Emails\Model\Job::class)->fetchById($row['job_id'])) + ->setDateTime(new \DateTimeImmutable($row['date_time'])) + ->setStatus($row['status']); + } + + public function fetchByJob(int $job_id): array + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ?"; + return $this->fetchMany($query, [$job_id]); + } + public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\Job + { + $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ? AND `status` = ?"; + return $this->fetchOne($query, [$job_id, $status]); + } + + protected function fieldsForInsert(): array + { + return [ + 'job_id', + 'date_time', + 'status' + ]; + } + protected function valuesForInsert(Define\Model $model): array + { + return [ + $model->getJob()->getId(), + $model->getDateTime()->format('Y-m-d H:i:s'), + $model->getStatus() + ]; + } + + protected function fieldsForUpdate(): array + { + return $this->fieldsForInsert(); + } + protected function fieldsForCreate(): array + { + return $this->fieldsForInsert(); + } + + protected function valuesForUpdate(Define\Model $model): array + { + return $this->valuesForInsert($model); + } + protected function valuesForCreate(array $data): array + { + return [ + $data['job_id'], + $data['date_time'], + $data['status'] + ]; + } + + protected function defaultFind(Define\Model $model): Define\Model + { + return $this->fetchByJobAndStatus($model->getJob()->getId(), $model->getStatus()); + } + protected function defaultSearch(array $data): Define\Model + { + return $this->fetchByJobAndStatus($data['job_id'], $data['status']); + } +} diff --git a/cli/Dockerfile b/cli/Dockerfile index 2ed89eb..d596ae4 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -3,7 +3,7 @@ FROM php:8-cli ENV PATH ${PATH}:/app/bin RUN apt-get update \ - && apt-get install -y cron git libzip-dev unzip \ + && apt-get install -y cron git libzip-dev unzip qpdf \ && rm -r /var/lib/apt/lists/* \ && docker-php-ext-install zip diff --git a/cli/common/Command/DecryptPdf.php b/cli/common/Command/DecryptPdf.php deleted file mode 100644 index e868843..0000000 --- a/cli/common/Command/DecryptPdf.php +++ /dev/null @@ -1,68 +0,0 @@ -setCommunicator($communicator); - parent::__construct($name); - } - - protected Communicator $communicator; - public function getCommunicator(): Communicator - { - return $this->communicator; - } - public function setCommunicator(Communicator $communicator): DecryptPdf - { - $this->communicator = $communicator; - return $this; - } - - protected function getAttachments(): array - { - $response = $this->getCommunicator()->get('/attachments/pending'); - return json_decode($response->getBody()->getContents())->attachments; - } - protected function decrypt(string $attachment): bool - { - $response = $this->getCommunicator()->put('/attachments/decrypt', ['attachments' => [$attachment]]); - return json_decode($response->getBody()->getContents())->status; - } - - public function execute(InputInterface $input, OutputInterface $output) - { - $io = new SymfonyStyle($input, $output); - $io->title('Decrypt Attachments'); - - $io->section('Grabbing Attachments'); - $attachments = $this->getAttachments(); - $io->text('Found ' . count($attachments) . ' attachments.'); - $io->section('Decrypting Attachments'); - foreach ($attachments as $attachment) { - $status = $this->decrypt($attachment); - if ($status) { - $io->success("{$attachment} decrypted correctly."); - } else { - $io->error("Problem decrypting {$attachment}."); - } - } - $io->success('Done.'); - - return Command::SUCCESS; - } -} diff --git a/cli/common/Command/GrabAttachments.php b/cli/common/Command/GrabAttachments.php deleted file mode 100644 index 23295e1..0000000 --- a/cli/common/Command/GrabAttachments.php +++ /dev/null @@ -1,66 +0,0 @@ -setCommunicator($communicator); - parent::__construct($name); - } - - protected Communicator $service; - public function getCommunicator(): Communicator - { - return $this->service; - } - public function setCommunicator(Communicator $service): GrabAttachments - { - $this->service = $service; - return $this; - } - - protected function getMessages(): array - { - $response = $this->getCommunicator()->get('/messages/pending'); - return json_decode($response->getBody()->getContents())->messages; - } - protected function grabAttachments(int $message_uid): int - { - $response = $this->getCommunicator()->put('/attachments/grab', ['messages' => [$message_uid]]); - return json_decode($response->getBody()->getContents())->attachment_count; - } - - public function execute(InputInterface $input, OutputInterface $output): int - { - $io = new SymfonyStyle($input, $output); - $io->title('Grab Attachments'); - - $io->section('Grabbing Messages'); - $messages = $this->getMessages(); - $io->text('Found ' . count($messages) . ' messages.'); - $io->section('Grabbing Attachments'); - foreach ($messages as $job) { - $message = $job->message; - $attachments = $this->grabAttachments($message->uid); - $io->text("Found {$attachments} attachments for message UID:{$message->uid}."); - } - $io->success('Done.'); - - return Command::SUCCESS; - } -} diff --git a/cli/common/Command/Jobs/Check.php b/cli/common/Command/Jobs/Check.php new file mode 100644 index 0000000..d3c9b23 --- /dev/null +++ b/cli/common/Command/Jobs/Check.php @@ -0,0 +1,79 @@ +logger->notice('Grabbing pending jobs.'); + $response = $this->communicator->get('/jobs/pending'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + return json_decode($body)->jobs; + } + protected function runJob($job): bool + { + $base_command = '/app/bin/emails'; + $cmd = [$base_command, $job->command]; + if ($job->arguments !== '') { + $cmd []= $job->arguments; + } + $cmd = implode(' ', $cmd); + $this->logger->notice("Running '{$cmd}'"); + $response = shell_exec($cmd); + $this->logger->info("Result: {$response}"); + return $response !== false; + } + + public function execute(InputInterface $input, OutputInterface $output) + { + $section1 = $output->section(); + $section2 = $output->section(); + $io1 = new SymfonyStyle($input, $section1); + $io2 = new SymfonyStyle($input, $section2); + $io1->title('Checking Pending Jobs'); + $pending_jobs = $this->getPendingJobs(); + $notice = 'Found ' . count($pending_jobs) . ' jobs'; + $io1->text($notice); + $this->logger->info($notice); + if (count($pending_jobs) > 0) { + $io1->section('Running Jobs'); + $io1->progressStart(count($pending_jobs)); + foreach ($pending_jobs as $job) { + $section2->clear(); + $io2->text("Running {$job->command}"); + if ($this->runJob($job)) { + $io2->success('Success'); + } else { + $io2->error('Failure'); + } + $io1->progressAdvance(); + } + } + $section2->clear(); + $io2->success('Done'); + + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Mailboxes/Check.php b/cli/common/Command/Mailboxes/Check.php new file mode 100644 index 0000000..c0fa6eb --- /dev/null +++ b/cli/common/Command/Mailboxes/Check.php @@ -0,0 +1,76 @@ +communicator->get('/mailboxes/registered'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + return json_decode($body)->mailboxes; + } + protected function checkMailbox($mailbox): bool + { + if ((new \DateTimeImmutable())->diff(new \DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { + return true; + } + $response = $this->communicator->get("/mailbox/{$mailbox->id}/check"); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return true; + } + return json_decode($body)->status; + } + + public function execute(InputInterface $input, OutputInterface $output): int + { + $section1 = $output->section(); + $section2 = $output->section(); + $io1 = new SymfonyStyle($input, $section1); + $io2 = new SymfonyStyle($input, $section2); + $io1->title('Checking for New Messages'); + $mailboxes = $this->getMailboxes(); + $notice = 'Found ' . count($mailboxes) . ' mailboxes'; + $io1->text($notice); + if (count($mailboxes) > 0) { + $io1->section('Checking for new messages'); + $io1->progressStart(count($mailboxes)); + foreach ($mailboxes as $mailbox) { + $section2->clear(); + $io2->text("Checking {$mailbox->name}"); + if ($this->checkMailbox($mailbox)) { + $io2->success("Found new emails in {$mailbox->name}"); + } else { + $io2->info("No new emails in {$mailbox->name}"); + } + $io1->progressAdvance(); + } + $io1->progressFinish(); + } + $section2->clear(); + $io2->success('Done'); + + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Messages.php b/cli/common/Command/Messages/Grab.php similarity index 54% rename from cli/common/Command/Messages.php rename to cli/common/Command/Messages/Grab.php index 848436e..89a85f9 100644 --- a/cli/common/Command/Messages.php +++ b/cli/common/Command/Messages/Grab.php @@ -1,65 +1,62 @@ setCommunicator($communicator); parent::__construct($name); } + protected function configure() + { + $this->addArgument('mailbox_id', InputArgument::REQUIRED, 'Mailbox ID to grab emails'); + } protected Communicator $communicator; public function getCommunicator(): Communicator { return $this->communicator; } - public function setCommunicator(Communicator $communicator): Messages + public function setCommunicator(Communicator $communicator): Grab { $this->communicator = $communicator; return $this; } - protected function getMailboxes(): array + protected function grabMessages(int $mailbox_id): int { - $response = $this->getCommunicator()->get('/mailboxes/registered'); - return json_decode($response->getBody()->getContents())->mailboxes; - } - protected function grabMessages(string $mailbox): int - { - $response = $this->getCommunicator()->put('/messages/grab', ['mailboxes' => [$mailbox]]); - $body = json_decode($response->getBody()->getContents()); - return $body->message_count; + $response = $this->getCommunicator()->get("/mailbox/{$mailbox_id}/grab"); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + return json_decode($body)->messages->count; } public function execute(InputInterface $input, OutputInterface $output) { $io = new SymfonyStyle($input, $output); - $io->title('Messages'); - $io->section('Grabbing Registered Mailboxes'); - - $mailboxes = $this->getMailboxes(); - $io->text('Found ' . count($mailboxes) . ' registered mailboxes.'); + $mailbox_id = $input->getArgument('mailbox_id'); + $io->title("Grabbing Messages for Mailbox ID {$mailbox_id}"); $io->section('Grabbing Messages'); - foreach ($mailboxes as $mailbox) { - $message_count = $this->grabMessages($mailbox->name); - $io->text("Found {$message_count} messages in {$mailbox->name}."); - } + $count = $this->grabMessages($mailbox_id); + $io->info("Found {$count} messages"); $io->success('Done.'); return Command::SUCCESS; diff --git a/cli/common/Middleware/Logging.php b/cli/common/Middleware/Logging.php index a95c37b..25021c5 100644 --- a/cli/common/Middleware/Logging.php +++ b/cli/common/Middleware/Logging.php @@ -1,5 +1,5 @@ getStatusCode() < 200 or $response->getStatusCode() >= 300) { @@ -33,6 +38,11 @@ class Communicator } return $response; } + + /** + * @throws HttpResponseException + * @throws JsonException + */ protected function request(string $method, string $uri, ?array $body = null): ResponseInterface { $options = []; @@ -45,20 +55,39 @@ class Communicator return $this->handleResponse($this->getClient()->request($method, $uri, $options)); } + /** + * @throws HttpResponseException + * @throws JsonException + */ public function get(string $uri): ResponseInterface { return $this->request('get', $uri); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function post(string $uri, array $data): ResponseInterface { return $this->request('post', $uri, $data); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function put(string $uri, array $data): ResponseInterface { return $this->request('put', $uri, $data); } + + /** + * @throws HttpResponseException + * @throws JsonException + */ public function delete(string $uri, array $data): ResponseInterface { return $this->request('delete', $uri, $data); } -} \ No newline at end of file +} diff --git a/cli/common/Wrapper/Application.php b/cli/common/Wrapper/Application.php index 8d8e4aa..c27dbf4 100644 --- a/cli/common/Wrapper/Application.php +++ b/cli/common/Wrapper/Application.php @@ -1,5 +1,5 @@ container = $container; return $this; } -} \ No newline at end of file +} diff --git a/cli/composer.json b/cli/composer.json index 2c1d0c1..4365fcb 100644 --- a/cli/composer.json +++ b/cli/composer.json @@ -15,7 +15,7 @@ }, "autoload": { "psr-4": { - "ProVM\\Common\\": "common/" + "ProVM\\": "common/" } }, "authors": [ diff --git a/cli/crontab b/cli/crontab index 49d2722..fc8ab81 100644 --- a/cli/crontab +++ b/cli/crontab @@ -1,3 +1,10 @@ # minutes hour day_of_month month day_of_week command -0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log -0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log +#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log +#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log + +# Pending jobs every minute +1 * * * * /app/bin/emails jobs:pending >> /logs/jobs.log +# Check mailboxes for new emails every weekday +0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log +# Check attachments every weekday +0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log diff --git a/cli/docker-compose.yml b/cli/docker-compose.yml index 38d19e3..ea1ec32 100644 --- a/cli/docker-compose.yml +++ b/cli/docker-compose.yml @@ -12,4 +12,5 @@ services: - .key.env volumes: - ${CLI_PATH:-.}/:/app - - ./logs/cli:/logs + - ${LOGS_PATH}/cli:/logs + - ${ATT_PATH}:/attachments diff --git a/cli/public/index.php b/cli/public/index.php index 63341a5..f8b2dac 100644 --- a/cli/public/index.php +++ b/cli/public/index.php @@ -7,9 +7,8 @@ $app = require_once implode(DIRECTORY_SEPARATOR, [ Monolog\ErrorHandler::register($app->getContainer()->get(Psr\Log\LoggerInterface::class)); try { $app->run(); -} catch (Error | Exception $e) { - $logger = $app->getContainer()->get(Psr\Log\LoggerInterface::class); - $logger->debug(Safe\json_encode(compact('_SERVER'))); - $logger->error($e); - throw $e; +} catch (Error $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->error($e); +} catch (Exception $e) { + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->warning($e); } diff --git a/cli/resources/commands/01_mailboxes.php b/cli/resources/commands/01_mailboxes.php new file mode 100644 index 0000000..843e8e4 --- /dev/null +++ b/cli/resources/commands/01_mailboxes.php @@ -0,0 +1,2 @@ +add($app->getContainer()->get(ProVM\Command\Mailboxes\Check::class)); diff --git a/cli/resources/commands/01_messages.php b/cli/resources/commands/01_messages.php index c034ee1..4313973 100644 --- a/cli/resources/commands/01_messages.php +++ b/cli/resources/commands/01_messages.php @@ -1,2 +1,2 @@ add($app->getContainer()->get(\ProVM\Common\Command\Messages::class)); +$app->add($app->getContainer()->get(ProVM\Command\Messages\Grab::class)); diff --git a/cli/resources/commands/02_attachments.php b/cli/resources/commands/02_attachments.php index 85793e1..d46dbc1 100644 --- a/cli/resources/commands/02_attachments.php +++ b/cli/resources/commands/02_attachments.php @@ -1,3 +1,4 @@ add($app->getContainer()->get(\ProVM\Common\Command\GrabAttachments::class)); -$app->add($app->getContainer()->get(\ProVM\Common\Command\DecryptPdf::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Check::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Grab::class)); +$app->add($app->getContainer()->get(ProVM\Command\Attachments\Decrypt::class)); diff --git a/cli/resources/commands/03_jobs.php b/cli/resources/commands/03_jobs.php new file mode 100644 index 0000000..bd6eed1 --- /dev/null +++ b/cli/resources/commands/03_jobs.php @@ -0,0 +1,2 @@ +add($app->getContainer()->get(ProVM\Command\Jobs\Check::class)); diff --git a/cli/setup/app.php b/cli/setup/app.php index 16c1e0f..d00dbc6 100644 --- a/cli/setup/app.php +++ b/cli/setup/app.php @@ -1,7 +1,7 @@ build()); +$app = new ProVM\Wrapper\Application($builder->build()); $folder = implode(DIRECTORY_SEPARATOR, [ __DIR__, diff --git a/cli/setup/middleware/98_log.php b/cli/setup/middleware/98_log.php index 4df6f6c..5628448 100644 --- a/cli/setup/middleware/98_log.php +++ b/cli/setup/middleware/98_log.php @@ -1,2 +1,2 @@ add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); +//$app->add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); diff --git a/cli/setup/settings/01_env.php b/cli/setup/settings/01_env.php index 58b2a65..107718a 100644 --- a/cli/setup/settings/01_env.php +++ b/cli/setup/settings/01_env.php @@ -1,5 +1,8 @@ $_ENV['API_URI'], - 'api_key' => sha1($_ENV['API_KEY']) + 'api_key' => sha1($_ENV['API_KEY']), + 'passwords' => function() { + return explode($_ENV['PASSWORDS_SEPARATOR'] ?? ',', $_ENV['PASSWORDS'] ?? ''); + }, ]; diff --git a/cli/setup/setups/02_api.php b/cli/setup/setups/02_api.php index b123345..bf102d3 100644 --- a/cli/setup/setups/02_api.php +++ b/cli/setup/setups/02_api.php @@ -2,12 +2,12 @@ use Psr\Container\ContainerInterface; return [ - \Psr\Http\Client\ClientInterface::class => function(ContainerInterface $container) { - return new \GuzzleHttp\Client([ + Psr\Http\Client\ClientInterface::class => function(ContainerInterface $container) { + return new GuzzleHttp\Client([ 'base_uri' => $container->get('api_uri'), 'headers' => [ 'Authorization' => "Bearer {$container->get('api_key')}" ] ]); } -]; \ No newline at end of file +]; diff --git a/cli/setup/setups/03_middleware.php b/cli/setup/setups/03_middleware.php index 1d6cc44..7d872ab 100644 --- a/cli/setup/setups/03_middleware.php +++ b/cli/setup/setups/03_middleware.php @@ -2,7 +2,7 @@ use Psr\Container\ContainerInterface; return [ - ProVM\Common\Middleware\Logging::class => function(ContainerInterface $container) { - return new ProVM\Common\Middleware\Logging($container->get('request_logger')); + ProVM\Middleware\Logging::class => function(ContainerInterface $container) { + return new ProVM\Middleware\Logging($container->get('request_logger')); } ]; diff --git a/cli/setup/setups/04_commands.php b/cli/setup/setups/04_commands.php new file mode 100644 index 0000000..50ae9d1 --- /dev/null +++ b/cli/setup/setups/04_commands.php @@ -0,0 +1,20 @@ + function(ContainerInterface $container) { + return new ProVM\Command\Attachments\DecryptPdf( + $container->get(ProVM\Service\Communicator::class), + $container->get(Psr\Log\LoggerInterface::class), + 'qpdf', + $container->get('passwords') + ); + }, + ProVM\Command\Mailboxes\Check::class => function(ContainerInterface $container) { + return new ProVM\Command\Mailboxes\Check( + $container->get(ProVM\Service\Communicator::class), + 1 + ); + } +]; diff --git a/cli/setup/setups/98_log.php b/cli/setup/setups/98_log.php index b33a5ce..7f6e30b 100644 --- a/cli/setup/setups/98_log.php +++ b/cli/setup/setups/98_log.php @@ -2,28 +2,50 @@ use Psr\Container\ContainerInterface; return [ - Monolog\Handler\RotatingFileHandler::class => function(ContainerInterface $container) { - $handler = new Monolog\Handler\RotatingFileHandler($container->get('log_file')); - $handler->setFormatter($container->get(Monolog\Formatter\LineFormatter::class)); - return $handler; + 'log_processors' => function(ContainerInterface $container) { + return [ + $container->get(Monolog\Processor\PsrLogMessageProcessor::class), + $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class), + ]; + }, + 'request_log_handler' => function(ContainerInterface $container) { + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); }, 'request_logger' => function(ContainerInterface $container) { - $logger = new Monolog\Logger('request_logger'); - $handler = new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log'])); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - $dedupHandler = new Monolog\Handler\DeduplicationHandler($handler, null, Monolog\Level::Info); - $logger->pushHandler($dedupHandler); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger( + 'request_logger', + [$container->get('request_log_handler')], + $container->get('log_processors') + ); + }, + 'file_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler($container->get('log_file'))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Error + ); + }, + 'debug_log_handler' => function(ContainerInterface $container) { + return new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'debug.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Debug, + Monolog\Level::Warning + ); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - $logger = new Monolog\Logger('file_logger'); - $logger->pushHandler($container->get(Monolog\Handler\RotatingFileHandler::class)); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return $container->get('file_logger'); + }, + 'file_logger' => function(ContainerInterface $container) { + return new Monolog\Logger( + 'file', + [ + $container->get('file_log_handler'), + $container->get('debug_log_handler') + ], + $container->get('log_processors') + ); }, ]; From 88f91c4bd52b8b945b7b6f3266fbf59d1e62549f Mon Sep 17 00:00:00 2001 From: Aldarien Date: Mon, 12 Jun 2023 21:14:07 -0400 Subject: [PATCH 3/3] Jobs setup --- api/common/Controller/Attachments.php | 64 +++---- api/common/Controller/Base.php | 5 +- api/common/Controller/Jobs.php | 66 +++++-- api/common/Controller/Messages.php | 56 +++--- .../Exception/Request/Auth/Forbidden.php | 15 ++ .../Exception/Request/Auth/Unauthorized.php | 15 ++ api/common/Middleware/Auth.php | 35 ++-- api/common/Middleware/Messages.php | 24 --- api/common/Service/Attachments.php | 2 +- api/common/Service/Auth.php | 33 ++++ api/common/Service/Jobs.php | 65 ++++--- api/common/Service/Messages.php | 16 +- api/docker-compose.yml | 2 +- api/nginx.conf | 15 +- api/resources/routes/02_attachments.php | 5 +- api/resources/routes/02_messages.php | 6 +- api/resources/routes/03_jobs.php | 15 ++ api/setup/app.php | 7 +- api/setup/middleware/04_db.php | 1 - api/setup/setups/02_services.php | 3 + api/setup/setups/03_factories.php | 4 +- api/setup/setups/97_auth.php | 2 +- api/setup/setups/98_log.php | 3 +- api/src/Model/Attachment.php | 4 +- api/src/Model/Job.php | 24 +-- api/src/Model/State/Job.php | 2 +- api/src/Repository/Job.php | 19 +- api/src/Repository/State/Job.php | 2 +- cli/common/Command/Jobs/Check.php | 35 +--- cli/common/Command/Jobs/Execute.php | 41 +++++ cli/common/Command/Mailboxes/Check.php | 31 +--- cli/common/Command/Messages/Grab.php | 31 +--- .../Exception/Response/EmptyResponse.php | 16 ++ .../Exception/Response/MissingResponse.php | 15 ++ cli/common/Middleware/Logging.php | 38 ---- cli/common/Service/Attachments.php | 114 ++++++++++++ cli/common/Service/Communicator.php | 21 +-- cli/common/Service/Jobs.php | 71 ++++++++ cli/common/Service/Mailboxes.php | 61 +++++++ cli/common/Service/Messages.php | 43 +++++ cli/common/Wrapper/Application.php | 1 - cli/crontab | 10 +- cli/docker-compose.yml | 1 + cli/resources/commands/03_jobs.php | 1 + cli/setup/middleware/98_log.php | 2 - cli/setup/settings/01_env.php | 2 + cli/setup/setups/04_commands.php | 17 +- cli/setup/setups/98_log.php | 3 +- emails-2022-11-30-03-59-49.sql | 94 ---------- ui/common/Controller/Jobs.php | 14 ++ ui/common/Middleware/Logging.php | 2 +- ui/nginx.conf | 5 +- ui/public/index.php | 8 +- ui/resources/routes/03_jobs.php | 4 + ui/resources/views/emails/messages.blade.php | 43 +++-- ui/resources/views/home.blade.php | 6 +- ui/resources/views/jobs/base.blade.php | 14 ++ ui/resources/views/jobs/list.blade.php | 165 ++++++++++++++++++ .../views/layout/body/header/navbar.blade.php | 1 + ui/setup/setups/98_log.php | 40 +++-- 60 files changed, 965 insertions(+), 495 deletions(-) create mode 100644 api/common/Exception/Request/Auth/Forbidden.php create mode 100644 api/common/Exception/Request/Auth/Unauthorized.php delete mode 100644 api/common/Middleware/Messages.php create mode 100644 api/common/Service/Auth.php create mode 100644 api/resources/routes/03_jobs.php create mode 100644 cli/common/Command/Jobs/Execute.php create mode 100644 cli/common/Exception/Response/EmptyResponse.php create mode 100644 cli/common/Exception/Response/MissingResponse.php delete mode 100644 cli/common/Middleware/Logging.php create mode 100644 cli/common/Service/Attachments.php create mode 100644 cli/common/Service/Jobs.php create mode 100644 cli/common/Service/Mailboxes.php create mode 100644 cli/common/Service/Messages.php delete mode 100644 cli/setup/middleware/98_log.php delete mode 100644 emails-2022-11-30-03-59-49.sql create mode 100644 ui/common/Controller/Jobs.php create mode 100644 ui/resources/routes/03_jobs.php create mode 100644 ui/resources/views/jobs/base.blade.php create mode 100644 ui/resources/views/jobs/list.blade.php diff --git a/api/common/Controller/Attachments.php b/api/common/Controller/Attachments.php index 68d3dd6..d95d028 100644 --- a/api/common/Controller/Attachments.php +++ b/api/common/Controller/Attachments.php @@ -1,19 +1,20 @@ toArray(); @@ -24,35 +25,7 @@ class Attachments ]; return $this->withJson($response, $output); } - public function grab(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Jobs $jobsService): ResponseInterface - { - $body = $request->getBody(); - $json = \Safe\json_decode($body->getContents()); - if (!isset($json->messages)) { - throw new MissingArgument('messages', 'array', 'message UIDs'); - } - $output = [ - 'messages' => $json->messages, - 'total' => count($json->messages), - 'saved' => [ - 'attachments' => [], - 'total' => 0 - ] - ]; - foreach ($json->messages as $message_id) { - if (!$jobsService->isPending($message_id)) { - continue; - } - if ($service->grab($message_id)) { - $job = $jobsService->find($message_id); - $jobsService->execute($job->getId()); - $output['saved']['attachments'] []= $job->toArray(); - $output['saved']['total'] ++; - } - } - return $this->withJson($response, $output); - } - public function get(ServerRequestInterface $request, ResponseInterface $response, Service $service, LoggerInterface $logger, int $attachment_id): ResponseInterface + public function get(ServerRequestInterface $request, ResponseInterface $response, Service\Attachments $service, LoggerInterface $logger, int $attachment_id): ResponseInterface { $attachment = $service->getRepository()->fetchById($attachment_id); @@ -61,4 +34,25 @@ class Attachments $response->getBody()->write($service->getFile($attachment_id)); return $response; } -} \ No newline at end of file + public function grab(ServerRequestInterface $request, ResponseInterface $response, Service\Attachments $service, Service\Messages $messagesService): ResponseInterface + { + $body = $request->getBody(); + $json = json_decode($body->getContents()); + if (!isset($json->messages)) { + throw new MissingArgument('messages', 'array', 'Messages UIDs'); + } + $output = [ + 'messages' => $json->messages, + 'attachments' => [], + 'total' => 0 + ]; + foreach ($json->messages as $message_uid) { + $message = $messagesService->getLocalMessage($message_uid); + $attachments = $service->grab($message->getId()); + $output['attachments'] = array_merge($output['attachments'], $attachments); + $output['total'] += count($attachments); + } + + return $this->withJson($response, $output); + } +} diff --git a/api/common/Controller/Base.php b/api/common/Controller/Base.php index 8611788..4b8013f 100644 --- a/api/common/Controller/Base.php +++ b/api/common/Controller/Base.php @@ -12,7 +12,8 @@ class Base public function __invoke(ServerRequestInterface $request, ResponseInterface $response): ResponseInterface { return $this->withJson($response, [ - 'version' => '1.0.0' + 'version' => '1.0.0', + 'app' => 'emails' ]); } -} \ No newline at end of file +} diff --git a/api/common/Controller/Jobs.php b/api/common/Controller/Jobs.php index 437555f..5979fdc 100644 --- a/api/common/Controller/Jobs.php +++ b/api/common/Controller/Jobs.php @@ -1,7 +1,6 @@ getRepository()->fetchAll(); + return $this->withJson($response, compact('jobs')); + } + public function schedule(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface { $body = $request->getBody(); $json = json_decode($body->getContents()); - if (!isset($json->messages)) { - throw new MissingArgument('messages', 'array', 'messages ids'); + if (!isset($json->jobs)) { + throw new MissingArgument('jobs', 'array', 'job commands with arguments'); } $output = [ - 'messages' => $json->messages, - 'total' => count($json->messages), + 'jobs' => $json->jobs, + 'total' => count($json->jobs), 'scheduled' => 0 ]; - foreach ($json->messages as $message_id) { - if ($service->schedule($message_id)) { - $message = $messagesService->getRepository()->fetchById($message_id); - $message->doesHaveScheduledDownloads(); - $messagesService->getRepository()->save($message); + foreach ($json->jobs as $job) { + if ($service->queue($job->command, $job->arguments)) { $output['scheduled'] ++; } } @@ -37,12 +38,47 @@ class Jobs } public function pending(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface { - $pending = array_map(function(Job $job) { - return $job->toArray(); - }, $service->getPending()); + $pending = $service->getPending(); $output = [ 'total' => count($pending), - 'pending' => $pending + 'jobs' => $pending + ]; + return $this->withJson($response, $output); + } + public function pendingCommands(ServerRequestInterface $request, ResponseInterface $response, Service $service): ResponseInterface + { + $body = $response->getBody(); + $json = json_decode($body->getContents()); + if (!isset($json->commands)) { + throw new MissingArgument('commands', 'array', 'job commands'); + } + $output = [ + 'commands' => $json->commands, + 'total' => count($json->commands), + 'pending' => [] + ]; + foreach ($json->commands as $command) { + $pending = $service->getPendingByCommand($command); + if (count($pending) === 0) { + continue; + } + $output['pending'][$command] = $pending; + } + return $this->withJson($response, $output); + } + public function finish(ServerRequestInterface $request, ResponseInterface $response, Service $service, $job_id): ResponseInterface + { + $output = [ + 'job_id' => $job_id, + 'status' => $service->finish($job_id) + ]; + return $this->withJson($response, $output); + } + public function failed(ServerRequestInterface $request, ResponseInterface $response, Service $service, $job_id): ResponseInterface + { + $output = [ + 'job_id' => $job_id, + 'status' => $service->failed($job_id) ]; return $this->withJson($response, $output); } diff --git a/api/common/Controller/Messages.php b/api/common/Controller/Messages.php index db86073..7b0d387 100644 --- a/api/common/Controller/Messages.php +++ b/api/common/Controller/Messages.php @@ -1,11 +1,11 @@ getMailboxes()->get($mailbox_id); $messages = array_map(function(Message $message) { @@ -37,7 +37,7 @@ class Messages ]; return $this->withJson($response, $output); } - public function valid(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Attachments $attachments, int $mailbox_id): ResponseInterface + public function valid(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, Service\Attachments $attachments, int $mailbox_id): ResponseInterface { $mailbox = $service->getMailboxes()->get($mailbox_id); $messages = array_values(array_filter(array_map(function(Message $message) use ($service, $attachments) { @@ -63,32 +63,48 @@ class Messages ]; return $this->withJson($response, $output); } - public function get(ServerRequestInterface $request, ResponseInterface $response, Service $service, int $message_id): ResponseInterface + public function get(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, int $message_id): ResponseInterface { $message = $service->getRepository()->fetchById($message_id); return $this->withJson($response, ['message' => $message->toArray()]); } - public function grab(ServerRequestInterface $request, ResponseInterface $response, Service $service, \ProVM\Common\Service\Attachments $attachmentsService): ResponseInterface + public function grab(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, \ProVM\Common\Service\Attachments $attachmentsService, $mailbox_id): ResponseInterface + { + $output = [ + 'mailbox_id' => $mailbox_id, + 'messages' => [], + 'count' => 0 + ]; + $mailbox = $service->getMailboxes()->get($mailbox_id); + $messages = $service->grab($mailbox->getName()); + foreach ($messages as $message_id) { + $message = $service->getLocalMessage($message_id); + if ($message->hasValidAttachments()) { + $attachmentsService->create($message->getId()); + } + } + $output['messages'] = $messages; + $output['count'] = count($messages); + return $this->withJson($response, $output); + } + public function schedule(ServerRequestInterface $request, ResponseInterface $response, Service\Messages $service, Service\Jobs $jobsService): ResponseInterface { $body = $request->getBody(); - $json = json_decode($body->getContents()); - if (!isset($json->mailboxes)) { - throw new MissingArgument('mailboxes', 'array', 'mailboxes names'); + $json = json_decode($body); + if (!isset($json->messages)) { + throw new MissingArgument('messages', 'array', 'messages IDs'); } $output = [ - 'mailboxes' => $json->mailboxes, - 'messages' => [], - 'message_count' => 0 + 'messages' => $json->messages, + 'scheduled' => 0 ]; - foreach ($json->mailboxes as $mailbox_name) { - $messages = $service->grab($mailbox_name); - foreach ($messages as $message) { - if ($message->hasValidAttachments()) { - $attachmentsService->create($message); - } + foreach ($json->messages as $message_id) { + if ($jobsService->queue('attachments:grab', [$message_id])) { + $message = $service->getRepository()->fetchById($message_id); + $message->doesHaveScheduledDownloads(); + $service->getRepository()->save($message); + $output['scheduled'] ++; } - $output['messages'] = array_merge($output['messages'], $messages); - $output['message_count'] += count($messages); } return $this->withJson($response, $output); } diff --git a/api/common/Exception/Request/Auth/Forbidden.php b/api/common/Exception/Request/Auth/Forbidden.php new file mode 100644 index 0000000..9c24549 --- /dev/null +++ b/api/common/Exception/Request/Auth/Forbidden.php @@ -0,0 +1,15 @@ +setResponseFactory($factory); $this->setLogger($logger); - $this->setAPIKey($api_key); } protected ResponseFactoryInterface $factory; protected LoggerInterface $logger; - protected string $api_key; public function getResponseFactory(): ResponseFactoryInterface { @@ -28,10 +28,6 @@ class Auth { return $this->logger; } - public function getAPIKey(): string - { - return $this->api_key; - } public function setResponseFactory(ResponseFactoryInterface $factory): Auth { @@ -43,30 +39,23 @@ class Auth $this->logger = $logger; return $this; } - public function setAPIKey(string $key): Auth - { - $this->api_key = $key; - return $this; - } public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface { if ($request->getMethod() === 'OPTIONS') { return $handler->handle($request); } - $auths = $request->getHeader('Authorization'); - foreach ($auths as $auth) { - if (str_contains($auth, 'Bearer')) { - $key = str_replace('Bearer ', '', $auth); - if (sha1($this->getAPIKey()) === $key) { - return $handler->handle($request); - } + try { + if ($this->service->validate($request)) { + return $handler->handle($request); } + } catch (Unauthorized $e) { + $response = $this->getResponseFactory()->createResponse($e->getCode()); + $response->getBody()->write(json_encode(['error' => $e->getCode(), 'message' => $e->getMessage()])); } - $this->getLogger()->debug(sha1($this->getAPIKey())); - $response = $this->getResponseFactory()->createResponse(401); - $response->getBody()->write(\Safe\json_encode(['error' => 401, 'message' => 'Incorrect token'])); + $response = $this->getResponseFactory()->createResponse(413); + $response->getBody()->write(\Safe\json_encode(['error' => 413, 'message' => 'Incorrect token'])); return $response ->withHeader('Content-Type', 'application/json'); } -} \ No newline at end of file +} diff --git a/api/common/Middleware/Messages.php b/api/common/Middleware/Messages.php deleted file mode 100644 index 13c13cb..0000000 --- a/api/common/Middleware/Messages.php +++ /dev/null @@ -1,24 +0,0 @@ -service->checkSchedule(); - } catch (BlankResult $e) { - $this->logger->notice($e); - } - return $handler->handle($request); - } -} diff --git a/api/common/Service/Attachments.php b/api/common/Service/Attachments.php index 1487b77..c5b6db9 100644 --- a/api/common/Service/Attachments.php +++ b/api/common/Service/Attachments.php @@ -117,7 +117,7 @@ class Attachments extends Base continue; } $name = $file->getBasename(".{$file->getExtension()}"); - list($subject, $date, $filename) = explode(' - ', $name); + list($date, $subject, $filename) = explode(' - ', $name); try { $message = $this->getMessages()->find($subject, $date)[0]; $filename = "{$filename}.{$file->getExtension()}"; diff --git a/api/common/Service/Auth.php b/api/common/Service/Auth.php new file mode 100644 index 0000000..7442981 --- /dev/null +++ b/api/common/Service/Auth.php @@ -0,0 +1,33 @@ +getHeaderKey($request); + if (sha1($this->api_key) === $key) { + return true; + } + return false; + } + + protected function getHeaderKey(ServerRequestInterface $request): string + { + if (!$request->hasHeader('Authorization')) { + throw new Unauthorized(); + } + $auths = $request->getHeader('Authorization'); + foreach ($auths as $auth) { + if (str_contains($auth, 'Bearer')) { + return str_replace('Bearer ', '', $auth); + } + } + throw new Unauthorized(); + } +} diff --git a/api/common/Service/Jobs.php b/api/common/Service/Jobs.php index 13ef3d8..f9822c4 100644 --- a/api/common/Service/Jobs.php +++ b/api/common/Service/Jobs.php @@ -1,40 +1,49 @@ setRepository($repository); } - protected Job $repository; + protected Repository\Job $repository; - public function getRepository(): Job + public function getRepository(): Repository\Job { return $this->repository; } - public function setRepository(Job $repository): Jobs + public function setRepository(Repository\Job $repository): Jobs { $this->repository = $repository; return $this; } - public function schedule(int $message_id): bool + public function queue(string $command, ?array $arguments = null): bool { $data = [ - 'message_id' => $message_id, - 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s') + 'command' => $command, + 'arguments' => implode(' ', $arguments) ]; try { $job = $this->getRepository()->create($data); $this->getRepository()->save($job); + $data = [ + 'job_id' => $job->getId(), + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Pending + ]; + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); return true; } catch (PDOException $e) { return false; @@ -44,28 +53,42 @@ class Jobs extends Base { return $this->getRepository()->fetchAllPending(); } - public function isPending(int $message_id): bool + public function getPendingByCommand(string $command): array { try { - $this->getRepository()->fetchPendingByMessage($message_id); - return true; + return $this->getRepository()->fetchAllPendingByCommand($command); } catch (BlankResult $e) { - return false; + return []; } } - public function find(int $message_id): \ProVM\Emails\Model\Job - { - return $this->getRepository()->fetchPendingByMessage($message_id); - } - public function execute(int $job_id): bool + public function finish(int $job_id): bool { + $data = [ + 'job_id' => $job_id, + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Executed + ]; try { - $job = $this->getRepository()->fetchById($job_id); - $job->wasExecuted(); - $this->getRepository()->save($job); + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); return true; } catch (PDOException $e) { return false; } } -} \ No newline at end of file + public function failed(int $job_id): bool + { + $data = [ + 'job_id' => $job_id, + 'date_time' => (new DateTimeImmutable())->format('Y-m-d H:i:s'), + 'status' => Model\State\Job::Failure + ]; + try { + $state = $this->stateRepository->create($data); + $this->stateRepository->save($state); + return true; + } catch (PDOException $e) { + return false; + } + } +} diff --git a/api/common/Service/Messages.php b/api/common/Service/Messages.php index a0cd5e0..ac52463 100644 --- a/api/common/Service/Messages.php +++ b/api/common/Service/Messages.php @@ -154,7 +154,6 @@ class Messages extends Base $message->doesHaveValidAttachments(); } } - error_log(json_encode(compact('message')).PHP_EOL,3,'/logs/debug'); $this->getRepository()->save($message); return true; } catch (PDOException $e) { @@ -196,20 +195,7 @@ class Messages extends Base $registered = $this->getMailboxes()->getRegistered(); foreach ($registered as $mailbox) { if (!$this->getMailboxes()->isUpdated($mailbox)) { - $this->logger->info("Updating messages from {$mailbox->getName()}"); - $this->grab($mailbox->getName()); - } - } - } - public function checkSchedule(): void - { - $messages = $this->getRepository()->fetchAll(); - foreach ($messages as $message) { - if ($message->hasAttachments() and $message->hasValidAttachments() and !$message->hasDownloadedAttachments() and !$message->hasScheduledDownloads()) { - if ($this->getJobsService()->schedule($message->getId())) { - $message->doesHaveDownloadedAttachments(); - $this->getRepository()->save($message); - } + $this->getJobsService()->queue('messages:grab', [$mailbox->getId()]); } } } diff --git a/api/docker-compose.yml b/api/docker-compose.yml index 173b1be..0f71511 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -7,7 +7,7 @@ services: - ${API_PATH:-.}:/app/api - ${API_PATH:-.}/nginx.conf:/etc/nginx/conf.d/api.conf ports: - - "${API_PORT:-8080}:8080" + - "${API_PORT:-8080}:81" api: profiles: - api diff --git a/api/nginx.conf b/api/nginx.conf index 6d22e8e..6a50944 100644 --- a/api/nginx.conf +++ b/api/nginx.conf @@ -1,15 +1,15 @@ server { - listen 0.0.0.0:8080; + listen 81; root /app/api/public; index index.php index.html index.htm; - access_log /var/logs/nginx/access.log; - error_log /var/logs/nginx/error.log; + access_log /var/logs/nginx/api.access.log; + error_log /var/logs/nginx/api.error.log; location / { - try_files $uri $uri/ /index.php?$query_string; + try_files $uri /index.php$is_args$args; } - location ~ \.php$ { + location ~ \.php { if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Headers' 'Authorization,Accept,Origin,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; @@ -26,11 +26,12 @@ server { try_files $uri =404; fastcgi_split_path_info ^(.+\.php)(/.+)$; + include fastcgi_params; fastcgi_pass api:9000; fastcgi_index index.php; - include fastcgi_params; fastcgi_param REQUEST_URI $request_uri; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param SCRIPT_NAME $fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; } -} \ No newline at end of file +} diff --git a/api/resources/routes/02_attachments.php b/api/resources/routes/02_attachments.php index 9b45b9d..8c53514 100644 --- a/api/resources/routes/02_attachments.php +++ b/api/resources/routes/02_attachments.php @@ -2,9 +2,8 @@ use ProVM\Common\Controller\Attachments; $app->group('/attachments', function($app) { - $app->put('/grab', [Attachments::class, 'grab']); - $app->get('/pending', [Attachments::class, 'pending']); - $app->post('/decrypt', [Attachments::class, 'decrypt']); + $app->put('/grab[/]', [Attachments::class, 'grab']); + $app->get('/pending[/]', [Attachments::class, 'pending']); $app->get('[/]', Attachments::class); }); $app->group('/attachment/{attachment_id}', function($app) { diff --git a/api/resources/routes/02_messages.php b/api/resources/routes/02_messages.php index ed080d5..4d10259 100644 --- a/api/resources/routes/02_messages.php +++ b/api/resources/routes/02_messages.php @@ -4,9 +4,9 @@ use ProVM\Common\Controller\Jobs; $app->group('/messages', function($app) { $app->put('/grab', [Messages::class, 'grab']); - $app->put('/schedule', [Jobs::class, 'schedule']); - $app->get('/pending', [Jobs::class, 'pending']); + $app->put('/schedule', [Messages::class, 'schedule']); + //$app->get('/pending', [Jobs::class, 'pending']); }); $app->group('/message/{message_id}', function($app) { $app->get('[/]', [Messages::class, 'get']); -}); \ No newline at end of file +}); diff --git a/api/resources/routes/03_jobs.php b/api/resources/routes/03_jobs.php new file mode 100644 index 0000000..1753d96 --- /dev/null +++ b/api/resources/routes/03_jobs.php @@ -0,0 +1,15 @@ +group('/jobs', function($app) { + $app->group('/pending', function($app) { + $app->put('/command[/]', [Jobs::class, 'pendingCommands']); + $app->get('[/]', [Jobs::class, 'pending']); + }); + $app->post('/schedule[/]', [Jobs::class, 'schedule']); + $app->get('[/]', Jobs::class); +}); +$app->group('/job/{job_id}', function($app) { + $app->get('/finish[/]', [Jobs::class, 'finish']); + $app->get('/failed[/]', [Jobs::class, 'failed']); +}); diff --git a/api/setup/app.php b/api/setup/app.php index 7367296..0ae6f29 100644 --- a/api/setup/app.php +++ b/api/setup/app.php @@ -1,7 +1,10 @@ addDefinitions($file->getRealPath()); } } -$app = \DI\Bridge\Slim\Bridge::create($builder->build()); +$app = Bridge::create($builder->build()); $folder = implode(DIRECTORY_SEPARATOR, [ __DIR__, diff --git a/api/setup/middleware/04_db.php b/api/setup/middleware/04_db.php index 9d831e8..b71467b 100644 --- a/api/setup/middleware/04_db.php +++ b/api/setup/middleware/04_db.php @@ -1,5 +1,4 @@ add($app->getContainer()->get(ProVM\Common\Middleware\Attachments::class)); -$app->add($app->getContainer()->get(ProVM\Common\Middleware\Messages::class)); $app->add($app->getContainer()->get(ProVM\Common\Middleware\Mailboxes::class)); $app->add($app->getContainer()->get(ProVM\Common\Middleware\Install::class)); diff --git a/api/setup/setups/02_services.php b/api/setup/setups/02_services.php index 2a09318..4348151 100644 --- a/api/setup/setups/02_services.php +++ b/api/setup/setups/02_services.php @@ -33,5 +33,8 @@ return [ $container->get(ProVM\Common\Factory\Model::class), $container->get('model_list') ); + }, + ProVM\Common\Service\Auth::class => function(ContainerInterface $container) { + return new ProVM\Common\Service\Auth($container->get('api_key')); } ]; diff --git a/api/setup/setups/03_factories.php b/api/setup/setups/03_factories.php index 281eafa..49b605e 100644 --- a/api/setup/setups/03_factories.php +++ b/api/setup/setups/03_factories.php @@ -8,9 +8,11 @@ return [ 'Mailbox' => ProVM\Emails\Repository\Mailbox::class, 'Message' => ProVM\Emails\Repository\Message::class, 'Attachment' => ProVM\Emails\Repository\Attachment::class, + 'Job' => ProVM\Emails\Repository\Job::class, "State\\Mailbox" => ProVM\Emails\Repository\State\Mailbox::class, "State\\Message" => ProVM\Emails\Repository\State\Message::class, - "State\\Attachment" => ProVM\Emails\Repository\State\Attachment::class + "State\\Attachment" => ProVM\Emails\Repository\State\Attachment::class, + "State\\Job" => ProVM\Emails\Repository\State\Job::class, ]); } ]; diff --git a/api/setup/setups/97_auth.php b/api/setup/setups/97_auth.php index 03fa0f3..9476f57 100644 --- a/api/setup/setups/97_auth.php +++ b/api/setup/setups/97_auth.php @@ -6,7 +6,7 @@ return [ return new ProVM\Common\Middleware\Auth( $container->get(Nyholm\Psr7\Factory\Psr17Factory::class), $container->get(Psr\Log\LoggerInterface::class), - $container->get('api_key') + $container->get(ProVM\Common\Service\Auth::class) ); } ]; diff --git a/api/setup/setups/98_log.php b/api/setup/setups/98_log.php index 0ebc4cb..39788a4 100644 --- a/api/setup/setups/98_log.php +++ b/api/setup/setups/98_log.php @@ -11,8 +11,7 @@ return [ ]; }, 'request_log_handler' => function(ContainerInterface $container) { - return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) - ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))); }, 'request_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/api/src/Model/Attachment.php b/api/src/Model/Attachment.php index 912a2e8..9cc8683 100644 --- a/api/src/Model/Attachment.php +++ b/api/src/Model/Attachment.php @@ -95,9 +95,9 @@ class Attachment implements Model public function getFullFilename(): string { return implode(' - ', [ - $this->getMessage()->getSubject(), $this->getMessage()->getDateTime()->format('Y-m-d His'), - $this->getFilename() + $this->getMessage()->getSubject(), + $this->getFilename(), ]); } diff --git a/api/src/Model/Job.php b/api/src/Model/Job.php index 5aa26ee..7b69d93 100644 --- a/api/src/Model/Job.php +++ b/api/src/Model/Job.php @@ -25,24 +25,20 @@ class Job implements Model { return $this->arguments ?? ''; } - public function isExecuted(): bool - { - return $this->lastState()->getStatus() === State\Job::Executed; - } public function setId(int $id): Job { $this->id = $id; return $this; } - public function setCommand(string $message): Job + public function setCommand(string $command): Job { - $this->message = $message; + $this->command = $command; return $this; } - public function setArguments(string $dateTime): Job + public function setArguments(string $arguments): Job { - $this->dateTime = $dateTime; + $this->arguments = $arguments; return $this; } @@ -82,10 +78,14 @@ class Job implements Model return $this; } + public function isExecuted(): bool + { + return $this->lastState()->getStatus() === State\Job::Executed; + } public function lastState(): State\Job { if (count($this->getStates()) === 0) { - throw new Stateless($this); + throw new Stateless($this->getId()); } return $this->getStates()[array_key_last($this->getStates())]; } @@ -94,9 +94,9 @@ class Job implements Model { return [ 'id' => $this->getId(), - 'message' => $this->getMessage()->toArray(), - 'date_time' => $this->getDateTime()->format('Y-m-d H:i:s'), - 'executed' => $this->isExecuted() + 'command' => $this->getCommand(), + 'arguments' => $this->getArguments(), + 'states' => $this->getStates() ]; } public function jsonSerialize(): mixed diff --git a/api/src/Model/State/Job.php b/api/src/Model/State/Job.php index 2826908..1d1e9af 100644 --- a/api/src/Model/State/Job.php +++ b/api/src/Model/State/Job.php @@ -57,7 +57,7 @@ class Job implements Model { return [ 'id' => $this->getId(), - 'job' => $this->getJob(), + 'job_id' => $this->getJob()->getId(), 'date' => $this->getDateTime(), 'status' => $this->getStatus() ]; diff --git a/api/src/Repository/Job.php b/api/src/Repository/Job.php index e8f6be9..17c39df 100644 --- a/api/src/Repository/Job.php +++ b/api/src/Repository/Job.php @@ -15,7 +15,7 @@ class Job extends Repository { parent::__construct($connection, $logger); $this->setFactory($factory) - ->setTable('attachments_jobs'); + ->setTable('jobs'); } protected Factory\Model $factory; @@ -98,19 +98,22 @@ CREATE TABLE {$this->getTable()} ( public function fetchAllPending(): array { $query = "SELECT a.* -FROM {$this->getTable()} a - JOIN `jobs_states` b ON b.job_id = a.id +FROM `{$this->getTable()}` a + JOIN (SELECT s1.* FROM `jobs_states` s1 JOIN (SELECT MAX(id) AS id, job_id FROM `jobs_states` GROUP BY job_id) s2 ON s2.id = s1.id) b ON b.`job_id` = a.`id` WHERE b.`status` = ?"; - return $this->fetchMany($query); + return $this->fetchMany($query, [Emails\Model\State\Job::Pending]); } - public function fetchByCommandAndArguments(string $command, string $arguments): \ProVM\Emails\Model\Job + public function fetchByCommandAndArguments(string $command, string $arguments): Emails\Model\Job { $query = "SELECT * FROM {$this->getTable()} WHERE `command` = ? AND `arguments` = ?"; return $this->fetchOne($query, [$command, $arguments]); } - public function fetchPendingByMessage(int $message_id): \ProVM\Emails\Model\Job + public function fetchAllPendingByCommand(string $command): array { - $query = "SELECT * FROM {$this->getTable()} WHERE `message_id` = ? AND `executed` = 0"; - return $this->fetchOne($query, [$message_id]); + $query = "SELECT a.* +FROM `{$this->getTable()}` a + JOIN (SELECT s1.* FROM `jobs_states` s1 JOIN (SELECT MAX(id) AS id, job_id FROM `jobs_states` GROUP BY job_id) s2 ON s2.id = s1.id) b ON b.`job_id` = a.`id` +WHERE a.`command` = ? AND b.`status` = ?"; + return $this->fetchMany($query, [$command, Emails\Model\State\Job::Pending]); } } diff --git a/api/src/Repository/State/Job.php b/api/src/Repository/State/Job.php index e7f9d57..11a6699 100644 --- a/api/src/Repository/State/Job.php +++ b/api/src/Repository/State/Job.php @@ -57,7 +57,7 @@ CREATE TABLE {$this->getTable()} ( $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ?"; return $this->fetchMany($query, [$job_id]); } - public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\Job + public function fetchByJobAndStatus(int $job_id, int $status): Emails\Model\State\Job { $query = "SELECT * FROM `{$this->getTable()}` WHERE `job_id` = ? AND `status` = ?"; return $this->fetchOne($query, [$job_id, $status]); diff --git a/cli/common/Command/Jobs/Check.php b/cli/common/Command/Jobs/Check.php index d3c9b23..5a5ecc6 100644 --- a/cli/common/Command/Jobs/Check.php +++ b/cli/common/Command/Jobs/Check.php @@ -1,14 +1,12 @@ logger->notice('Grabbing pending jobs.'); - $response = $this->communicator->get('/jobs/pending'); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return []; - } - return json_decode($body)->jobs; - } - protected function runJob($job): bool - { - $base_command = '/app/bin/emails'; - $cmd = [$base_command, $job->command]; - if ($job->arguments !== '') { - $cmd []= $job->arguments; - } - $cmd = implode(' ', $cmd); - $this->logger->notice("Running '{$cmd}'"); - $response = shell_exec($cmd); - $this->logger->info("Result: {$response}"); - return $response !== false; - } - public function execute(InputInterface $input, OutputInterface $output) { $section1 = $output->section(); @@ -53,17 +27,16 @@ class Check extends Command $io1 = new SymfonyStyle($input, $section1); $io2 = new SymfonyStyle($input, $section2); $io1->title('Checking Pending Jobs'); - $pending_jobs = $this->getPendingJobs(); + $pending_jobs = $this->service->getPending(); $notice = 'Found ' . count($pending_jobs) . ' jobs'; $io1->text($notice); - $this->logger->info($notice); if (count($pending_jobs) > 0) { $io1->section('Running Jobs'); $io1->progressStart(count($pending_jobs)); foreach ($pending_jobs as $job) { $section2->clear(); $io2->text("Running {$job->command}"); - if ($this->runJob($job)) { + if ($this->service->run($job)) { $io2->success('Success'); } else { $io2->error('Failure'); diff --git a/cli/common/Command/Jobs/Execute.php b/cli/common/Command/Jobs/Execute.php new file mode 100644 index 0000000..0c41a4e --- /dev/null +++ b/cli/common/Command/Jobs/Execute.php @@ -0,0 +1,41 @@ +addArgument('job_id', InputArgument::REQUIRED, 'Job ID to be executed'); + } + + public function execute(InputInterface $input, OutputInterface $output) + { + $io = new SymfonyStyle($input, $output); + $job_id = $input->getArgument('job_id'); + $job = $this->service->get($job_id); + if ($this->service->run($job)) { + $io->success('Success'); + } else { + $io->error('Failed'); + } + return Command::SUCCESS; + } +} diff --git a/cli/common/Command/Mailboxes/Check.php b/cli/common/Command/Mailboxes/Check.php index c0fa6eb..eb2f94c 100644 --- a/cli/common/Command/Mailboxes/Check.php +++ b/cli/common/Command/Mailboxes/Check.php @@ -5,9 +5,8 @@ use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -use ProVM\Service\Communicator; use Symfony\Component\Console\Style\SymfonyStyle; -use function Safe\json_decode; +use ProVM\Service\Mailboxes; #[AsCommand( name: 'mailboxes:check', @@ -16,33 +15,11 @@ use function Safe\json_decode; )] class Check extends Command { - public function __construct(protected Communicator $communicator, protected int $min_check_days, string $name = null) + public function __construct(protected Mailboxes $service, string $name = null) { parent::__construct($name); } - protected function getMailboxes(): array - { - $response = $this->communicator->get('/mailboxes/registered'); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return []; - } - return json_decode($body)->mailboxes; - } - protected function checkMailbox($mailbox): bool - { - if ((new \DateTimeImmutable())->diff(new \DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { - return true; - } - $response = $this->communicator->get("/mailbox/{$mailbox->id}/check"); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return true; - } - return json_decode($body)->status; - } - public function execute(InputInterface $input, OutputInterface $output): int { $section1 = $output->section(); @@ -50,7 +27,7 @@ class Check extends Command $io1 = new SymfonyStyle($input, $section1); $io2 = new SymfonyStyle($input, $section2); $io1->title('Checking for New Messages'); - $mailboxes = $this->getMailboxes(); + $mailboxes = $this->service->getAll(); $notice = 'Found ' . count($mailboxes) . ' mailboxes'; $io1->text($notice); if (count($mailboxes) > 0) { @@ -59,7 +36,7 @@ class Check extends Command foreach ($mailboxes as $mailbox) { $section2->clear(); $io2->text("Checking {$mailbox->name}"); - if ($this->checkMailbox($mailbox)) { + if ($this->service->check($mailbox)) { $io2->success("Found new emails in {$mailbox->name}"); } else { $io2->info("No new emails in {$mailbox->name}"); diff --git a/cli/common/Command/Messages/Grab.php b/cli/common/Command/Messages/Grab.php index 89a85f9..1be0fd0 100644 --- a/cli/common/Command/Messages/Grab.php +++ b/cli/common/Command/Messages/Grab.php @@ -7,19 +7,17 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; -use ProVM\Service\Communicator; -use function Safe\json_decode; +use ProVM\Service\Mailboxes; #[AsCommand( name: 'messages:grab', - description: 'Run grab messages job for registered mailboxes', + description: 'Run grab messages job for mailbox', hidden: false )] class Grab extends Command { - public function __construct(Communicator $communicator, string $name = null) + public function __construct(protected Mailboxes $service, string $name = null) { - $this->setCommunicator($communicator); parent::__construct($name); } protected function configure() @@ -27,27 +25,6 @@ class Grab extends Command $this->addArgument('mailbox_id', InputArgument::REQUIRED, 'Mailbox ID to grab emails'); } - protected Communicator $communicator; - public function getCommunicator(): Communicator - { - return $this->communicator; - } - public function setCommunicator(Communicator $communicator): Grab - { - $this->communicator = $communicator; - return $this; - } - - protected function grabMessages(int $mailbox_id): int - { - $response = $this->getCommunicator()->get("/mailbox/{$mailbox_id}/grab"); - $body = $response->getBody()->getContents(); - if (trim($body) === '') { - return 0; - } - return json_decode($body)->messages->count; - } - public function execute(InputInterface $input, OutputInterface $output) { $io = new SymfonyStyle($input, $output); @@ -55,7 +32,7 @@ class Grab extends Command $mailbox_id = $input->getArgument('mailbox_id'); $io->title("Grabbing Messages for Mailbox ID {$mailbox_id}"); $io->section('Grabbing Messages'); - $count = $this->grabMessages($mailbox_id); + $count = $this->service->grabMessages($mailbox_id); $io->info("Found {$count} messages"); $io->success('Done.'); diff --git a/cli/common/Exception/Response/EmptyResponse.php b/cli/common/Exception/Response/EmptyResponse.php new file mode 100644 index 0000000..5fff71f --- /dev/null +++ b/cli/common/Exception/Response/EmptyResponse.php @@ -0,0 +1,16 @@ +setLogger($logger); - } - - protected LoggerInterface $logger; - - public function getLogger(): LoggerInterface - { - return $this->logger; - } - - public function setLogger(LoggerInterface $logger): Logging - { - $this->logger = $logger; - return $this; - } - - public function __invoke(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface - { - $response = $handler->handle($request); - $output = [ - 'uri' => var_export($request->getUri(), true), - 'body' => $request->getBody()->getContents() - ]; - $this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); - return $response; - } -} diff --git a/cli/common/Service/Attachments.php b/cli/common/Service/Attachments.php new file mode 100644 index 0000000..681b291 --- /dev/null +++ b/cli/common/Service/Attachments.php @@ -0,0 +1,114 @@ +logger->info('Finding all downloaded attachment files'); + $folder = '/attachments'; + $files = new \FilesystemIterator($folder); + foreach ($files as $file) { + if ($file->isDir()) { + continue; + } + yield $file->getRealPath(); + } + } + public function getAll(): array + { + if (!isset($this->attachments)) { + $this->logger->info('Grabbing all attachments'); + $response = $this->communicator->get('/attachments'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + $this->attachments = []; + return $this->attachments; + } + $this->attachments = json_decode($body)->attachments; + } + return $this->attachments; + } + public function get(int $attachment_id): object + { + $this->logger->info("Getting attachment {$attachment_id}"); + $uri = "/attachment/{$attachment_id}"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->attachment)) { + throw new MissingResponse('attachment'); + } + return $json->attachment; + } + public function find(string $filename): int + { + $this->logger->info("Finding attachment {$filename}"); + foreach ($this->getAll() as $attachment) { + if ($attachment->fullfilename === $filename) { + return $attachment->id; + } + } + throw new \Exception("{$filename} is not in the database"); + } + public function isEncrypted(string $filename): bool + { + if (!file_exists($filename)) { + throw new \InvalidArgumentException("File not found {$filename}"); + } + $escaped_filename = escapeshellarg($filename); + $cmd = "{$this->base_command} --is-encrypted {$escaped_filename}"; + exec($cmd, $output, $retcode); + return $retcode == 0; + } + public function scheduleDecrypt(int $attachment_id): bool + { + $this->logger->info("Scheduling decryption of attachment {$attachment_id}"); + $uri = "/attachment/{$attachment_id}/decrypt"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->status)) { + throw new MissingResponse('status'); + } + return $json->status; + } + public function decrypt(string $basename): bool + { + $this->logger->info("Decrypting {$basename}"); + $in_filename = implode('/', ['attachments', $basename]); + $out_filename = implode('/', ['attachments', 'decrypted', $basename]); + if (file_exists($out_filename)) { + throw new \Exception("{$basename} already decrypted"); + } + foreach ($this->passwords as $password) { + $cmd = $this->base_command . ' -password=' . escapeshellarg($password) . ' -decrypt ' . escapeshellarg($in_filename) . ' ' . escapeshellarg($out_filename); + exec($cmd, $output, $retcode); + $success = $retcode == 0; + if ($success) { + return true; + } + if (file_exists($out_filename)) { + unlink($out_filename); + } + unset($output); + } + return false; + } + +} diff --git a/cli/common/Service/Communicator.php b/cli/common/Service/Communicator.php index b22589c..c655079 100644 --- a/cli/common/Service/Communicator.php +++ b/cli/common/Service/Communicator.php @@ -4,29 +4,12 @@ namespace ProVM\Service; use HttpResponseException; use Psr\Http\Client\ClientInterface; use Psr\Http\Message\ResponseInterface; -use Psr\Log\LoggerInterface; use Safe\Exceptions\JsonException; use function Safe\json_encode; class Communicator { - public function __construct(ClientInterface $client) - { - $this->setClient($client); - } - - protected ClientInterface $client; - - public function getClient(): ClientInterface - { - return $this->client; - } - - public function setClient(ClientInterface $client): Communicator - { - $this->client = $client; - return $this; - } + public function __construct(protected ClientInterface $client) {} /** * @throws HttpResponseException @@ -52,7 +35,7 @@ class Communicator ]; $options['body'] = json_encode($body); } - return $this->handleResponse($this->getClient()->request($method, $uri, $options)); + return $this->handleResponse($this->client->request($method, $uri, $options)); } /** diff --git a/cli/common/Service/Jobs.php b/cli/common/Service/Jobs.php new file mode 100644 index 0000000..d3543ef --- /dev/null +++ b/cli/common/Service/Jobs.php @@ -0,0 +1,71 @@ +logger->info('Getting pending jobs'); + $response = $this->communicator->get('/jobs/pending'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + $json = json_decode($body); + if (!isset($json->jobs)) { + return []; + } + return $json->jobs; + } + public function get(int $job_id): object + { + $this->logger->info("Getting Job {$job_id}"); + $uri = "/job/{$job_id}"; + return $this->send($uri, 'job'); + } + public function run(object $job): bool + { + $this->logger->debug("Running Job {$job->id}"); + $base_command = '/app/bin/emails'; + $cmd = [$base_command, $job->command]; + if ($job->arguments !== '') { + $cmd []= $job->arguments; + } + $cmd = implode(' ', $cmd); + $response = shell_exec($cmd); + if ($response !== false) { + return $this->finished($job->id); + } + return $this->failure($job->id); + } + + protected function finished(int $job_id): bool + { + $uri = "/job/{$job_id}/finish"; + return $this->send($uri, 'status'); + } + protected function failure(int $job_id): bool + { + $uri = "/job/{$job_id}/failed"; + return $this->send($uri, 'status'); + } + protected function send(string $uri, string $param): mixed + { + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->{$param})) { + throw new MissingResponse($param); + } + return $json->{$param}; + } +} diff --git a/cli/common/Service/Mailboxes.php b/cli/common/Service/Mailboxes.php new file mode 100644 index 0000000..4d8eabf --- /dev/null +++ b/cli/common/Service/Mailboxes.php @@ -0,0 +1,61 @@ +logger->info('Getting all registered mailboxes'); + $response = $this->communicator->get('/mailboxes/registered'); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return []; + } + $json = json_decode($body); + if (!isset($json->mailboxes)) { + return []; + } + return $json->mailboxes; + } + public function check(object $mailbox): bool + { + $this->logger->info("Checking mailbox {$mailbox->id}"); + if ((new DateTimeImmutable())->diff(new DateTimeImmutable($mailbox->last_checked->date->date))->days < $this->min_check_days) { + return true; + } + $uri = "/mailbox/{$mailbox->id}/check"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->status)) { + throw new MissingResponse('status'); + } + return $json->status; + } + public function grabMessages(int $mailbox_id): int + { + $this->logger->info("Grabbing messages for {$mailbox_id}"); + $uri = "/mailbox/{$mailbox_id}/messages/grab"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + $json = json_decode($body); + if (!isset($json->count)) { + return 0; + } + return $json->count; + } +} diff --git a/cli/common/Service/Messages.php b/cli/common/Service/Messages.php new file mode 100644 index 0000000..3854dbd --- /dev/null +++ b/cli/common/Service/Messages.php @@ -0,0 +1,43 @@ +logger->info("Getting message {$message_id}"); + $uri = "/message/{$message_id}"; + $response = $this->communicator->get($uri); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + throw new EmptyResponse($uri); + } + $json = json_decode($body); + if (!isset($json->message)) { + throw new MissingResponse('message'); + } + return $json->message; + } + public function grabAttachments(string $message_uid): int + { + $this->logger->info("Grabbing attachments for message UID {$message_uid}"); + $uri = '/attachments/grab'; + $response = $this->communicator->put($uri, ['messages' => [$message_uid]]); + $body = $response->getBody()->getContents(); + if (trim($body) === '') { + return 0; + } + $json = json_decode($body); + if (!isset($json->total)) { + return 0; + } + return $json->total; + } +} diff --git a/cli/common/Wrapper/Application.php b/cli/common/Wrapper/Application.php index c27dbf4..7ffac08 100644 --- a/cli/common/Wrapper/Application.php +++ b/cli/common/Wrapper/Application.php @@ -3,7 +3,6 @@ namespace ProVM\Wrapper; use Psr\Container\ContainerInterface; use Symfony\Component\Console\Application as Base; - class Application extends Base { public function __construct(ContainerInterface $container, string $name = 'UNKNOWN', string $version = 'UNKNOWN') diff --git a/cli/crontab b/cli/crontab index fc8ab81..b567b58 100644 --- a/cli/crontab +++ b/cli/crontab @@ -1,10 +1,10 @@ # minutes hour day_of_month month day_of_week command -#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log -#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log +#0 2 * * 2-6 /app/bin/emails messages:grab >> /logs/messages.log +#0 3 * * 2-6 /app/bin/emails attachments:grab >> /logs/attachments.log # Pending jobs every minute -1 * * * * /app/bin/emails jobs:pending >> /logs/jobs.log +* * * * * /app/bin/emails jobs:check >> /logs/jobs.log # Check mailboxes for new emails every weekday -0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log +0 0 * * 2-6 /app/bin/emails mailboxes:check >> /logs/mailboxes.log # Check attachments every weekday -0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log +0 1 * * 2-6 /app/bin/emails attachments:check >> /logs/attachments.log diff --git a/cli/docker-compose.yml b/cli/docker-compose.yml index ea1ec32..e86b638 100644 --- a/cli/docker-compose.yml +++ b/cli/docker-compose.yml @@ -12,5 +12,6 @@ services: - .key.env volumes: - ${CLI_PATH:-.}/:/app + - ${CLI_PATH}/crontab:/var/spool/cron/crontabs/root - ${LOGS_PATH}/cli:/logs - ${ATT_PATH}:/attachments diff --git a/cli/resources/commands/03_jobs.php b/cli/resources/commands/03_jobs.php index bd6eed1..ea36967 100644 --- a/cli/resources/commands/03_jobs.php +++ b/cli/resources/commands/03_jobs.php @@ -1,2 +1,3 @@ add($app->getContainer()->get(ProVM\Command\Jobs\Check::class)); +$app->add($app->getContainer()->get(ProVM\Command\Jobs\Execute::class)); diff --git a/cli/setup/middleware/98_log.php b/cli/setup/middleware/98_log.php deleted file mode 100644 index 5628448..0000000 --- a/cli/setup/middleware/98_log.php +++ /dev/null @@ -1,2 +0,0 @@ -add($app->getContainer()->get(ProVM\Common\Middleware\Logging::class)); diff --git a/cli/setup/settings/01_env.php b/cli/setup/settings/01_env.php index 107718a..76b4051 100644 --- a/cli/setup/settings/01_env.php +++ b/cli/setup/settings/01_env.php @@ -2,7 +2,9 @@ return [ 'api_uri' => $_ENV['API_URI'], 'api_key' => sha1($_ENV['API_KEY']), + 'base_command' => 'qpdf', 'passwords' => function() { return explode($_ENV['PASSWORDS_SEPARATOR'] ?? ',', $_ENV['PASSWORDS'] ?? ''); }, + 'min_check_days' => 1 ]; diff --git a/cli/setup/setups/04_commands.php b/cli/setup/setups/04_commands.php index 50ae9d1..23fd362 100644 --- a/cli/setup/setups/04_commands.php +++ b/cli/setup/setups/04_commands.php @@ -3,18 +3,19 @@ use Psr\Container\ContainerInterface; return [ - ProVM\Command\Attachments\DecryptPdf::class => function(ContainerInterface $container) { - return new ProVM\Command\Attachments\DecryptPdf( + ProVM\Service\Mailboxes::class => function(ContainerInterface $container) { + return new ProVM\Service\Mailboxes( $container->get(ProVM\Service\Communicator::class), $container->get(Psr\Log\LoggerInterface::class), - 'qpdf', - $container->get('passwords') + $container->get('min_check_days') ); }, - ProVM\Command\Mailboxes\Check::class => function(ContainerInterface $container) { - return new ProVM\Command\Mailboxes\Check( + ProVM\Service\Attachments::class => function(ContainerInterface $container) { + return new ProVM\Service\Attachments( $container->get(ProVM\Service\Communicator::class), - 1 + $container->get(Psr\Log\LoggerInterface::class), + $container->get('passwords'), + $container->get('base_command') ); - } + }, ]; diff --git a/cli/setup/setups/98_log.php b/cli/setup/setups/98_log.php index 7f6e30b..ef4a697 100644 --- a/cli/setup/setups/98_log.php +++ b/cli/setup/setups/98_log.php @@ -10,8 +10,7 @@ return [ ]; }, 'request_log_handler' => function(ContainerInterface $container) { - return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))) - ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)); + return (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('logs_folder'), 'requests.log']))); }, 'request_logger' => function(ContainerInterface $container) { return new Monolog\Logger( diff --git a/emails-2022-11-30-03-59-49.sql b/emails-2022-11-30-03-59-49.sql deleted file mode 100644 index 3fd22d9..0000000 --- a/emails-2022-11-30-03-59-49.sql +++ /dev/null @@ -1,94 +0,0 @@ --- Adminer 4.8.1 MySQL 5.5.5-10.9.3-MariaDB-1:10.9.3+maria~ubu2204 dump - -SET NAMES utf8; -SET time_zone = '+00:00'; -SET foreign_key_checks = 0; -SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; - -SET NAMES utf8mb4; - -DROP TABLE IF EXISTS `attachments`; -CREATE TABLE `attachments` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `filename` varchar(255) NOT NULL, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `attachments_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `attachments_jobs`; -CREATE TABLE `attachments_jobs` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `date_time` datetime NOT NULL, - `executed` int(1) DEFAULT 0, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `attachments_jobs_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `attachments_states`; -CREATE TABLE `attachments_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `attachment_id` int(10) unsigned NOT NULL, - `name` varchar(100) NOT NULL, - `value` int(1) unsigned NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `attachment_id` (`attachment_id`), - CONSTRAINT `attachments_states_ibfk_1` FOREIGN KEY (`attachment_id`) REFERENCES `attachments` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `mailboxes`; -CREATE TABLE `mailboxes` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(100) NOT NULL, - `validity` int(11) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `mailboxes_states`; -CREATE TABLE `mailboxes_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `mailbox_id` int(10) unsigned NOT NULL, - `date_time` datetime NOT NULL, - `count` int(10) unsigned NOT NULL, - `uids` text NOT NULL, - PRIMARY KEY (`id`), - KEY `mailbox_id` (`mailbox_id`), - CONSTRAINT `mailboxes_states_ibfk_2` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `messages`; -CREATE TABLE `messages` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `mailbox_id` int(10) unsigned NOT NULL, - `position` int(10) unsigned NOT NULL, - `uid` varchar(255) NOT NULL, - `subject` varchar(255) NOT NULL, - `from` varchar(100) NOT NULL, - `date_time` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `mailbox_id` (`mailbox_id`), - CONSTRAINT `messages_ibfk_1` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -DROP TABLE IF EXISTS `messages_states`; -CREATE TABLE `messages_states` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `message_id` int(10) unsigned NOT NULL, - `name` varchar(100) NOT NULL, - `value` int(1) unsigned NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `message_id` (`message_id`), - CONSTRAINT `messages_states_ibfk_2` FOREIGN KEY (`message_id`) REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - --- 2022-11-30 03:59:49 diff --git a/ui/common/Controller/Jobs.php b/ui/common/Controller/Jobs.php new file mode 100644 index 0000000..091539f --- /dev/null +++ b/ui/common/Controller/Jobs.php @@ -0,0 +1,14 @@ +render($response, 'jobs.list'); + } +} diff --git a/ui/common/Middleware/Logging.php b/ui/common/Middleware/Logging.php index c60465d..a1e97a0 100644 --- a/ui/common/Middleware/Logging.php +++ b/ui/common/Middleware/Logging.php @@ -29,7 +29,7 @@ class Logging { $response = $handler->handle($request); $output = [ - 'uri' => var_export($request->getUri(), true), + 'uri' => print_r($request->getUri(), true), 'body' => $request->getParsedBody() ]; $this->getLogger()->info(\Safe\json_encode($output, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); diff --git a/ui/nginx.conf b/ui/nginx.conf index 893e62a..43895cc 100644 --- a/ui/nginx.conf +++ b/ui/nginx.conf @@ -3,6 +3,9 @@ server { root /app/ui/public; index index.php index.html index.htm; + access_log /var/logs/nginx/ui.access.log; + error_log /var/logs/nginx/ui.error.log; + location / { try_files $uri $uri/ /index.php?$query_string; } @@ -16,4 +19,4 @@ server { fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; } -} \ No newline at end of file +} diff --git a/ui/public/index.php b/ui/public/index.php index 9e2bc0e..3408a32 100644 --- a/ui/public/index.php +++ b/ui/public/index.php @@ -8,11 +8,5 @@ Monolog\ErrorHandler::register($app->getContainer()->get(Psr\Log\LoggerInterface try { $app->run(); } catch (Error | Exception $e) { - $logger = $app->getContainer()->get(Psr\Log\LoggerInterface::class); - if (isset($_REQUEST)) { - $logger->debug(Safe\json_encode(compact('_REQUEST'))); - } - $logger->debug(Safe\json_encode(compact('_SERVER'))); - $logger->error($e); - throw $e; + $app->getContainer()->get(Psr\Log\LoggerInterface::class)->error($e); } diff --git a/ui/resources/routes/03_jobs.php b/ui/resources/routes/03_jobs.php new file mode 100644 index 0000000..f9d8f0a --- /dev/null +++ b/ui/resources/routes/03_jobs.php @@ -0,0 +1,4 @@ +get('/jobs', Jobs::class); diff --git a/ui/resources/views/emails/messages.blade.php b/ui/resources/views/emails/messages.blade.php index 4af8bb1..089eaee 100644 --- a/ui/resources/views/emails/messages.blade.php +++ b/ui/resources/views/emails/messages.blade.php @@ -5,7 +5,12 @@
@endsection +@push('page_styles') + +@endpush + @push('page_scripts') + + +@endpush diff --git a/ui/resources/views/layout/body/header/navbar.blade.php b/ui/resources/views/layout/body/header/navbar.blade.php index 92fa22b..a0a622f 100644 --- a/ui/resources/views/layout/body/header/navbar.blade.php +++ b/ui/resources/views/layout/body/header/navbar.blade.php @@ -1,5 +1,6 @@
diff --git a/ui/setup/setups/98_log.php b/ui/setup/setups/98_log.php index ef8b9d6..f12fc26 100644 --- a/ui/setup/setups/98_log.php +++ b/ui/setup/setups/98_log.php @@ -2,8 +2,13 @@ use Psr\Container\ContainerInterface; return [ - Monolog\Handler\DeduplicationHandler::class => function(ContainerInterface $container) { - return new Monolog\Handler\DeduplicationHandler($container->get(Monolog\Handler\RotatingFileHandler::class)); + 'log_processors' => function(ContainerInterface $container) { + return [ + $container->get(Monolog\Processor\PsrLogMessageProcessor::class), + $container->get(Monolog\Processor\WebProcessor::class), + $container->get(Monolog\Processor\IntrospectionProcessor::class), + $container->get(Monolog\Processor\MemoryPeakUsageProcessor::class) + ]; }, Monolog\Handler\RotatingFileHandler::class => function(ContainerInterface $container) { $handler = new Monolog\Handler\RotatingFileHandler($container->get('log_file')); @@ -11,22 +16,23 @@ return [ return $handler; }, 'request_logger' => function(ContainerInterface $container) { - $logger = new Monolog\Logger('request_logger'); - $handler = new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'requests.log'])); - $handler->setFormatter($container->get(Monolog\Formatter\SyslogFormatter::class)); - $dedupHandler = new Monolog\Handler\DeduplicationHandler($handler, null, Monolog\Level::Info); - $logger->pushHandler($dedupHandler); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger('request_logger', [ + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'requests.log']))), + ], $container->get('log_processors')); }, Psr\Log\LoggerInterface::class => function(ContainerInterface $container) { - $logger = new Monolog\Logger('file_logger'); - $logger->pushHandler($container->get(Monolog\Handler\DeduplicationHandler::class)); - $logger->pushProcessor($container->get(Monolog\Processor\PsrLogMessageProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\IntrospectionProcessor::class)); - $logger->pushProcessor($container->get(Monolog\Processor\MemoryUsageProcessor::class)); - return $logger; + return new Monolog\Logger('file', [ + new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler($container->get('log_file'))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Error + ), + new Monolog\Handler\FilterHandler( + (new Monolog\Handler\RotatingFileHandler(implode(DIRECTORY_SEPARATOR, [$container->get('folders')->logs, 'debug.log']))) + ->setFormatter(new Monolog\Formatter\LineFormatter(null, null, true)), + Monolog\Level::Debug, + Monolog\Level::Warning + ) + ], $container->get('log_processors')); }, ];