From cbe14ea5aed493eddb80ba66c36d139a9fdf528b Mon Sep 17 00:00:00 2001 From: Philippe Delodder <7259755+phdelodder@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:48:07 +0200 Subject: [PATCH] Drop a header block the server repeats at the top of the body (#177) Closes #173. Co-authored-by: Philippe Delodder <7259755+phdelodder@users.noreply.github.com> --- CHANGELOG.md | 9 + CONTRIBUTING.md | 2 +- docs/de/providers/imap-smtp.md | 10 +- docs/providers/imap-smtp.md | 7 + .../Backfill/RepeatedHeaderBackfillTask.php | 184 +++++++++++++++ src/Domain/Helper/RepeatedHeaderHelper.php | 72 ++++++ src/Repository/Mail/MessageRepository.php | 38 +++ src/Service/Imap/MessageSyncer.php | 47 ++++ .../Backfill/RepeatedHeaderBackfillTest.php | 188 +++++++++++++++ .../Helper/RepeatedHeaderHelperTest.php | 70 ++++++ .../Service/Imap/RepeatedHeaderIngestTest.php | 217 ++++++++++++++++++ 11 files changed, 842 insertions(+), 2 deletions(-) create mode 100644 src/Command/Backfill/RepeatedHeaderBackfillTask.php create mode 100644 src/Domain/Helper/RepeatedHeaderHelper.php create mode 100644 tests/Command/Backfill/RepeatedHeaderBackfillTest.php create mode 100644 tests/Domain/Helper/RepeatedHeaderHelperTest.php create mode 100644 tests/Service/Imap/RepeatedHeaderIngestTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 61e86d0f..3a0c7784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,15 @@ gets an hour, but the hour stops at midnight. Events already synced are correcte next changes them, or right away by removing the calendar and adding it again. [#170](https://github.com/karatektus/pl_mail/issues/170). +**Some IMAP messages no longer show their own headers above the content.** A Stalwart server +answers some messages' body with the whole message, headers included, and plMail stored that as the +body: the message opened with `Delivered-To:`, `Received:` and the rest of its header block printed +as text, and its original held the headers twice. An exact repeat of the header block at the top of +the body is now dropped before the message is read. Mail already stored that way is repaired by +`app:backfill repeated-headers`, from the stored original and without asking the server; it reads +the original of every IMAP message once. No migration. +[#173](https://github.com/karatektus/pl_mail/issues/173), by @phdelodder. + ## v0.3.6 — 2026-10-16 **Several Gmail accounts import without failing on a duplicate key.** With more than one Gmail diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7fdaa0a7..867b5978 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -918,7 +918,7 @@ php bin/mirror-wiki.php --check | `app:mail:wake-snoozed` | Return snoozed conversations whose time has come. Runs every minute | | `app:mail:release-held` | Show mail that has been held for the assistant longer than **Wait at most** allows. Runs every minute and finds nothing on an ordinary day — the live worker releases held mail itself. It is the backstop for that worker being down, and it warns when it has to act | | `app:calendar:materialise [--dry-run]` | Redraw the occurrences of recurring events whose horizon no longer reaches far enough. Runs nightly; without it a long-untouched series eventually runs out of dates | -| `app:backfill [task]` | Run a one-off backfill over stored data; with no argument it lists the tasks and asks. The tasks that re-read mail (`events`, `insights`) cover the last 24 hours unless `--since` says otherwise (`7d`, `2026-09-01`, `all`). `events` re-runs calendar extraction, `proposals` re-reads mail for dates written in prose, `safe-html-table-links` restores links that were wrapped around a table and came out empty, `safe-html-conditional-comments` removes Outlook's `` from messages that show it as text | +| `app:backfill [task]` | Run a one-off backfill over stored data; with no argument it lists the tasks and asks. The tasks that re-read mail (`events`, `insights`) cover the last 24 hours unless `--since` says otherwise (`7d`, `2026-09-01`, `all`). `events` re-runs calendar extraction, `proposals` re-reads mail for dates written in prose, `safe-html-table-links` restores links that were wrapped around a table and came out empty, `safe-html-conditional-comments` removes Outlook's `` from messages that show it as text, `repeated-headers` takes out the header block some IMAP servers repeated at the top of a message body | | `app:imap:idle ` | Hold an IMAP IDLE connection for a single mailbox | | `app:imap:supervise` | Spawn and watch one `app:imap:idle` process per IDLE-enabled mailbox | | `app:imap:test [--account=ID]` | Test an IMAP connection and folder listing | diff --git a/docs/de/providers/imap-smtp.md b/docs/de/providers/imap-smtp.md index a76f7c15..a416f574 100644 --- a/docs/de/providers/imap-smtp.md +++ b/docs/de/providers/imap-smtp.md @@ -1,4 +1,4 @@ - + # IMAP und SMTP @@ -172,3 +172,11 @@ markiert ihn mit `\Noselect` und lässt sich nicht darauf ein, ihn zu öffnen; p mit ausgeschalteter Synchronisation an und fragt ihn nie ab. Die Ordner darunter sind gewöhnlich und werden wie üblich synchronisiert. Ein Ordner, den der Server später auswählbar macht, wird nicht von selbst wieder eingeschaltet. + +**Manche Server schicken die Kopfzeilen einer Nachricht als Anfang ihres Inhalts noch einmal.** Bei +einem Stalwart-Server wurde das bei einigen Nachrichten beobachtet, bei anderen nicht. plMail erkennt +eine exakte Wiederholung des Kopfblocks und lässt sie weg, sodass die Nachricht einmal angezeigt und +gespeichert wird. Mail, die vorher synchronisiert wurde, zeigt ihre Zeilen `Delivered-To:` und +`Received:` über dem Inhalt; `app:backfill repeated-headers` repariert sie aus dem gespeicherten +Original, ohne den Server zu fragen. Es liest dafür das Original jeder IMAP-Nachricht einmal, auf +einem großen Postfach dauert das also eine Weile. diff --git a/docs/providers/imap-smtp.md b/docs/providers/imap-smtp.md index 71da0d06..f690296a 100644 --- a/docs/providers/imap-smtp.md +++ b/docs/providers/imap-smtp.md @@ -159,3 +159,10 @@ mark it read here. Mail, Sent Mail, Drafts and the rest. Gmail marks it `\Noselect` and refuses to open it, so plMail creates it with sync switched off and never polls it; the folders underneath are ordinary and sync as usual. A folder the server later makes selectable is not switched back on by itself. + +**Some servers send a message's headers again as the start of its body.** A Stalwart server has +been seen doing this for some messages and not others. plMail recognises an exact repeat of the +header block and drops it, so the message is shown and stored once. Mail synced before that was +fixed shows its `Delivered-To:` and `Received:` lines above the content; `app:backfill +repeated-headers` repairs it from the stored original, without asking the server. It reads every +IMAP message's original once, so on a large mailbox it takes a while. diff --git a/src/Command/Backfill/RepeatedHeaderBackfillTask.php b/src/Command/Backfill/RepeatedHeaderBackfillTask.php new file mode 100644 index 00000000..9505ca65 --- /dev/null +++ b/src/Command/Backfill/RepeatedHeaderBackfillTask.php @@ -0,0 +1,184 @@ +messages->countImapWithStoredRaw(); + + if (0 === $total) { + $io->success('Nothing to read — no IMAP message has its original stored.'); + + return Command::SUCCESS; + } + + $io->progressStart($total); + + $lastId = 0; + $repaired = 0; + + while (true) { + $messages = $this->messages->findImapWithStoredRaw($lastId, self::BATCH_SIZE); + + if (count($messages) === 0) { + break; + } + + foreach ($messages as $message) { + $lastId = (int) $message->id; + + if (true === $this->repair($message)) { + ++$repaired; + } + + $io->progressAdvance(); + } + + $this->em->flush(); + $this->em->clear(); + } + + $io->progressFinish(); + $io->success(sprintf('%d message(s) repaired, %d read.', $repaired, $total)); + + return Command::SUCCESS; + } + + // ── Private ─────────────────────────────────────────────────────────────── + + /** + * @return bool true when the message carried a repeated header block + */ + private function repair(Message $message): bool + { + $path = (string) $message->rawPath; + + try { + if (false === $this->rawStorage->exists($path)) { + return false; + } + + $raw = (string) file_get_contents($this->rawStorage->getAbsolutePath($path)); + + // MessageSyncer::rawOf() joins the halves with exactly this. + $split = strpos($raw, "\r\n\r\n"); + + if (false === $split) { + return false; + } + + $header = substr($raw, 0, $split); + $body = RepeatedHeaderHelper::strip($header, substr($raw, $split + 4)); + + if (null === $body) { + return false; + } + + $repaired = $header . "\r\n\r\n" . $body; + + // Parsed before anything is written, so a message the library + // cannot read is left exactly as it was rather than half repaired. + $parsed = ImapMessage::fromString($repaired, ImapConnectionFactory::config()); + } catch (Throwable $e) { + // Skipped rather than fatal: one unreadable file is one message + // that stays as it was, and stopping here would leave every later + // one unrepaired too. + $this->logger->warning('Could not read a stored message for the repeated-header repair', [ + 'messageId' => $message->id, + 'error' => $e->getMessage(), + ]); + + return false; + } + + $message->rawPath = $this->rawStorage->store((int) $message->account->id, (int) $message->id, $repaired); + + if (IMAP::MESSAGE_TYPE_MULTIPART !== $parsed->getStructure()?->type) { + $message->bodyText = CharsetHelper::ensureUtf8($parsed->getTextBody()); + $message->bodyHtml = CharsetHelper::ensureUtf8($parsed->getHTMLBody()); + + // bodyHtmlSafe is what the reading pane renders; bodyHtml alone + // would repair the database and leave the screen as it was. + $this->sanitizer->sanitize($message); + } + + return true; + } +} diff --git a/src/Domain/Helper/RepeatedHeaderHelper.php b/src/Domain/Helper/RepeatedHeaderHelper.php new file mode 100644 index 00000000..d03d949a --- /dev/null +++ b/src/Domain/Helper/RepeatedHeaderHelper.php @@ -0,0 +1,72 @@ +getSingleScalarResult(); } + /** + * IMAP messages after $afterId whose original bytes are on disk, in id + * order: the set RepeatedHeaderBackfillTask reads. + * + * Every one of them, not the ones whose body looks wrong. A repeated + * header block is only visible in body_html for a single-part message; a + * multipart one renders correctly and carries the doubled block in its + * .eml all the same, and only reading the file finds it. + * + * @return list + */ + public function findImapWithStoredRaw(int $afterId, int $limit): array + { + return $this->imapWithStoredRawQueryBuilder($afterId) + ->orderBy('m.id', 'ASC') + ->setMaxResults($limit) + ->getQuery() + ->getResult(); + } + + public function countImapWithStoredRaw(): int + { + return (int) $this->imapWithStoredRawQueryBuilder(0) + ->select('COUNT(m.id)') + ->getQuery() + ->getSingleScalarResult(); + } + + /** Shared so the count and the walk can never disagree about the set. */ + private function imapWithStoredRawQueryBuilder(int $afterId): \Doctrine\ORM\QueryBuilder + { + return $this->createQueryBuilder('m') + ->andWhere('m.id > :afterId') + ->andWhere('m.imapUid IS NOT NULL') + ->andWhere('m.rawPath IS NOT NULL') + ->setParameter('afterId', $afterId); + } + /** Shared so the count and the walk can never disagree about the set. */ private function misfiledBodyPartQueryBuilder(int $afterId): \Doctrine\ORM\QueryBuilder { diff --git a/src/Service/Imap/MessageSyncer.php b/src/Service/Imap/MessageSyncer.php index 5632fd59..995a5c24 100644 --- a/src/Service/Imap/MessageSyncer.php +++ b/src/Service/Imap/MessageSyncer.php @@ -11,6 +11,7 @@ use App\Domain\Helper\CharsetHelper; use App\Domain\Helper\ImapConnectionFactory; use App\Domain\Helper\MessageIdHelper; use App\Domain\Helper\MimeHeaderHelper; +use App\Domain\Helper\RepeatedHeaderHelper; use App\Entity\Mail\Mailbox; use App\Entity\Mail\Message; use App\Entity\Mail\MessagePart; @@ -28,6 +29,7 @@ use Psr\Log\LoggerInterface; use Webklex\PHPIMAP\Attribute; use Webklex\PHPIMAP\Client; use Webklex\PHPIMAP\Message as ImapMessage; +use Webklex\PHPIMAP\Support\AttachmentCollection; use App\Service\Mail\HeaderNormalizer; class MessageSyncer @@ -531,6 +533,10 @@ class MessageSyncer } try { + // Before anything reads the body, so the columns, the parts and + // the stored .eml all come from the same bytes. + $this->dropRepeatedHeader($imapMessage, $uid, $mailbox->fullPath); + $message = $this->buildMessage($imapMessage, $mailbox, $accountId); $this->em->persist($message); @@ -739,6 +745,47 @@ class MessageSyncer return true; } + /** + * Parse the body again without the header block a server repeated at the + * top of it. + * + * Some messages on a Stalwart server answer RFC822.TEXT with the whole + * message, headers included, while RFC822.HEADER is right (#173). webklex + * builds its parts from whatever the TEXT literal held, so a single-part + * HTML mail was stored with its own `Delivered-To: …` block as the first + * lines of body_html, and rawOf() wrote the headers into the .eml twice. + * See RepeatedHeaderHelper for how a repeat is recognised. + * + * Read from the structure, not getRawBody(): getRawBody() keeps the first + * answer it gives, so asking it here would pin the doubled bytes for + * rawOf() below. parseRawBody() appends to the bodies and attachments an + * earlier parse left behind, which is why both are emptied first. + */ + private function dropRepeatedHeader(ImapMessage $imapMessage, int $uid, string $mailboxPath): void + { + $header = $imapMessage->getHeader(); + $structure = $imapMessage->getStructure(); + + if (null === $header || null === $structure) { + return; + } + + $body = RepeatedHeaderHelper::strip($header->raw, $structure->raw); + + if (null === $body) { + return; + } + + $imapMessage->bodies = []; + $imapMessage->setAttachments(AttachmentCollection::make()); + $imapMessage->parseRawBody($body); + + $this->logger->info('Dropped a header block the server repeated at the top of the body', [ + 'uid' => $uid, + 'mailbox' => $mailboxPath, + ]); + } + /** * Reassemble the message as it arrived: raw headers, blank line, raw body. * webklex keeps both around after parsing, so nothing is re-fetched. diff --git a/tests/Command/Backfill/RepeatedHeaderBackfillTest.php b/tests/Command/Backfill/RepeatedHeaderBackfillTest.php new file mode 100644 index 00000000..1d9ffb51 --- /dev/null +++ b/tests/Command/Backfill/RepeatedHeaderBackfillTest.php @@ -0,0 +1,188 @@ + .eml files written to disk, which no rollback removes */ + private array $storedRawPaths = []; + + protected function setUp(): void + { + self::bootKernel(); + + $container = self::getContainer(); + $this->em = $container->get(EntityManagerInterface::class); + $this->connection = $container->get(Connection::class); + $this->task = $container->get(RepeatedHeaderBackfillTask::class); + $this->rawStorage = $container->get(RawMessageStorage::class); + + $this->connection->beginTransaction(); + + $this->user = $this->seedUser(); + $this->account = $this->seedAccount(); + } + + protected function tearDown(): void + { + foreach ($this->storedRawPaths as $path) { + $this->rawStorage->delete($path); + } + + if (true === $this->connection->isTransactionActive()) { + $this->connection->rollBack(); + } + + parent::tearDown(); + } + + public function testASinglePartMessageGetsItsBodyBackRatherThanItsHeadersAgain(): void + { + $header = $this->header('text/html; charset=utf-8'); + $html = "\r\n

Your parcel is on its way.

\r\n"; + + $id = $this->message($header . "\r\n\r\n" . $header . "\r\n\r\n" . $html, bodyHtml: $header . "\r\n\r\n" . $html); + + self::assertSame(Command::SUCCESS, $this->runTask()); + + $message = $this->reload($id); + + self::assertStringStartsWith('', (string) $message->bodyHtml); + self::assertStringNotContainsString('Delivered-To', (string) $message->bodyHtmlSafe); + self::assertStringContainsString('Your parcel is on its way.', (string) $message->bodyHtmlSafe); + self::assertSame($header . "\r\n\r\n" . $html, $this->raw($message)); + } + + public function testAMultipartMessageHasItsFileRepairedAndItsBodiesLeftAlone(): void + { + $header = $this->header('multipart/alternative; boundary="B"'); + $parts = "--B\r\n" + . "Content-Type: text/plain; charset=utf-8\r\n\r\nYour parcel is on its way.\r\n" + . "--B\r\n" + . "Content-Type: text/html; charset=utf-8\r\n\r\n

Your parcel is on its way.

\r\n" + . "--B--\r\n"; + + // Bodies the parse would not produce, standing for a repair the sync + // made: the backfill must not replace them with its own reading. + $id = $this->message( + $header . "\r\n\r\n" . $header . "\r\n\r\n" . $parts, + bodyHtml: '

As the sync stored it.

', + bodyText: 'As the sync stored it.', + ); + + $this->runTask(); + + $message = $this->reload($id); + + self::assertSame($header . "\r\n\r\n" . $parts, $this->raw($message)); + self::assertSame('

As the sync stored it.

', $message->bodyHtml); + self::assertSame('As the sync stored it.', $message->bodyText); + } + + public function testAMessageStoredOnceIsLeftAsItWas(): void + { + $header = $this->header('text/plain; charset=utf-8'); + $raw = $header . "\r\n\r\nDelivered-To: is a line of this body.\r\n"; + + $id = $this->message($raw, bodyText: "Delivered-To: is a line of this body.\r\n"); + + $this->runTask(); + + $message = $this->reload($id); + + self::assertSame($raw, $this->raw($message)); + self::assertSame("Delivered-To: is a line of this body.\r\n", $message->bodyText); + } + + // ---------------------------------------------------------------- helpers + + private function runTask(): int + { + return $this->task->run(new SymfonyStyle(new ArrayInput([]), new BufferedOutput())); + } + + private function header(string $contentType): string + { + return "Delivered-To: {$this->account->email}\r\n" + . "From: Parcels \r\n" + . "Subject: Your parcel\r\n" + . "Date: Mon, 21 Sep 2026 10:00:00 +0200\r\n" + . sprintf("Message-ID: <%s@example.test>\r\n", uniqid('parcel-', true)) + . "MIME-Version: 1.0\r\n" + . "Content-Type: {$contentType}"; + } + + private function message(string $raw, ?string $bodyHtml = null, ?string $bodyText = null): int + { + $message = new Message(); + $message->account = $this->account; + $message->subject = 'Your parcel'; + $message->fromAddress = 'parcels@example.test'; + $message->receivedAt = new DateTimeImmutable(); + $message->hasAttachments = false; + $message->imapUid = 1; + $message->bodyHtml = $bodyHtml; + $message->bodyText = $bodyText; + + $this->em->persist($message); + $this->em->flush(); + + $message->rawPath = $this->rawStorage->store((int) $this->account->id, (int) $message->id, $raw); + $this->storedRawPaths[] = $message->rawPath; + + $this->em->flush(); + + return (int) $message->id; + } + + private function reload(int $id): Message + { + $this->em->clear(); + + $message = $this->em->getRepository(Message::class)->find($id); + + self::assertInstanceOf(Message::class, $message); + + return $message; + } + + private function raw(Message $message): string + { + return (string) file_get_contents($this->rawStorage->getAbsolutePath((string) $message->rawPath)); + } +} diff --git a/tests/Domain/Helper/RepeatedHeaderHelperTest.php b/tests/Domain/Helper/RepeatedHeaderHelperTest.php new file mode 100644 index 00000000..699b23a5 --- /dev/null +++ b/tests/Domain/Helper/RepeatedHeaderHelperTest.php @@ -0,0 +1,70 @@ +\r\n" + . "Content-Type: text/html; charset=utf-8"; + + #[DataProvider('repeats')] + public function testARepeatedHeaderBlockIsTakenOffTheBody(string $rawHeader, string $rawBody, string $expected): void + { + self::assertSame($expected, RepeatedHeaderHelper::strip($rawHeader, $rawBody)); + } + + /** + * @return iterable + */ + public static function repeats(): iterable + { + $html = "\r\n

Hello

\r\n"; + + yield 'with the blank line, header as IMAP sends it' => [self::HEADER . "\r\n\r\n", self::HEADER . "\r\n\r\n" . $html, $html]; + yield 'with the blank line, header already trimmed' => [self::HEADER, self::HEADER . "\r\n\r\n" . $html, $html]; + yield 'without a blank line before the DOCTYPE' => [self::HEADER . "\r\n\r\n", self::HEADER . "\r\n" . $html, $html]; + yield 'a body that opens with an empty line keeps it' => [self::HEADER, self::HEADER . "\r\n\r\n\r\n" . $html, "\r\n" . $html]; + yield 'bare line feeds' => [self::HEADER, self::HEADER . "\n\n" . $html, $html]; + yield 'nothing after the headers' => [self::HEADER, self::HEADER . "\r\n\r\n", '']; + } + + #[DataProvider('nearMisses')] + public function testABodyThatDoesNotRepeatTheHeadersRatherThanLooksLikeItIsLeftAlone(string $rawHeader, string $rawBody): void + { + self::assertNull(RepeatedHeaderHelper::strip($rawHeader, $rawBody)); + } + + /** + * @return iterable + */ + public static function nearMisses(): iterable + { + yield 'an ordinary body' => [self::HEADER, "\r\n

Hello

\r\n"]; + + // Header-shaped, but not this message's headers. + yield 'a text body that opens like a header' => [self::HEADER, "Note: the parcel is late.\r\n"]; + + // The same first lines, then the block goes on: the server's header + // literal is the whole block, so a longer one is not a repeat of it. + yield 'the block continues past the last header line' => [self::HEADER, self::HEADER . "; format=flowed\r\n\r\nHello\r\n"]; + + yield 'an empty header' => ['', "Hello\r\n"]; + } +} diff --git a/tests/Service/Imap/RepeatedHeaderIngestTest.php b/tests/Service/Imap/RepeatedHeaderIngestTest.php new file mode 100644 index 00000000..39413aac --- /dev/null +++ b/tests/Service/Imap/RepeatedHeaderIngestTest.php @@ -0,0 +1,217 @@ + .eml files written to disk, which no rollback removes */ + private array $storedRawPaths = []; + + protected function setUp(): void + { + self::bootKernel(); + + $this->em = static::getContainer()->get(EntityManagerInterface::class); + $this->connection = static::getContainer()->get(Connection::class); + $this->importer = static::getContainer()->get(ImapMailboxImporter::class); + $this->rawStorage = static::getContainer()->get(RawMessageStorage::class); + + $this->connection->beginTransaction(); + + $this->user = $this->seedUser(); + $this->account = $this->seedAccount(); + + $mailbox = new Mailbox(); + $mailbox->account = $this->account; + $mailbox->name = 'Deleted Items'; + $mailbox->fullPath = 'Deleted Items'; + $mailbox->isSyncEnabled = true; + $mailbox->lastSeenUid = 3; + $mailbox->importFloorUid = 4; + + $this->em->persist($mailbox); + $this->em->flush(); + + $this->mailboxId = (int) $mailbox->id; + } + + protected function tearDown(): void + { + foreach ($this->storedRawPaths as $path) { + $this->rawStorage->delete($path); + } + + if (true === $this->connection->isTransactionActive()) { + $this->connection->rollBack(); + } + + parent::tearDown(); + } + + public function testTheBodyIsTheBodyRatherThanTheHeadersAgain(): void + { + $html = "\r\n

Your parcel is on its way.

\r\n"; + + $this->importer->importPage($this->mailboxId, FakeImportClient::holding([ + 1 => $this->doubled(1, "Content-Type: text/html; charset=utf-8\r\n", "\r\n\r\n" . $html), + // No blank line between the repeated block and the DOCTYPE, the + // other shape the issue reports. + 2 => $this->doubled(2, "Content-Type: text/html; charset=utf-8\r\n", "\r\n" . $html), + ])); + + $rows = $this->rows(); + + self::assertSame([1, 2], array_keys($rows)); + + foreach ($rows as $uid => $row) { + self::assertStringNotContainsString('Delivered-To', (string) $row['body_html'], "uid {$uid}: the headers are not body text"); + self::assertStringStartsWith('', (string) $row['body_html'], "uid {$uid}"); + self::assertStringNotContainsString('Delivered-To', (string) $row['body_html_safe'], "uid {$uid}: nor on screen"); + self::assertSame(1, substr_count($this->raw($row), 'Delivered-To: '), "uid {$uid}: the .eml carries the headers once"); + } + + // Header and body separated by exactly one blank line, which is what + // makes the .eml a message any client opens — including the one that + // came without a blank line at all. + self::assertStringContainsString("\r\nMIME-Version: 1.0\r\nContent-Type: text/html; charset=utf-8\r\n\r\n", $this->raw($rows[2])); + } + + public function testAMultipartMessageIsStoredOnceRatherThanWithItsHeadersTwice(): void + { + $parts = "\r\n\r\n--B\r\n" + . "Content-Type: text/plain; charset=utf-8\r\n\r\nYour parcel is on its way.\r\n" + . "--B\r\n" + . "Content-Type: text/html; charset=utf-8\r\n\r\n

Your parcel is on its way.

\r\n" + . "--B--\r\n"; + + $this->importer->importPage($this->mailboxId, FakeImportClient::holding([ + 3 => $this->doubled(3, "Content-Type: multipart/alternative; boundary=\"B\"\r\n", $parts), + ])); + + $row = $this->rows()[3]; + + self::assertSame(1, substr_count($this->raw($row), 'Delivered-To: ')); + self::assertStringContainsString('

Your parcel is on its way.

', (string) $row['body_html']); + self::assertSame('Your parcel is on its way.', trim((string) $row['body_text'])); + self::assertFalse((bool) $row['has_attachments'], 'no part became an attachment on the way'); + } + + public function testAnOrdinaryMessageIsStoredAsItCame(): void + { + $header = $this->header(1, "Content-Type: text/plain; charset=utf-8\r\n"); + + $this->importer->importPage($this->mailboxId, FakeImportClient::holding([ + 1 => $this->fromString($header . "\r\nDelivered-To: is a line of this body.\r\n", 1), + ])); + + $row = $this->rows()[1]; + + // A body that opens like a header is still the body. Only an exact + // repeat of the header block is taken off. + self::assertSame('Delivered-To: is a line of this body.', trim((string) $row['body_text'])); + self::assertSame(2, substr_count($this->raw($row), 'Delivered-To: ')); + } + + // ---------------------------------------------------------------- helpers + + /** + * The message as such a server answers it: the header block, a blank line, + * the same header block, then $rest — which starts with the blank line, + * or with a lone line break where the server sent none. + */ + private function doubled(int $uid, string $contentType, string $rest): ImapMessage + { + $header = rtrim($this->header($uid, $contentType), "\r\n"); + + return $this->fromString($header . "\r\n\r\n" . $header . $rest, $uid); + } + + private function header(int $uid, string $contentType): string + { + return "Delivered-To: {$this->account->email}\r\n" + . "From: Parcels \r\n" + . "To: {$this->account->email}\r\n" + . "Subject: Parcel {$uid}\r\n" + . "Date: Mon, 21 Sep 2026 10:00:00 +0200\r\n" + . "Message-ID: \r\n" + . "MIME-Version: 1.0\r\n" + . $contentType; + } + + private function fromString(string $raw, int $uid): ImapMessage + { + return ImapMessage::fromString($raw, ImapConnectionFactory::config()) + ->setUid($uid) + ->setFlags(FlagCollection::make(['Seen' => 'Seen'])); + } + + /** + * @param array $row + */ + private function raw(array $row): string + { + $path = (string) $row['raw_path']; + + self::assertNotSame('', $path, 'the sync wrote no .eml'); + + $this->storedRawPaths[] = $path; + + return (string) file_get_contents($this->rawStorage->getAbsolutePath($path)); + } + + /** + * @return array> keyed by IMAP uid + */ + private function rows(): array + { + $rows = []; + + foreach ($this->connection->fetchAllAssociative( + 'SELECT imap_uid, body_html, body_html_safe, body_text, has_attachments, raw_path FROM message WHERE mailbox_id = ? ORDER BY imap_uid', + [$this->mailboxId], + ) as $row) { + $rows[(int) $row['imap_uid']] = $row; + } + + return $rows; + } +}