diff --git a/.gitignore b/.gitignore index 142b8f21..8145dcc5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,15 @@ # VS Code stuff. /.vscode/ +# Visual studio stuff. +/.vs/ + +# Test stuff. +/.pytest_cache/ + +# Coverage info. +/.coverage + # Build files and folders /build/ *.egg-info/ @@ -30,6 +39,7 @@ __pycache__/ !/example-msg-files/expected-outputs/2013-11-18_0026 Test for TIF files/* !/example-msg-files/expected-outputs/2016-02-23_0657 MSG Test File/ !/example-msg-files/expected-outputs/2016-02-23_0657 MSG Test File/* +!/example-msg-files/signed.msg !/example-msg-files/strangeDate.msg !/example-msg-files/unicode.msg !/example-msg-files/multi-stream.msg diff --git a/CHANGELOG.md b/CHANGELOG.md index 8592172d..994cdba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +**v0.57.1** +* [[TeamMsgExtractor #489](https://github.com/TeamMsgExtractor/msg-extractor/issues/489)] Fix an issue where `MessageSignedBase.body` would raise an error when the plain text body was not utf-8 encoded bytes. This error was due to `extract_msg.utils.unwrapMultipart()` accidentally returning the plain text body as bytes instead of string. +* Change some of the command line documentation. +* Fix some of the imports in `extract_msg/__init__.py`. +* Add additional tests to module. + **v0.57.0** * [[Advisory GHSA-cxpg-prmv-g4pp](https://github.com/TeamMsgExtractor/msg-extractor/security/advisories/GHSA-cxpg-prmv-g4pp)] Fixed a vulnerability related to unchecked input passed out to wkhtmltopdf when converting the MSG file to PDF. Javascript is now strongly blocked from being enabled and will be stripped from the HTML where appropriate before reaching wkhtmltopdf. Additionally, the option `--wk-allow-external-files` on the command line (or `wkAllowExternalFiles` to `MessageBase.save()`) will be required for external files to be loaded into the HTML to convert. Without these, they will be stripped before reaching wkhtmltopdf for security. * [[Advisory GHSA-mqxr-j8rh-p4q4](https://github.com/TeamMsgExtractor/msg-extractor/security/advisories/GHSA-mqxr-j8rh-p4q4)] Fixed an issue where an attacker controlled MSG file could take advantage of a poorly written section of code in the RTF deencapsulation function on the `extract-msg` side to waste massive amounts of time and memory resources. diff --git a/README.rst b/README.rst index 3d609335..c26ff199 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ refer to the usage information provided from the program's help dialog: --pdf Saves the body as a PDF. If this is not possible, will error. --wk-path WKPATH Overrides the path for finding wkhtmltopdf. --wk-options [WKOPTIONS ...] - Sets additional options to be used in wkhtmltopdf. Should be a series of options and values, replacing the - or -- in the beginning with + or ++ respectively. For example: --wk-options "+O Landscape" + Sets additional options to be used in wkhtmltopdf. Should be a series of options and values, replacing the - or -- in the beginning with + or ++ respectively. For example: --wk-options "+O Landscape" "++no-images" --wk-allow-external-files Enables external files to appear in the PDF output. WARNING: This allows wkpdftohtml to make url requests that are unchecked, and as such may be a security vulnerability in untrusted emails. Only use this option if you trust the source of the MSG file. --prepared-html When used in conjunction with --html, sets whether the HTML output should be prepared for embedded attachments. @@ -268,8 +268,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.57.0-blue.svg - :target: https://pypi.org/project/extract-msg/0.57.0/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.57.1-blue.svg + :target: https://pypi.org/project/extract-msg/0.57.1/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.8+-brightgreen.svg :target: https://www.python.org/downloads/release/python-3810/ diff --git a/example-msg-files/signed.msg b/example-msg-files/signed.msg new file mode 100644 index 00000000..82f97572 Binary files /dev/null and b/example-msg-files/signed.msg differ diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 939dd08a..8bbb2290 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2026-10-24' -__version__ = '0.57.0' +__date__ = '2026-10-28' +__version__ = '0.57.1' __all__ = [ # Modules: @@ -37,9 +37,11 @@ 'enums', 'exceptions', 'msg_classes', + 'open_msg', 'null_date', 'properties', 'structures', + 'utils', # Classes: 'Attachment', @@ -61,6 +63,7 @@ # Ensure these are imported before anything else. from . import constants, enums, exceptions +from . import utils from . import attachments, msg_classes, null_date, properties, structures from .attachments import Attachment, AttachmentBase, SignedAttachment diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 5992e1b4..9ea725be 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -417,7 +417,7 @@ def getCommandArgs(args: Sequence[str]) -> argparse.Namespace: help='Overrides the path for finding wkhtmltopdf.') # --wk-options OPTIONS parser.add_argument('--wk-options', dest='wkOptions', nargs='*', - help='Sets additional options to be used in wkhtmltopdf. Should be a series of options and values, replacing the - or -- in the beginning with + or ++, respectively. For example: --wk-options "+O Landscape"') + help='Sets additional options to be used in wkhtmltopdf. Should be a series of options and values, replacing the - or -- in the beginning with + or ++, respectively. For example: --wk-options "+O Landscape" "++no-images"') # --wk-allow-external-files parser.add_argument('--wk-allow-external-files', dest='allowExternalFiles', action='store_true', help='Enables external files to appear in the PDF output. WARNING: This allows wkpdftohtml to make url requests that are unchecked, and as such may be a security vulnerability in untrusted emails. Only use this option if you trust the source of the MSG file.') @@ -1045,6 +1045,7 @@ def stripRtf(rtfBody: bytes) -> bytes: return rtfBody + def _stripRtfCloseHelper(match: re.Match) -> bytes: if (ret := match.expand(b'\\g<0>')).count(b'\\htmlrtf0') > 1: return ret @@ -1080,8 +1081,6 @@ def _stripRtfHelper(match: re.Match) -> bytes: return res - - def tryGetMimetype(att: AttachmentBase, mimetype: Union[str, None]) -> Union[str, None]: """ Uses an optional dependency to try and get the mimetype of an attachment. @@ -1305,7 +1304,10 @@ def unwrapMultipart(mp: Union[bytes, str, email.message.Message]) -> Dict: elif attachment['mimetype'] == 'text/plain': if plainBody: logger.warning('Found multiple candidates for plain text body.') - plainBody = data + # We need to get the content again and ensure it is a string. If + # not, decode it to one. + content = inputToString(node.get_content(), node.get_content_charset('utf-8')) + plainBody = content elif attachment['mimetype'] == 'text/html': if htmlBody: logger.warning('Found multiple candidates for HTML body.') diff --git a/extract_msg_tests/__init__.py b/extract_msg_tests/__init__.py index 3686d6f6..bf6568d4 100644 --- a/extract_msg_tests/__init__.py +++ b/extract_msg_tests/__init__.py @@ -6,6 +6,7 @@ 'OleWriterEditingTests', 'OleWriterExportTests', 'PropTests', + 'SignedTests', 'UtilTests', 'ValidationTests', 'WkTests', @@ -17,6 +18,7 @@ from .msg_base_tests import MSGBaseTests from .ole_writer_tests import OleWriterEditingTests, OleWriterExportTests from .prop_tests import PropTests +from .signed_tests import SignedTests from .util_tests import UtilTests from .validation_tests import ValidationTests from .wk_tests import WkTests diff --git a/extract_msg_tests/cmd_line_tests.py b/extract_msg_tests/cmd_line_tests.py index 71326083..32a93263 100644 --- a/extract_msg_tests/cmd_line_tests.py +++ b/extract_msg_tests/cmd_line_tests.py @@ -1,13 +1,89 @@ +from __future__ import annotations + __all__ = [ 'CommandLineTests', ] +import io +import logging import subprocess import sys import unittest +import extract_msg + +from typing import Tuple + from .constants import TEST_FILE_DIR, USER_TEST_DIR +from extract_msg import __main__ + + +class _StdRedirect: + def __init__(self, data: bytes = None): + if data: + self.inBuffer = io.BytesIO(data) + else: + self.inBuffer = io.BytesIO() + self.outBuffer = io.BytesIO() + self.errBuffer = io.BytesIO() + self.inReplacer = io.TextIOWrapper(self.inBuffer, sys.stdin.encoding, newline = '') + self.outReplacer = io.TextIOWrapper(self.outBuffer, sys.stdout.encoding, newline = '', write_through = True) + self.errReplacer = io.TextIOWrapper(self.errBuffer, sys.stderr.encoding, newline = '', write_through = True) + self.stdin = None + self.stdout = None + self.stderr = None + self.entered = False + + def bufferInput(self, data: bytes) -> _StdRedirect: + """ + Set the stdin data to :param data:. + """ + self.inBuffer.truncate(0) + self.inBuffer.seek(0) + self.inBuffer.write(data) + self.inBuffer.seek(0) + return self + + def __enter__(self, *args, **kwargs): + if not self.entered: + self.entered = True + self.stdin = sys.stdin + self.stdout = sys.stdout + self.stderr = sys.stderr + # I feel like this should be simple, but without doing both it tends + # to just leave the data null in the buffer. + self.inBuffer.seek(0) + self.outBuffer.truncate(0) + self.outBuffer.seek(0) + self.errBuffer.truncate(0) + self.errBuffer.seek(0) + self.outReplacer.truncate(0) + self.outReplacer.seek(0) + self.errReplacer.truncate(0) + self.errReplacer.seek(0) + sys.stdin = self.inReplacer + sys.stdout = self.outReplacer + sys.stderr = self.errReplacer + + def __exit__(self, *args, **kwargs): + if self.entered: + self.entered = False + sys.stdin = self.stdin + self.stdin = None + sys.stdout = self.stdout + self.stdout = None + sys.stderr = self.stderr + self.stderr = None + + def dumpRead(self) -> Tuple[bytes, bytes]: + """ + Dumps the output to stdout and stderr. + """ + self.outBuffer.seek(0) + self.errBuffer.seek(0) + return self.outBuffer.read(), self.errBuffer.read() + class CommandLineTests(unittest.TestCase): @@ -28,6 +104,61 @@ def testStdin(self, testFileDir = TEST_FILE_DIR): self.assertEqual(stdout1, stdout2) self.assertEqual(stderr1, stderr2) + def testDirectRunStdin(self, testFileDir = TEST_FILE_DIR): + """ + Attempt the stdin tests without calling a subprocess. + """ + ioRedirect = _StdRedirect() + for path in testFileDir.glob('*.msg'): + with ioRedirect.bufferInput(b''): + # First, let's do the file on the disk. + __main__.main([__main__.__name__, '--dump-stdout', str(path)]) + + stdout1, stderr1 = ioRedirect.dumpRead() + + with open(path, 'rb') as f: + fileRead = f.read() + + with ioRedirect.bufferInput(fileRead): + # Now, do the same thing with stdin. + __main__.main([__main__.__name__, '-s', '--dump-stdout']) + + stdout2, stderr2 = ioRedirect.dumpRead() + + # Now, compare the two. + with self.subTest(path): + self.assertEqual(stdout1, stdout2) + self.assertEqual(stderr1, stderr2) + @unittest.skipIf(USER_TEST_DIR is None, 'User test files not defined.') def testUserStdin(self): - self.testStdin(USER_TEST_DIR) \ No newline at end of file + self.testStdin(USER_TEST_DIR) + self.testDirectRunStdin(USER_TEST_DIR) + + def testGetCommandArgs(self): + """ + Does various tests on the expected behavior of getCommandArgs(). + """ + normalArgs = [ + __main__.__name__, + '--wk-options', + '++no-images', + '+O Landscape', + '++encoding', + 'Latin1', + '-vv' + ] + expectedWk = [ + "--no-images", + "-O", + "Landscape", + "--encoding", + "Latin1", + ] + + parsed = extract_msg.utils.getCommandArgs(normalArgs) + with self.subTest('WK Options'): + self.assertListEqual(parsed.wkOptions, expectedWk) + with self.subTest('Verbosity'): + self.assertEqual(parsed.verbose, 2) + self.assertEqual(parsed.logLevel, logging.INFO) \ No newline at end of file diff --git a/extract_msg_tests/message_tests.py b/extract_msg_tests/message_tests.py index 7b89244f..e6acade6 100644 --- a/extract_msg_tests/message_tests.py +++ b/extract_msg_tests/message_tests.py @@ -5,6 +5,7 @@ import base64 import unittest + from email.message import EmailMessage from .constants import TEST_FILE_DIR diff --git a/extract_msg_tests/signed_tests.py b/extract_msg_tests/signed_tests.py new file mode 100644 index 00000000..da133e97 --- /dev/null +++ b/extract_msg_tests/signed_tests.py @@ -0,0 +1,35 @@ +__all__ = [ + 'SignedTests', +] + + +import pathlib +import unittest + +from .constants import TEST_FILE_DIR, USER_TEST_DIR +from extract_msg import openMsg +from extract_msg.msg_classes import MessageSignedBase + + +class SignedTests(unittest.TestCase): + def testUnwrap(self, testFileDir: pathlib.Path = TEST_FILE_DIR): + """ + Tests the results of extract_msg.utils.unwrapMultipart(). + + The initial test is ensuring the attachments generate at all. After + that, test to make sure the types in the signedBody and signedHtmlBody + are correct. + """ + for path in TEST_FILE_DIR.glob('*.msg'): + with openMsg(path, strict = False, delayAttachments = True) as msg: + if isinstance(msg, MessageSignedBase): + with self.subTest(str(path)): + msg.attachments + if msg.signedBody: + self.assertIsInstance(msg.signedBody, str) + if msg.signedHtmlBody: + self.assertIsInstance(msg.signedHtmlBody, bytes) + + @unittest.skipIf(USER_TEST_DIR is None, 'User test files not defined.') + def testUserStdin(self): + self.testUnwrap(USER_TEST_DIR)