[pull] main from python:main

Doc/library/xml.dom.minidom.rst

@@ -154,6 +154,11 @@ module documentation.  This section lists the differences between the API and
    .. versionchanged:: 3.9
       The *standalone* parameter was added.
 
+   .. versionchanged:: next
+      Namespace declarations missing for the serialized element
+      and its attributes are now written.
+      It now works for :class:`!DocumentFragment` nodes.
+
 .. method:: Node.toxml(encoding=None, standalone=None)
 
    Return a string or byte string containing the XML represented by
@@ -175,6 +180,9 @@ module documentation.  This section lists the differences between the API and
    .. versionchanged:: 3.9
       The *standalone* parameter was added.
 
+   .. versionchanged:: next
+      It now works for :class:`!DocumentFragment` nodes.
+
 .. method:: Node.toprettyxml(indent="\t", newl="\n", encoding=None, \
                              standalone=None)
 
@@ -203,6 +211,7 @@ module documentation.  This section lists the differences between the API and
    .. versionchanged:: next
       Whitespace is no longer added inside an element with mixed content
       or marked with ``xml:space="preserve"``.
+      It now works for :class:`!DocumentFragment` nodes.
 
 .. _dom-example:
 

Doc/whatsnew/3.16.rst

@@ -725,6 +725,12 @@ xml
   rather than defaulted from the DTD.
   (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.)
 
+* The :meth:`~xml.dom.minidom.Node.writexml`,
+  :meth:`~xml.dom.minidom.Node.toxml` and
+  :meth:`~xml.dom.minidom.Node.toprettyxml` methods
+  now work for :class:`!DocumentFragment` nodes in :mod:`xml.dom.minidom`.
+  (Contributed by Serhiy Storchaka in :gh:`54092`.)
+
 * :class:`~xml.etree.ElementTree.XMLPullParser` and
   :func:`~xml.etree.ElementTree.iterparse` now support the *target* parameter.
   The reported object is the value returned by the corresponding method of

Lib/idlelib/config.py

@@ -25,7 +25,7 @@
 """
 # TODOs added Oct 2014, tjr
 
-from configparser import ConfigParser
+from configparser import ConfigParser, Error as ConfigParserError
 import os
 import sys
 
@@ -74,7 +74,7 @@ def GetOptionList(self, section):
     def Load(self):
         "Load the configuration file from disk."
         if self.file and os.path.exists(self.file):
-            with open(self.file, encoding='utf-8', errors='replace') as f:
+            with open(self.file, encoding='utf-8') as f:
                 self.read_file(f)
 
 class IdleUserConfParser(IdleConfParser):
@@ -159,6 +159,7 @@ def __init__(self, _utest=False):
         self.defaultCfg = {}
         self.userCfg = {}
         self.cfg = {}  # TODO use to select userCfg vs defaultCfg
+        self.file_load_errors = []  # (file, error) for unparsable cfg files.
 
         # See https://bugs.python.org/issue4630#msg356516 for following.
         # self.blink_off_time = <first editor text>['insertofftime']
@@ -795,7 +796,28 @@ def LoadCfgFiles(self):
         "Load all configuration files."
         for key in self.defaultCfg:
             self.defaultCfg[key].Load()
-            self.userCfg[key].Load() #same keys
+            try:
+                self.userCfg[key].Load()  # same keys
+            except (ConfigParserError, UnicodeDecodeError) as err:
+                # Move an invalid user file aside instead of losing it
+                # or failing to start (gh-66172).
+                file = self.userCfg[key].file
+                self.file_load_errors.append((file, err))
+                try:
+                    os.replace(file, file + '.bad')
+                except OSError:
+                    pass
+
+    def file_load_error_message(self):
+        "Return a warning about invalid config files, or None."
+        if not self.file_load_errors:
+            return None
+        files = '\n'.join(
+            f'  {file}:\n    {type(err).__name__}: {str(err).splitlines()[0]}'
+            for file, err in self.file_load_errors)
+        return ('The following IDLE configuration files could not be read.  '
+                'They were renamed by appending ".bad", and default settings '
+                'are used instead:\n\n' + files)
 
     def SaveUserCfgFiles(self):
         "Write all loaded user configuration files to disk."

Lib/idlelib/idle_test/test_config.py

@@ -312,6 +312,49 @@ def test_load_cfg_files(self):
         eq(conf.userCfg['foo'].Get('Foo Bar', 'foo'), 'newbar')
         eq(conf.userCfg['foo'].GetOptionList('Foo Bar'), ['foo'])
 
+    def test_load_cfg_files_bad_format(self):
+        # gh-66172: rename an unparsable user file and save the exception.
+        conf = self.new_config(_utest=True)
+        tmpdir = tempfile.TemporaryDirectory()
+        self.addCleanup(tmpdir.cleanup)
+        confpath = os.path.join(tmpdir.name, 'config-extensions.cfg')
+        with open(confpath, 'w') as f:
+            f.write('enable=1\n')  # No section header.
+        conf.defaultCfg['foo'] = config.IdleConfParser('')  # Empty, valid.
+        conf.userCfg['foo'] = config.IdleUserConfParser(confpath)
+
+        self.assertIsNone(conf.file_load_error_message())
+        conf.LoadCfgFiles()  # Must not raise.
+
+        self.assertEqual(len(conf.file_load_errors), 1)
+        file, err = conf.file_load_errors[0]
+        self.assertEqual(file, confpath)
+        # The bad file is moved aside, not left to be overwritten or deleted.
+        self.assertFalse(os.path.exists(confpath))
+        with open(confpath + '.bad') as f:
+            self.assertEqual(f.read(), 'enable=1\n')
+        message = conf.file_load_error_message()
+        self.assertIn(confpath, message)
+        self.assertIn('MissingSectionHeaderError', message)
+
+    def test_load_cfg_files_bad_encoding(self):
+        # gh-66172: a file that is not valid UTF-8 is handled like a bad parse.
+        conf = self.new_config(_utest=True)
+        tmpdir = tempfile.TemporaryDirectory()
+        self.addCleanup(tmpdir.cleanup)
+        confpath = os.path.join(tmpdir.name, 'config-main.cfg')
+        with open(confpath, 'wb') as f:
+            f.write(b'[Section]\nkey = \xff\n')  # Invalid UTF-8.
+        conf.defaultCfg['foo'] = config.IdleConfParser('')  # Empty, valid.
+        conf.userCfg['foo'] = config.IdleUserConfParser(confpath)
+
+        conf.LoadCfgFiles()  # Must not raise.
+
+        self.assertEqual(len(conf.file_load_errors), 1)
+        self.assertIsInstance(conf.file_load_errors[0][1], UnicodeDecodeError)
+        self.assertFalse(os.path.exists(confpath))
+        self.assertTrue(os.path.exists(confpath + '.bad'))
+
     def test_save_user_cfg_files(self):
         conf = self.mock_config()
 

Lib/idlelib/pyshell.py

@@ -1612,6 +1612,12 @@ def main():
     root.withdraw()
     fix_scaling(root)
 
+    # Warn about configuration files that could not be parsed (gh-66172).
+    config_error = idleConf.file_load_error_message()
+    if config_error:
+        messagebox.showwarning('IDLE Configuration Warning', config_error,
+                               parent=root)
+
     # set application icon
     icondir = os.path.join(os.path.dirname(__file__), 'Icons')
     if system() == 'Windows':

Lib/test/test_minidom.py

@@ -571,6 +571,145 @@ def testWriteXML(self):
         dom.unlink()
         self.assertEqual(str, domstr)
 
+    def testWriteXMLDocumentFragment(self):
+        dom = parseString('<doc><a b="c"/>text<!--comment--></doc>')
+        frag = dom.createDocumentFragment()
+        for node in list(dom.documentElement.childNodes):
+            frag.appendChild(node)
+        self.assertEqual(frag.toxml(), '<a b="c"/>text<!--comment-->')
+        self.assertEqual(frag.toprettyxml(),
+                         '<a b="c"/>\ntext\n<!--comment-->\n')
+        # the fragment itself does not add a level of indentation
+        writer = io.StringIO()
+        frag.writexml(writer, "  ", "  ", "\n")
+        self.assertEqual(writer.getvalue(),
+                         '  <a b="c"/>\n  text\n  <!--comment-->\n')
+        self.assertEqual(dom.createDocumentFragment().toxml(), '')
+
+    def testWriteXMLNamespaceDeclarations(self):
+        dom = Document()
+        root = dom.appendChild(
+            dom.createElementNS("http://xml.python.org/ns", "p:root"))
+        child = root.appendChild(
+            dom.createElementNS("http://xml.python.org/ns", "p:child"))
+        child.setAttributeNS("http://xml.python.org/ns2", "q:attr", "value")
+        self.assertEqual(dom.documentElement.toxml(),
+                '<p:root xmlns:p="http://xml.python.org/ns">'
+                '<p:child xmlns:q="http://xml.python.org/ns2" '
+                'q:attr="value"/></p:root>')
+        dom.unlink()
+
+    def testWriteXMLDefaultNamespace(self):
+        dom = Document()
+        root = dom.appendChild(
+            dom.createElementNS("http://xml.python.org/ns", "root"))
+        root.appendChild(
+            dom.createElementNS("http://xml.python.org/ns", "child"))
+        # An element in no namespace undeclares the default namespace.
+        root.appendChild(dom.createElement("nons"))
+        self.assertEqual(dom.documentElement.toxml(),
+                '<root xmlns="http://xml.python.org/ns">'
+                '<child/><nons xmlns=""/></root>')
+        dom.unlink()
+
+    def testWriteXMLAttributeNamespacePrefix(self):
+        dom = Document()
+        root = dom.appendChild(dom.createElement("root"))
+        # Attributes cannot use the default namespace, a prefix is invented.
+        root.setAttributeNS("http://xml.python.org/ns", "attr", "value")
+        root.setAttributeNS("http://xml.python.org/ns2", "attr2", "value2")
+        self.assertEqual(dom.documentElement.toxml(),
+                '<root xmlns:ns0="http://xml.python.org/ns" '
+                'xmlns:ns1="http://xml.python.org/ns2" '
+                'ns0:attr="value" ns1:attr2="value2"/>')
+        # The same namespace gets the same prefix.
+        root.setAttributeNS("http://xml.python.org/ns", "attr3", "value3")
+        self.assertEqual(dom.documentElement.toxml(),
+                '<root xmlns:ns0="http://xml.python.org/ns" '
+                'xmlns:ns1="http://xml.python.org/ns2" '
+                'ns0:attr="value" ns1:attr2="value2" ns0:attr3="value3"/>')
+        dom.unlink()
+
+    def testWriteXMLAttributeNamespacePrefixReused(self):
+        # A prefix already bound to the namespace of the attribute is used.
+        dom = Document()
+        root = dom.appendChild(
+            dom.createElementNS("http://xml.python.org/ns", "p:root"))
+        root.setAttributeNS("http://xml.python.org/ns", "attr", "value")
+        self.assertEqual(dom.documentElement.toxml(),
+                '<p:root xmlns:p="http://xml.python.org/ns" p:attr="value"/>')
+        # The prefix can be bound for an ancestor.
+        child = root.appendChild(dom.createElement("child"))
+        child.setAttributeNS("http://xml.python.org/ns", "attr", "value")
+        self.assertEqual(child.toxml(), '<child p:attr="value"/>')
+        # The prefix bound for a preceding attribute is reused.
+        root.setAttributeNS("http://xml.python.org/ns3", "q:attr3", "value3")
+        root.setAttributeNS("http://xml.python.org/ns3", "attr4", "value4")
+        self.assertEqual(dom.documentElement.toxml(),
+                '<p:root xmlns:p="http://xml.python.org/ns" '
+                'xmlns:q="http://xml.python.org/ns3" '
+                'p:attr="value" q:attr3="value3" q:attr4="value4">'
+                '<child p:attr="value"/></p:root>')
+        root.removeAttributeNS("http://xml.python.org/ns3", "attr3")
+        root.removeAttributeNS("http://xml.python.org/ns3", "attr4")
+        # The prefix must not be taken by an explicit declaration.
+        root.setAttributeNS(xml.dom.XMLNS_NAMESPACE, "xmlns:ns0", "other")
+        root.setAttributeNS("http://xml.python.org/ns2", "attr2", "value2")
+        self.assertEqual(dom.documentElement.toxml(),
+                '<p:root xmlns:p="http://xml.python.org/ns" '
+                'xmlns:ns1="http://xml.python.org/ns2" '
+                'p:attr="value" xmlns:ns0="other" ns1:attr2="value2">'
+                '<child p:attr="value"/></p:root>')
+        dom.unlink()
+
+    def testWriteXMLXMLPrefix(self):
+        dom = Document()
+        root = dom.appendChild(dom.createElement("root"))
+        # The "xml" prefix is bound by definition and is never declared.
+        root.setAttributeNS(xml.dom.XML_NAMESPACE, "xml:lang", "en")
+        self.assertEqual(dom.documentElement.toxml(), '<root xml:lang="en"/>')
+        dom.unlink()
+
+    def testWriteXMLExistingNamespaceDeclarations(self):
+        for str in [
+            '<p:root xmlns:p="http://xml.python.org/ns"><p:child/></p:root>',
+            '<root xmlns="http://xml.python.org/ns"><child xmlns=""/></root>',
+            '<p:root xmlns:p="http://xml.python.org/ns">'
+                '<p:child xmlns:p="http://xml.python.org/ns2"/></p:root>',
+            '<root xmlns:p="http://xml.python.org/ns" p:attr="value"/>',
+        ]:
+            with self.subTest(str=str):
+                dom = parseString(str)
+                self.assertEqual(dom.documentElement.toxml(), str)
+                dom.unlink()
+
+    def testWriteXMLNotANamespaceDeclaration(self):
+        # an attribute whose name only starts with "xmlns" is not one
+        dom = parseString('<root xmlns="http://xml.python.org/ns">'
+                          '<child xmlnsabc="v"><g/></child></root>')
+        self.assertEqual(dom.documentElement.toxml(),
+                '<root xmlns="http://xml.python.org/ns">'
+                '<child xmlnsabc="v"><g/></child></root>')
+        dom.unlink()
+
+        dom = Document()
+        root = dom.appendChild(
+            dom.createElementNS("http://xml.python.org/ns", "root"))
+        child = root.appendChild(dom.createElement("child"))
+        child.setAttribute("xmlnsabc", "v")
+        self.assertEqual(dom.documentElement.toxml(),
+                '<root xmlns="http://xml.python.org/ns">'
+                '<child xmlns="" xmlnsabc="v"/></root>')
+        dom.unlink()
+
+    def testWriteXMLDoesNotModifyDocument(self):
+        dom = Document()
+        root = dom.appendChild(
+            dom.createElementNS("http://xml.python.org/ns", "p:root"))
+        root.toxml()
+        self.assertEqual(root.attributes.length, 0)
+        dom.unlink()
+
     def test_toxml_quote_text(self):
         dom = Document()
         elem = dom.appendChild(dom.createElement('elem'))

Lib/test/test_nturl2path.py

@@ -1,7 +1,9 @@
+import os
 import sys
 import unittest
 import urllib.parse
 
+from test.support import os_helper
 from test.support import warnings_helper
 
 
@@ -36,7 +38,6 @@ def test_pathname2url(self):
         self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/')
         self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c')
         self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c')
-        self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9')
         self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo")
         # NTFS alternate data streams
         self.assertEqual(fn('C:\\foo:bar'), '///C:/foo%3Abar')
@@ -47,7 +48,7 @@ def test_pathname2url(self):
         self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/')
         self.assertEqual(fn('\\\\some\\share\\'), '//some/share/')
         self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c')
-        self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9')
+        self.assertEqual(fn('\\\\some\\share\\a\\b%#c'), '//some/share/a/b%25%23c')
         # Alternate path separator
         self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c')
         self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c')
@@ -60,14 +61,28 @@ def test_pathname2url(self):
         for url in urls:
             self.assertEqual(fn(nturl2path.url2pathname(url)), url)
 
+    @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII')
+    def test_pathname2url_nonascii(self):
+        encoding = sys.getfilesystemencoding()
+        errors = sys.getfilesystemencodeerrors()
+        char = os_helper.FS_NONASCII
+        quoted = urllib.parse.quote(char, encoding=encoding, errors=errors)
+        self.assertEqual(nturl2path.pathname2url(f'C:\\a\\b{char}'),
+                         '///C:/a/b' + quoted)
+        self.assertEqual(nturl2path.pathname2url(f'\\\\some\\share\\a\\b{char}'),
+                         '//some/share/a/b' + quoted)
+
+    @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE,
+                         'need os_helper.TESTFN_UNDECODABLE')
     def test_pathname2url_surrogates(self):
         # gh-156713: the filesystem encoding and error handler are used,
         # so that paths containing surrogate characters can be converted.
         encoding = sys.getfilesystemencoding()
         errors = sys.getfilesystemencodeerrors()
-        tail = urllib.parse.quote('a\udcff', encoding=encoding, errors=errors)
-        self.assertEqual(nturl2path.pathname2url('C:\\a\udcff'),
-                         '///C:/' + tail)
+        path = os.fsdecode(os_helper.TESTFN_UNDECODABLE)
+        url = urllib.parse.quote(path, encoding=encoding, errors=errors)
+        self.assertEqual(nturl2path.pathname2url('C:\\' + path),
+                         '///C:/' + url)
 
     def test_url2pathname(self):
         fn = nturl2path.url2pathname
@@ -114,14 +129,17 @@ def test_url2pathname(self):
             self.assertEqual(fn(nturl2path.pathname2url(path)), path)
 
 
+    @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE,
+                         'need os_helper.TESTFN_UNDECODABLE')
     def test_url2pathname_surrogates(self):
         # gh-156713: the filesystem encoding and error handler are used, so
         # that URLs containing percent-encoded surrogates can be converted.
         encoding = sys.getfilesystemencoding()
         errors = sys.getfilesystemencodeerrors()
-        url = urllib.parse.quote('a\udcff', encoding=encoding, errors=errors)
+        path = os.fsdecode(os_helper.TESTFN_UNDECODABLE)
+        url = urllib.parse.quote(path, encoding=encoding, errors=errors)
         self.assertEqual(nturl2path.url2pathname('///C:/' + url),
-                         'C:\\a\udcff')
+                         'C:\\' + path)
 
 
 if __name__ == '__main__':