diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index c29cf4bf8fe..c66269a5719 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -47,7 +47,7 @@ class _Printer(object): data = None for filename in self.__filenames: try: - with open(filename, "r") as fp: + with open(filename, encoding='utf-8') as fp: data = fp.read() break except OSError: diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 411aa53a0b6..555f598de7a 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1150,6 +1150,8 @@ class FileHandler(StreamHandler): self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + if "b" not in mode: + self.encoding = io.text_encoding(encoding) self.errors = errors self.delay = delay # bpo-26789: FileHandler keeps a reference to the builtin open() @@ -2022,8 +2024,10 @@ def basicConfig(**kwargs): filename = kwargs.pop("filename", None) mode = kwargs.pop("filemode", 'a') if filename: - if 'b'in mode: + if 'b' in mode: errors = None + else: + encoding = io.text_encoding(encoding) h = FileHandler(filename, mode, encoding=encoding, errors=errors) else: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 867ef4ebc76..b0d5885989e 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -23,7 +23,7 @@ Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ -import logging, socket, os, pickle, struct, time, re +import io, logging, socket, os, pickle, struct, time, re from stat import ST_DEV, ST_INO, ST_MTIME import queue import threading @@ -150,6 +150,8 @@ class RotatingFileHandler(BaseRotatingHandler): # on each run. if maxBytes > 0: mode = 'a' + if "b" not in mode: + encoding = io.text_encoding(encoding) BaseRotatingHandler.__init__(self, filename, mode, encoding=encoding, delay=delay, errors=errors) self.maxBytes = maxBytes @@ -205,6 +207,7 @@ class TimedRotatingFileHandler(BaseRotatingHandler): def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None, errors=None): + encoding = io.text_encoding(encoding) BaseRotatingHandler.__init__(self, filename, 'a', encoding=encoding, delay=delay, errors=errors) self.when = when.upper() @@ -442,6 +445,8 @@ class WatchedFileHandler(logging.FileHandler): """ def __init__(self, filename, mode='a', encoding=None, delay=False, errors=None): + if "b" not in mode: + encoding = io.text_encoding(encoding) logging.FileHandler.__init__(self, filename, mode=mode, encoding=encoding, delay=delay, errors=errors) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 72feaedbe75..35ad0b93e7d 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -363,7 +363,7 @@ class GetSourceBase(unittest.TestCase): fodderModule = None def setUp(self): - with open(inspect.getsourcefile(self.fodderModule)) as fp: + with open(inspect.getsourcefile(self.fodderModule), encoding="utf-8") as fp: self.source = fp.read() def sourcerange(self, top, bottom): @@ -773,8 +773,8 @@ class TestNoEOL(GetSourceBase): def setUp(self): self.tempdir = TESTFN + '_dir' os.mkdir(self.tempdir) - with open(os.path.join(self.tempdir, - 'inspect_fodder3%spy' % os.extsep), 'w') as f: + with open(os.path.join(self.tempdir, 'inspect_fodder3%spy' % os.extsep), + 'w', encoding='utf-8') as f: f.write("class X:\n pass # No EOL") with DirsOnSysPath(self.tempdir): import inspect_fodder3 as mod3 @@ -1805,7 +1805,7 @@ class TestGetattrStatic(unittest.TestCase): def test_no_dict_no_slots_instance_member(self): # returns descriptor - with open(__file__) as handle: + with open(__file__, encoding='utf-8') as handle: self.assertEqual(inspect.getattr_static(handle, 'name'), type(handle).name) def test_inherited_slots(self): @@ -4045,7 +4045,7 @@ def foo(): def assertInspectEqual(self, path, source): inspected_src = inspect.getsource(source) - with open(path) as src: + with open(path, encoding='utf-8') as src: self.assertEqual( src.read().splitlines(True), inspected_src.splitlines(True) @@ -4056,7 +4056,7 @@ def foo(): with _ready_to_import('reload_bug', self.src_before) as (name, path): module = importlib.import_module(name) self.assertInspectEqual(path, module) - with open(path, 'w') as src: + with open(path, 'w', encoding='utf-8') as src: src.write(self.src_after) self.assertInspectEqual(path, module) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 8d44d497bd2..6266aa7c33b 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -14,11 +14,11 @@ def _captured_script(script): indented = script.replace('\n', '\n ') wrapped = dedent(f""" import contextlib - with open({w}, 'w') as spipe: + with open({w}, 'w', encoding='utf-8') as spipe: with contextlib.redirect_stdout(spipe): {indented} """) - return wrapped, open(r) + return wrapped, open(r, encoding='utf-8') def clean_up_interpreters(): @@ -411,7 +411,7 @@ class TestInterpreterRun(TestBase): def test_fork(self): interp = interpreters.create() import tempfile - with tempfile.NamedTemporaryFile('w+') as file: + with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: file.write('') file.flush() @@ -421,7 +421,7 @@ class TestInterpreterRun(TestBase): try: os.fork() except RuntimeError: - with open('{file.name}', 'w') as out: + with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) interp.run(script) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index d493b19525a..48a3cca3fbd 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2601,8 +2601,8 @@ class TextIOWrapperTest(unittest.TestCase): self.assertEqual(t.encoding, "utf-8") self.assertEqual(t.line_buffering, True) self.assertEqual("\xe9\n", t.readline()) - self.assertRaises(TypeError, t.__init__, b, newline=42) - self.assertRaises(ValueError, t.__init__, b, newline='xyzzy') + self.assertRaises(TypeError, t.__init__, b, encoding="utf-8", newline=42) + self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') def test_uninitialized(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) @@ -3732,7 +3732,7 @@ class CTextIOWrapperTest(TextIOWrapperTest): r = self.BytesIO(b"\xc3\xa9\n\n") b = self.BufferedReader(r, 1000) t = self.TextIOWrapper(b, encoding="utf-8") - self.assertRaises(ValueError, t.__init__, b, newline='xyzzy') + self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') self.assertRaises(ValueError, t.read) t = self.TextIOWrapper.__new__(self.TextIOWrapper) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index a7658b5f476..20cb9c06809 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -337,13 +337,13 @@ class TestCase(unittest.TestCase): # Test a file def test_iter_file(self): - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: for i in range(5): f.write("%d\n" % i) finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: self.check_for_loop(f, ["0\n", "1\n", "2\n", "3\n", "4\n"], pickle=False) self.check_for_loop(f, [], pickle=False) @@ -366,13 +366,13 @@ class TestCase(unittest.TestCase): self.assertRaises(TypeError, list, list) self.assertRaises(TypeError, list, 42) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: for i in range(5): f.write("%d\n" % i) finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: self.assertEqual(list(f), ["0\n", "1\n", "2\n", "3\n", "4\n"]) f.seek(0, 0) @@ -399,13 +399,13 @@ class TestCase(unittest.TestCase): self.assertRaises(TypeError, tuple, list) self.assertRaises(TypeError, tuple, 42) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: for i in range(5): f.write("%d\n" % i) finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: self.assertEqual(tuple(f), ("0\n", "1\n", "2\n", "3\n", "4\n")) f.seek(0, 0) @@ -476,14 +476,14 @@ class TestCase(unittest.TestCase): self.assertEqual(max(d.values()), 3) self.assertEqual(min(iter(d.values())), 1) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: f.write("medium line\n") f.write("xtra large line\n") f.write("itty-bitty line\n") finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: self.assertEqual(min(f), "itty-bitty line\n") f.seek(0, 0) @@ -509,13 +509,13 @@ class TestCase(unittest.TestCase): i < len(d) and dkeys[i] or None) for i in range(3)] - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: for i in range(10): f.write("xy" * i + "\n") # line i has len 2*i+1 finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: self.assertEqual(list(map(len, f)), list(range(1, 21, 2))) finally: @@ -556,12 +556,12 @@ class TestCase(unittest.TestCase): self.i = i+1 return i - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: f.write("a\n" "bbb\n" "cc\n") finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: self.assertEqual(list(zip(IntsFrom(0), f, IntsFrom(-100))), [(0, "a\n", -100), @@ -624,13 +624,13 @@ class TestCase(unittest.TestCase): return "fooled you!" return next(self.it) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: f.write("a\n" + "b\n" + "c\n") finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") # Nasty: string.join(s) can't know whether unicode.join() is needed # until it's seen all of s's elements. But in this case, f's # iterator cannot be restarted. So what we're testing here is @@ -676,12 +676,12 @@ class TestCase(unittest.TestCase): self.assertIn((k, v), d.items()) self.assertNotIn((v, k), d.items()) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: f.write("a\n" "b\n" "c\n") finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: for chunk in "abc": f.seek(0, 0) @@ -713,12 +713,12 @@ class TestCase(unittest.TestCase): self.assertEqual(countOf(d.values(), 2j), 1) self.assertEqual(countOf(d.values(), 1j), 0) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: f.write("a\n" "b\n" "c\n" "b\n") finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: for letter, count in ("a", 1), ("b", 2), ("c", 1), ("d", 0): f.seek(0, 0) @@ -748,12 +748,12 @@ class TestCase(unittest.TestCase): self.assertRaises(TypeError, indexOf, indexOf, indexOf) self.assertRaises(ZeroDivisionError, indexOf, BadIterableClass(), 1) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: f.write("a\n" "b\n" "c\n" "d\n" "e\n") finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: fiter = iter(f) self.assertEqual(indexOf(fiter, "b\n"), 1) @@ -774,7 +774,7 @@ class TestCase(unittest.TestCase): # Test iterators with file.writelines(). def test_writelines(self): - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") try: self.assertRaises(TypeError, f.writelines, None) @@ -813,7 +813,7 @@ class TestCase(unittest.TestCase): f.writelines(Whatever(6, 6+2000)) f.close() - f = open(TESTFN) + f = open(TESTFN, encoding="utf-8") expected = [str(i) + "\n" for i in range(1, 2006)] self.assertEqual(list(f), expected) @@ -857,14 +857,14 @@ class TestCase(unittest.TestCase): a, b, c = {1: 42, 2: 42, 3: 42}.values() self.assertEqual((a, b, c), (42, 42, 42)) - f = open(TESTFN, "w") + f = open(TESTFN, "w", encoding="utf-8") lines = ("a\n", "bb\n", "ccc\n") try: for line in lines: f.write(line) finally: f.close() - f = open(TESTFN, "r") + f = open(TESTFN, "r", encoding="utf-8") try: a, b, c = f self.assertEqual((a, b, c), lines) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 5411188acb6..0386690cad9 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -125,7 +125,7 @@ class TestTool(unittest.TestCase): outfile = os_helper.TESTFN + '.out' rc, out, err = assert_python_ok('-m', 'json.tool', infile, outfile) self.addCleanup(os.remove, outfile) - with open(outfile, "r") as fp: + with open(outfile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) self.assertEqual(rc, 0) self.assertEqual(out, b'') diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index cfc6ba89e77..59e00da242a 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -116,7 +116,7 @@ class LineCacheTests(unittest.TestCase): # Check module loading for entry in MODULES: filename = os.path.join(MODULE_PATH, entry) + '.py' - with open(filename) as file: + with open(filename, encoding='utf-8') as file: for index, line in enumerate(file): self.assertEqual(line, getline(filename, index + 1)) @@ -126,7 +126,7 @@ class LineCacheTests(unittest.TestCase): def test_no_ending_newline(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "w") as fp: + with open(os_helper.TESTFN, "w", encoding='utf-8') as fp: fp.write(SOURCE_3) lines = linecache.getlines(os_helper.TESTFN) self.assertEqual(lines, ["\n", "def f():\n", " return 3\n"]) @@ -153,18 +153,18 @@ class LineCacheTests(unittest.TestCase): # Create a source file and cache its contents source_name = os_helper.TESTFN + '.py' self.addCleanup(os_helper.unlink, source_name) - with open(source_name, 'w') as source: + with open(source_name, 'w', encoding='utf-8') as source: source.write(SOURCE_1) getline(source_name, 1) # Keep a copy of the old contents source_list = [] - with open(source_name) as source: + with open(source_name, encoding='utf-8') as source: for index, line in enumerate(source): self.assertEqual(line, getline(source_name, index + 1)) source_list.append(line) - with open(source_name, 'w') as source: + with open(source_name, 'w', encoding='utf-8') as source: source.write(SOURCE_2) # Try to update a bogus cache entry @@ -176,7 +176,7 @@ class LineCacheTests(unittest.TestCase): # Update the cache and check whether it matches the new source file linecache.checkcache(source_name) - with open(source_name) as source: + with open(source_name, encoding='utf-8') as source: for index, line in enumerate(source): self.assertEqual(line, getline(source_name, index + 1)) source_list.append(line) diff --git a/Lib/test/test_lltrace.py b/Lib/test/test_lltrace.py index 5f7d56474f0..8f1a92e5c72 100644 --- a/Lib/test/test_lltrace.py +++ b/Lib/test/test_lltrace.py @@ -13,7 +13,7 @@ class TestLLTrace(unittest.TestCase): # bpo-34113. The crash happened at the command line console of # debug Python builds with __ltrace__ enabled (only possible in console), # when the interal Python stack was negatively adjusted - with open(os_helper.TESTFN, 'w') as fd: + with open(os_helper.TESTFN, 'w', encoding='utf-8') as fd: self.addCleanup(os_helper.unlink, os_helper.TESTFN) fd.write(textwrap.dedent("""\ import code diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 859baa4738b..452ba785234 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -553,7 +553,7 @@ class HandlerTest(BaseTest): os.close(fd) if not existing: os.unlink(fn) - h = logging.handlers.WatchedFileHandler(fn, delay=True) + h = logging.handlers.WatchedFileHandler(fn, encoding='utf-8', delay=True) if existing: dev, ino = h.dev, h.ino self.assertEqual(dev, -1) @@ -616,7 +616,7 @@ class HandlerTest(BaseTest): if sys.platform in ('linux', 'darwin'): cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) for cls, args in cases: - h = cls(*args) + h = cls(*args, encoding="utf-8") self.assertTrue(os.path.exists(fn)) h.close() os.unlink(fn) @@ -645,7 +645,7 @@ class HandlerTest(BaseTest): remover = threading.Thread(target=remove_loop, args=(fn, del_count)) remover.daemon = True remover.start() - h = logging.handlers.WatchedFileHandler(fn, delay=delay) + h = logging.handlers.WatchedFileHandler(fn, encoding='utf-8', delay=delay) f = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s') h.setFormatter(f) try: @@ -677,7 +677,7 @@ class HandlerTest(BaseTest): def __init__(self): super().__init__() self.sub_handler = logging.StreamHandler( - stream=open('/dev/null', 'wt')) + stream=open('/dev/null', 'wt', encoding='utf-8')) def emit(self, record): self.sub_handler.acquire() @@ -4355,7 +4355,7 @@ class ModuleLevelMiscTest(BaseTest): # basicConfig() opens the file, but logging.shutdown() closes # it at Python exit. When A.__del__() is called, # FileHandler._open() must be called again to re-open the file. - logging.basicConfig(filename={filename!r}) + logging.basicConfig(filename={filename!r}, encoding="utf-8") a = A() @@ -4365,7 +4365,7 @@ class ModuleLevelMiscTest(BaseTest): """) assert_python_ok("-c", code) - with open(filename) as fp: + with open(filename, encoding="utf-8") as fp: self.assertEqual(fp.read().rstrip(), "ERROR:root:log in __del__") def test_recursion_error(self): @@ -4557,13 +4557,13 @@ class BasicConfigTest(unittest.TestCase): h2.close() os.remove(fn) - logging.basicConfig(filename='test.log') + logging.basicConfig(filename='test.log', encoding='utf-8') self.assertEqual(len(logging.root.handlers), 1) handler = logging.root.handlers[0] self.assertIsInstance(handler, logging.FileHandler) - expected = logging.FileHandler('test.log', 'a') + expected = logging.FileHandler('test.log', 'a', encoding='utf-8') self.assertEqual(handler.stream.mode, expected.stream.mode) self.assertEqual(handler.stream.name, expected.stream.name) self.addCleanup(cleanup, handler, expected, 'test.log') @@ -5161,7 +5161,7 @@ class BaseFileTest(BaseTest): class FileHandlerTest(BaseFileTest): def test_delay(self): os.unlink(self.fn) - fh = logging.FileHandler(self.fn, delay=True) + fh = logging.FileHandler(self.fn, encoding='utf-8', delay=True) self.assertIsNone(fh.stream) self.assertFalse(os.path.exists(self.fn)) fh.handle(logging.makeLogRecord({})) @@ -5176,19 +5176,20 @@ class RotatingFileHandlerTest(BaseFileTest): def test_should_not_rollover(self): # If maxbytes is zero rollover never occurs - rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=0) + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=0) self.assertFalse(rh.shouldRollover(None)) rh.close() def test_should_rollover(self): - rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=1) + rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=1) self.assertTrue(rh.shouldRollover(self.next_rec())) rh.close() def test_file_created(self): # checks that the file is created and assumes it was created # by us - rh = logging.handlers.RotatingFileHandler(self.fn) + rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8") rh.emit(self.next_rec()) self.assertLogFile(self.fn) rh.close() @@ -5197,7 +5198,7 @@ class RotatingFileHandlerTest(BaseFileTest): def namer(name): return name + ".test" rh = logging.handlers.RotatingFileHandler( - self.fn, backupCount=2, maxBytes=1) + self.fn, encoding="utf-8", backupCount=2, maxBytes=1) rh.namer = namer rh.emit(self.next_rec()) self.assertLogFile(self.fn) @@ -5218,7 +5219,7 @@ class RotatingFileHandlerTest(BaseFileTest): os.rename(source, dest + ".rotated") rh = HandlerWithNamerAndRotator( - self.fn, backupCount=2, maxBytes=1) + self.fn, encoding="utf-8", backupCount=2, maxBytes=1) self.assertEqual(rh.namer(self.fn), self.fn + ".test") rh.emit(self.next_rec()) self.assertLogFile(self.fn) @@ -5241,7 +5242,7 @@ class RotatingFileHandlerTest(BaseFileTest): os.remove(source) rh = logging.handlers.RotatingFileHandler( - self.fn, backupCount=2, maxBytes=1) + self.fn, encoding="utf-8", backupCount=2, maxBytes=1) rh.rotator = rotator rh.namer = namer m1 = self.next_rec()