mirrors/cpython
mirrors/cpython
0
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2024-11-21 12:59:38 +01:00
Code Issues Releases Activity
main
cpython/Lib/test/test_mmap.py
Michał Górny 342e654b8e
gh-124213: Skip tests failing inside systemd-nspawn --suppress-sync=true (#124215) 
Add a helper function that checks whether the test suite is running
inside a systemd-nspawn container, and skip the few tests failing
with `--suppress-sync=true` in that case.  The tests are failing because
`--suppress-sync=true` stubs out `fsync()`, `fdatasync()` and `msync()`
calls, and therefore they always return success without checking for
invalid arguments.

Call `os.open(__file__, os.O_RDONLY | os.O_SYNC)` and check the errno to
detect whether `--suppress-sync=true` is actually used, and skip
the tests only in that scenario.
2024-09-20 13:37:49 +00:00

1203 lines
44 KiB
Python
Raw Permalink Blame History

from test.support import (
requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple,
in_systemd_nspawn_sync_suppressed,
)
from test.support.import_helper import import_module
from test.support.os_helper import TESTFN, unlink
from test.support.script_helper import assert_python_ok
import unittest
import errno
import os
import re
import itertools
import random
import socket
import string
import sys
import textwrap
import weakref
# Skip test if we can't import mmap.
mmap = import_module('mmap')
PAGESIZE = mmap.PAGESIZE
tagname_prefix = f'python_{os.getpid()}_test_mmap'
def random_tagname(length=10):
suffix = ''.join(random.choices(string.ascii_uppercase, k=length))
return f'{tagname_prefix}_{suffix}'
# Python's mmap module dup()s the file descriptor. Emscripten's FS layer
# does not materialize file changes through a dupped fd to a new mmap.
if is_emscripten:
raise unittest.SkipTest("incompatible with Emscripten's mmap emulation.")
class MmapTests(unittest.TestCase):
def setUp(self):
if os.path.exists(TESTFN):
os.unlink(TESTFN)
def tearDown(self):
try:
os.unlink(TESTFN)
except OSError:
pass
def test_basic(self):
# Test mmap module on Unix systems and Windows
# Create a file to be mmap'ed.
f = open(TESTFN, 'bw+')
try:
# Write 2 pages worth of data to the file
f.write(b'\0'* PAGESIZE)
f.write(b'foo')
f.write(b'\0'* (PAGESIZE-3) )
f.flush()
m = mmap.mmap(f.fileno(), 2 * PAGESIZE)
finally:
f.close()
# Simple sanity checks
tp = str(type(m)) # SF bug 128713: segfaulted on Linux
self.assertEqual(m.find(b'foo'), PAGESIZE)
self.assertEqual(len(m), 2*PAGESIZE)
self.assertEqual(m[0], 0)
self.assertEqual(m[0:3], b'\0\0\0')
# Shouldn't crash on boundary (Issue #5292)
self.assertRaises(IndexError, m.__getitem__, len(m))
self.assertRaises(IndexError, m.__setitem__, len(m), b'\0')
# Modify the file's content
m[0] = b'3'[0]
m[PAGESIZE +3: PAGESIZE +3+3] = b'bar'
# Check that the modification worked
self.assertEqual(m[0], b'3'[0])
self.assertEqual(m[0:3], b'3\0\0')
self.assertEqual(m[PAGESIZE-1 : PAGESIZE + 7], b'\0foobar\0')
m.flush()
# Test doing a regular expression match in an mmap'ed file
match = re.search(b'[A-Za-z]+', m)
if match is None:
self.fail('regex match on mmap failed!')
else:
start, end = match.span(0)
length = end - start
self.assertEqual(start, PAGESIZE)
self.assertEqual(end, PAGESIZE + 6)
# test seeking around (try to overflow the seek implementation)
self.assertTrue(m.seekable())
self.assertEqual(m.seek(0, 0), 0)
self.assertEqual(m.tell(), 0)
self.assertEqual(m.seek(42, 1), 42)
self.assertEqual(m.tell(), 42)
self.assertEqual(m.seek(0, 2), len(m))
self.assertEqual(m.tell(), len(m))
# Try to seek to negative position...
self.assertRaises(ValueError, m.seek, -1)
# Try to seek beyond end of mmap...
self.assertRaises(ValueError, m.seek, 1, 2)
# Try to seek to negative position...
self.assertRaises(ValueError, m.seek, -len(m)-1, 2)
# Try resizing map
try:
m.resize(512)
except SystemError:
# resize() not supported
# No messages are printed, since the output of this test suite
# would then be different across platforms.
pass
else:
# resize() is supported
self.assertEqual(len(m), 512)
# Check that we can no longer seek beyond the new size.
self.assertRaises(ValueError, m.seek, 513, 0)
# Check that the underlying file is truncated too
# (bug #728515)
f = open(TESTFN, 'rb')
try:
f.seek(0, 2)
self.assertEqual(f.tell(), 512)
finally:
f.close()
self.assertEqual(m.size(), 512)
m.close()
def test_access_parameter(self):
# Test for "access" keyword parameter
mapsize = 10
with open(TESTFN, "wb") as fp:
fp.write(b"a"*mapsize)
with open(TESTFN, "rb") as f:
m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_READ)
self.assertEqual(m[:], b'a'*mapsize, "Readonly memory map data incorrect.")
# Ensuring that readonly mmap can't be slice assigned
try:
m[:] = b'b'*mapsize
except TypeError:
pass
else:
self.fail("Able to write to readonly memory map")
# Ensuring that readonly mmap can't be item assigned
try:
m[0] = b'b'
except TypeError:
pass
else:
self.fail("Able to write to readonly memory map")
# Ensuring that readonly mmap can't be write() to
try:
m.seek(0, 0)
m.write(b'abc')
except TypeError:
pass
else:
self.fail("Able to write to readonly memory map")
# Ensuring that readonly mmap can't be write_byte() to
try:
m.seek(0, 0)
m.write_byte(b'd')
except TypeError:
pass
else:
self.fail("Able to write to readonly memory map")
# Ensuring that readonly mmap can't be resized
try:
m.resize(2*mapsize)
except SystemError: # resize is not universally supported
pass
except TypeError:
pass
else:
self.fail("Able to resize readonly memory map")
with open(TESTFN, "rb") as fp:
self.assertEqual(fp.read(), b'a'*mapsize,
"Readonly memory map data file was modified")
# Opening mmap with size too big
with open(TESTFN, "r+b") as f:
try:
m = mmap.mmap(f.fileno(), mapsize+1)
except ValueError:
# we do not expect a ValueError on Windows
# CAUTION: This also changes the size of the file on disk, and
# later tests assume that the length hasn't changed. We need to
# repair that.
if sys.platform.startswith('win'):
self.fail("Opening mmap with size+1 should work on Windows.")
else:
# we expect a ValueError on Unix, but not on Windows
if not sys.platform.startswith('win'):
self.fail("Opening mmap with size+1 should raise ValueError.")
m.close()
if sys.platform.startswith('win'):
# Repair damage from the resizing test.
with open(TESTFN, 'r+b') as f:
f.truncate(mapsize)
# Opening mmap with access=ACCESS_WRITE
with open(TESTFN, "r+b") as f:
m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_WRITE)
# Modifying write-through memory map
m[:] = b'c'*mapsize
self.assertEqual(m[:], b'c'*mapsize,
"Write-through memory map memory not updated properly.")
m.flush()
m.close()
with open(TESTFN, 'rb') as f:
stuff = f.read()
self.assertEqual(stuff, b'c'*mapsize,
"Write-through memory map data file not updated properly.")
# Opening mmap with access=ACCESS_COPY
with open(TESTFN, "r+b") as f:
m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_COPY)
# Modifying copy-on-write memory map
m[:] = b'd'*mapsize
self.assertEqual(m[:], b'd' * mapsize,
"Copy-on-write memory map data not written correctly.")
m.flush()
with open(TESTFN, "rb") as fp:
self.assertEqual(fp.read(), b'c'*mapsize,
"Copy-on-write test data file should not be modified.")
# Ensuring copy-on-write maps cannot be resized
self.assertRaises(TypeError, m.resize, 2*mapsize)
m.close()
# Ensuring invalid access parameter raises exception
with open(TESTFN, "r+b") as f:
self.assertRaises(ValueError, mmap.mmap, f.fileno(), mapsize, access=4)
if os.name == "posix":
# Try incompatible flags, prot and access parameters.
with open(TESTFN, "r+b") as f:
self.assertRaises(ValueError, mmap.mmap, f.fileno(), mapsize,
flags=mmap.MAP_PRIVATE,
prot=mmap.PROT_READ, access=mmap.ACCESS_WRITE)
# Try writing with PROT_EXEC and without PROT_WRITE
prot = mmap.PROT_READ | getattr(mmap, 'PROT_EXEC', 0)
with open(TESTFN, "r+b") as f:
try:
m = mmap.mmap(f.fileno(), mapsize, prot=prot)
except PermissionError:
# on macOS 14, PROT_READ | PROT_EXEC is not allowed
pass
else:
self.assertRaises(TypeError, m.write, b"abcdef")
self.assertRaises(TypeError, m.write_byte, 0)
m.close()
@unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows')
def test_trackfd_parameter(self):
size = 64
with open(TESTFN, "wb") as f:
f.write(b"a"*size)
for close_original_fd in True, False:
with self.subTest(close_original_fd=close_original_fd):
with open(TESTFN, "r+b") as f:
with mmap.mmap(f.fileno(), size, trackfd=False) as m:
if close_original_fd:
f.close()
self.assertEqual(len(m), size)
with self.assertRaises(OSError) as err_cm:
m.size()
self.assertEqual(err_cm.exception.errno, errno.EBADF)
with self.assertRaises(ValueError):
m.resize(size * 2)
with self.assertRaises(ValueError):
m.resize(size // 2)
self.assertEqual(m.closed, False)
# Smoke-test other API
m.write_byte(ord('X'))
m[2] = ord('Y')
m.flush()
with open(TESTFN, "rb") as f:
self.assertEqual(f.read(4), b'XaYa')
self.assertEqual(m.tell(), 1)
m.seek(0)
self.assertEqual(m.tell(), 0)
self.assertEqual(m.read_byte(), ord('X'))
self.assertEqual(m.closed, True)
self.assertEqual(os.stat(TESTFN).st_size, size)
@unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows')
def test_trackfd_neg1(self):
size = 64
with mmap.mmap(-1, size, trackfd=False) as m:
with self.assertRaises(OSError):
m.size()
with self.assertRaises(ValueError):
m.resize(size // 2)
self.assertEqual(len(m), size)
m[0] = ord('a')
assert m[0] == ord('a')
@unittest.skipIf(os.name != 'nt', 'trackfd only fails on Windows')
def test_no_trackfd_parameter_on_windows(self):
# 'trackffd' is an invalid keyword argument for this function
size = 64
with self.assertRaises(TypeError):
mmap.mmap(-1, size, trackfd=True)
with self.assertRaises(TypeError):
mmap.mmap(-1, size, trackfd=False)
def test_bad_file_desc(self):
# Try opening a bad file descriptor...
self.assertRaises(OSError, mmap.mmap, -2, 4096)
def test_tougher_find(self):
# Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2,
# searching for data with embedded \0 bytes didn't work.
with open(TESTFN, 'wb+') as f:
data = b'aabaac\x00deef\x00\x00aa\x00'
n = len(data)
f.write(data)
f.flush()
m = mmap.mmap(f.fileno(), n)
for start in range(n+1):
for finish in range(start, n+1):
slice = data[start : finish]
self.assertEqual(m.find(slice), data.find(slice))
self.assertEqual(m.find(slice + b'x'), -1)
m.close()
def test_find_end(self):
# test the new 'end' parameter works as expected
with open(TESTFN, 'wb+') as f:
data = b'one two ones'
n = len(data)
f.write(data)
f.flush()
m = mmap.mmap(f.fileno(), n)
self.assertEqual(m.find(b'one'), 0)
self.assertEqual(m.find(b'ones'), 8)
self.assertEqual(m.find(b'one', 0, -1), 0)
self.assertEqual(m.find(b'one', 1), 8)
self.assertEqual(m.find(b'one', 1, -1), 8)
self.assertEqual(m.find(b'one', 1, -2), -1)
self.assertEqual(m.find(bytearray(b'one')), 0)
for i in range(-n-1, n+1):
for j in range(-n-1, n+1):
for p in [b"o", b"on", b"two", b"ones", b"s"]:
expected = data.find(p, i, j)
self.assertEqual(m.find(p, i, j), expected, (p, i, j))
def test_find_does_not_access_beyond_buffer(self):
try:
flags = mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS
PAGESIZE = mmap.PAGESIZE
PROT_NONE = 0
PROT_READ = mmap.PROT_READ
except AttributeError as e:
raise unittest.SkipTest("mmap flags unavailable") from e
for i in range(0, 2049):
with mmap.mmap(-1, PAGESIZE * (i + 1),
flags=flags, prot=PROT_NONE) as guard:
with mmap.mmap(-1, PAGESIZE * (i + 2048),
flags=flags, prot=PROT_READ) as fm:
fm.find(b"fo", -2)
def test_rfind(self):
# test the new 'end' parameter works as expected
with open(TESTFN, 'wb+') as f:
data = b'one two ones'
n = len(data)
f.write(data)
f.flush()
m = mmap.mmap(f.fileno(), n)
self.assertEqual(m.rfind(b'one'), 8)
self.assertEqual(m.rfind(b'one '), 0)
self.assertEqual(m.rfind(b'one', 0, -1), 8)
self.assertEqual(m.rfind(b'one', 0, -2), 0)
self.assertEqual(m.rfind(b'one', 1, -1), 8)
self.assertEqual(m.rfind(b'one', 1, -2), -1)
self.assertEqual(m.rfind(bytearray(b'one')), 8)
def test_double_close(self):
# make sure a double close doesn't crash on Solaris (Bug# 665913)
with open(TESTFN, 'wb+') as f:
f.write(2**16 * b'a') # Arbitrary character
with open(TESTFN, 'rb') as f:
mf = mmap.mmap(f.fileno(), 2**16, access=mmap.ACCESS_READ)
mf.close()
mf.close()
def test_entire_file(self):
# test mapping of entire file by passing 0 for map length
with open(TESTFN, "wb+") as f:
f.write(2**16 * b'm') # Arbitrary character
with open(TESTFN, "rb+") as f, \
mmap.mmap(f.fileno(), 0) as mf:
self.assertEqual(len(mf), 2**16, "Map size should equal file size.")
self.assertEqual(mf.read(2**16), 2**16 * b"m")
def test_length_0_offset(self):
# Issue #10916: test mapping of remainder of file by passing 0 for
# map length with an offset doesn't cause a segfault.
# NOTE: allocation granularity is currently 65536 under Win64,
# and therefore the minimum offset alignment.
with open(TESTFN, "wb") as f:
f.write((65536 * 2) * b'm') # Arbitrary character
with open(TESTFN, "rb") as f:
with mmap.mmap(f.fileno(), 0, offset=65536, access=mmap.ACCESS_READ) as mf:
self.assertRaises(IndexError, mf.__getitem__, 80000)
def test_length_0_large_offset(self):
# Issue #10959: test mapping of a file by passing 0 for
# map length with a large offset doesn't cause a segfault.
with open(TESTFN, "wb") as f:
f.write(115699 * b'm') # Arbitrary character
with open(TESTFN, "w+b") as f:
self.assertRaises(ValueError, mmap.mmap, f.fileno(), 0,
offset=2147418112)
def test_move(self):
# make move works everywhere (64-bit format problem earlier)
with open(TESTFN, 'wb+') as f:
f.write(b"ABCDEabcde") # Arbitrary character
f.flush()
mf = mmap.mmap(f.fileno(), 10)
mf.move(5, 0, 5)
self.assertEqual(mf[:], b"ABCDEABCDE", "Map move should have duplicated front 5")
mf.close()
# more excessive test
data = b"0123456789"
for dest in range(len(data)):
for src in range(len(data)):
for count in range(len(data) - max(dest, src)):
expected = data[:dest] + data[src:src+count] + data[dest+count:]
m = mmap.mmap(-1, len(data))
m[:] = data
m.move(dest, src, count)
self.assertEqual(m[:], expected)
m.close()
# segfault test (Issue 5387)
m = mmap.mmap(-1, 100)
offsets = [-100, -1, 0, 1, 100]
for source, dest, size in itertools.product(offsets, offsets, offsets):
try:
m.move(source, dest, size)
except ValueError:
pass
offsets = [(-1, -1, -1), (-1, -1, 0), (-1, 0, -1), (0, -1, -1),
(-1, 0, 0), (0, -1, 0), (0, 0, -1)]
for source, dest, size in offsets:
self.assertRaises(ValueError, m.move, source, dest, size)
m.close()
m = mmap.mmap(-1, 1) # single byte
self.assertRaises(ValueError, m.move, 0, 0, 2)
self.assertRaises(ValueError, m.move, 1, 0, 1)
self.assertRaises(ValueError, m.move, 0, 1, 1)
m.move(0, 0, 1)
m.move(0, 0, 0)
def test_anonymous(self):
# anonymous mmap.mmap(-1, PAGE)
m = mmap.mmap(-1, PAGESIZE)
for x in range(PAGESIZE):
self.assertEqual(m[x], 0,
"anonymously mmap'ed contents should be zero")
for x in range(PAGESIZE):
b = x & 0xff
m[x] = b
self.assertEqual(m[x], b)
def test_read_all(self):
m = mmap.mmap(-1, 16)
self.addCleanup(m.close)
# With no parameters, or None or a negative argument, reads all
m.write(bytes(range(16)))
m.seek(0)
self.assertEqual(m.read(), bytes(range(16)))
m.seek(8)
self.assertEqual(m.read(), bytes(range(8, 16)))
m.seek(16)
self.assertEqual(m.read(), b'')
m.seek(3)
self.assertEqual(m.read(None), bytes(range(3, 16)))
m.seek(4)
self.assertEqual(m.read(-1), bytes(range(4, 16)))
m.seek(5)
self.assertEqual(m.read(-2), bytes(range(5, 16)))
m.seek(9)
self.assertEqual(m.read(-42), bytes(range(9, 16)))
def test_read_invalid_arg(self):
m = mmap.mmap(-1, 16)
self.addCleanup(m.close)
self.assertRaises(TypeError, m.read, 'foo')
self.assertRaises(TypeError, m.read, 5.5)
self.assertRaises(TypeError, m.read, [1, 2, 3])
def test_extended_getslice(self):
# Test extended slicing by comparing with list slicing.
s = bytes(reversed(range(256)))
m = mmap.mmap(-1, len(s))
m[:] = s
self.assertEqual(m[:], s)
indices = (0, None, 1, 3, 19, 300, sys.maxsize, -1, -2, -31, -300)
for start in indices:
for stop in indices:
# Skip step 0 (invalid)
for step in indices[1:]:
self.assertEqual(m[start:stop:step],
s[start:stop:step])
def test_extended_set_del_slice(self):
# Test extended slicing by comparing with list slicing.
s = bytes(reversed(range(256)))
m = mmap.mmap(-1, len(s))
indices = (0, None, 1, 3, 19, 300, sys.maxsize, -1, -2, -31, -300)
for start in indices:
for stop in indices:
# Skip invalid step 0
for step in indices[1:]:
m[:] = s
self.assertEqual(m[:], s)
L = list(s)
# Make sure we have a slice of exactly the right length,
# but with different data.
data = L[start:stop:step]
data = bytes(reversed(data))
L[start:stop:step] = data
m[start:stop:step] = data
self.assertEqual(m[:], bytes(L))
def make_mmap_file (self, f, halfsize):
# Write 2 pages worth of data to the file
f.write (b'\0' * halfsize)
f.write (b'foo')
f.write (b'\0' * (halfsize - 3))
f.flush ()
return mmap.mmap (f.fileno(), 0)
def test_empty_file (self):
f = open (TESTFN, 'w+b')
f.close()
with open(TESTFN, "rb") as f :
self.assertRaisesRegex(ValueError,
"cannot mmap an empty file",
mmap.mmap, f.fileno(), 0,
access=mmap.ACCESS_READ)
def test_offset (self):
f = open (TESTFN, 'w+b')
try: # unlink TESTFN no matter what
halfsize = mmap.ALLOCATIONGRANULARITY
m = self.make_mmap_file (f, halfsize)
m.close ()
f.close ()
mapsize = halfsize * 2
# Try invalid offset
f = open(TESTFN, "r+b")
for offset in [-2, -1, None]:
try:
m = mmap.mmap(f.fileno(), mapsize, offset=offset)
self.assertEqual(0, 1)
except (ValueError, TypeError, OverflowError):
pass
else:
self.assertEqual(0, 0)
f.close()
# Try valid offset, hopefully 8192 works on all OSes
f = open(TESTFN, "r+b")
m = mmap.mmap(f.fileno(), mapsize - halfsize, offset=halfsize)
self.assertEqual(m[0:3], b'foo')
f.close()
# Try resizing map
try:
m.resize(512)
except SystemError:
pass
else:
# resize() is supported
self.assertEqual(len(m), 512)
# Check that we can no longer seek beyond the new size.
self.assertRaises(ValueError, m.seek, 513, 0)
# Check that the content is not changed
self.assertEqual(m[0:3], b'foo')
# Check that the underlying file is truncated too
f = open(TESTFN, 'rb')
f.seek(0, 2)
self.assertEqual(f.tell(), halfsize + 512)
f.close()
self.assertEqual(m.size(), halfsize + 512)
m.close()
finally:
f.close()
try:
os.unlink(TESTFN)
except OSError:
pass
def test_subclass(self):
class anon_mmap(mmap.mmap):
def __new__(klass, *args, **kwargs):
return mmap.mmap.__new__(klass, -1, *args, **kwargs)
anon_mmap(PAGESIZE)
@unittest.skipUnless(hasattr(mmap, 'PROT_READ'), "needs mmap.PROT_READ")
def test_prot_readonly(self):
mapsize = 10
with open(TESTFN, "wb") as fp:
fp.write(b"a"*mapsize)
with open(TESTFN, "rb") as f:
m = mmap.mmap(f.fileno(), mapsize, prot=mmap.PROT_READ)
self.assertRaises(TypeError, m.write, "foo")
def test_error(self):
self.assertIs(mmap.error, OSError)
def test_io_methods(self):
data = b"0123456789"
with open(TESTFN, "wb") as fp:
fp.write(b"x"*len(data))
with open(TESTFN, "r+b") as f:
m = mmap.mmap(f.fileno(), len(data))
# Test write_byte()
for i in range(len(data)):
self.assertEqual(m.tell(), i)
m.write_byte(data[i])
self.assertEqual(m.tell(), i+1)
self.assertRaises(ValueError, m.write_byte, b"x"[0])
self.assertEqual(m[:], data)
# Test read_byte()
m.seek(0)
for i in range(len(data)):
self.assertEqual(m.tell(), i)
self.assertEqual(m.read_byte(), data[i])
self.assertEqual(m.tell(), i+1)
self.assertRaises(ValueError, m.read_byte)
# Test read()
m.seek(3)
self.assertEqual(m.read(3), b"345")
self.assertEqual(m.tell(), 6)
# Test write()
m.seek(3)
m.write(b"bar")
self.assertEqual(m.tell(), 6)
self.assertEqual(m[:], b"012bar6789")
m.write(bytearray(b"baz"))
self.assertEqual(m.tell(), 9)
self.assertEqual(m[:], b"012barbaz9")
self.assertRaises(ValueError, m.write, b"ba")
def test_non_ascii_byte(self):
for b in (129, 200, 255): # > 128
m = mmap.mmap(-1, 1)
m.write_byte(b)
self.assertEqual(m[0], b)
m.seek(0)
self.assertEqual(m.read_byte(), b)
m.close()
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_tagname(self):
data1 = b"0123456789"
data2 = b"abcdefghij"
assert len(data1) == len(data2)
tagname1 = random_tagname()
tagname2 = random_tagname()
# Test same tag
m1 = mmap.mmap(-1, len(data1), tagname=tagname1)
m1[:] = data1
m2 = mmap.mmap(-1, len(data2), tagname=tagname1)
m2[:] = data2
self.assertEqual(m1[:], data2)
self.assertEqual(m2[:], data2)
m2.close()
m1.close()
# Test different tag
m1 = mmap.mmap(-1, len(data1), tagname=tagname1)
m1[:] = data1
m2 = mmap.mmap(-1, len(data2), tagname=tagname2)
m2[:] = data2
self.assertEqual(m1[:], data1)
self.assertEqual(m2[:], data2)
m2.close()
m1.close()
with self.assertRaisesRegex(TypeError, 'tagname'):
mmap.mmap(-1, 8, tagname=1)
@cpython_only
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_sizeof(self):
m1 = mmap.mmap(-1, 100)
tagname = random_tagname()
m2 = mmap.mmap(-1, 100, tagname=tagname)
self.assertGreater(sys.getsizeof(m2), sys.getsizeof(m1))
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_crasher_on_windows(self):
# Should not crash (Issue 1733986)
tagname = random_tagname()
m = mmap.mmap(-1, 1000, tagname=tagname)
try:
mmap.mmap(-1, 5000, tagname=tagname)[:] # same tagname, but larger size
except:
pass
m.close()
# Should not crash (Issue 5385)
with open(TESTFN, "wb") as fp:
fp.write(b"x"*10)
f = open(TESTFN, "r+b")
m = mmap.mmap(f.fileno(), 0)
f.close()
try:
m.resize(0) # will raise OSError
except:
pass
try:
m[:]
except:
pass
m.close()
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_invalid_descriptor(self):
# socket file descriptors are valid, but out of range
# for _get_osfhandle, causing a crash when validating the
# parameters to _get_osfhandle.
s = socket.socket()
try:
with self.assertRaises(OSError):
m = mmap.mmap(s.fileno(), 10)
finally:
s.close()
def test_context_manager(self):
with mmap.mmap(-1, 10) as m:
self.assertFalse(m.closed)
self.assertTrue(m.closed)
def test_context_manager_exception(self):
# Test that the OSError gets passed through
with self.assertRaises(Exception) as exc:
with mmap.mmap(-1, 10) as m:
raise OSError
self.assertIsInstance(exc.exception, OSError,
"wrong exception raised in context manager")
self.assertTrue(m.closed, "context manager failed")
def test_weakref(self):
# Check mmap objects are weakrefable
mm = mmap.mmap(-1, 16)
wr = weakref.ref(mm)
self.assertIs(wr(), mm)
del mm
gc_collect()
self.assertIs(wr(), None)
def test_write_returning_the_number_of_bytes_written(self):
mm = mmap.mmap(-1, 16)
self.assertEqual(mm.write(b""), 0)
self.assertEqual(mm.write(b"x"), 1)
self.assertEqual(mm.write(b"yz"), 2)
self.assertEqual(mm.write(b"python"), 6)
def test_resize_past_pos(self):
m = mmap.mmap(-1, 8192)
self.addCleanup(m.close)
m.read(5000)
try:
m.resize(4096)
except SystemError:
self.skipTest("resizing not supported")
self.assertEqual(m.read(14), b'')
self.assertRaises(ValueError, m.read_byte)
self.assertRaises(ValueError, m.write_byte, 42)
self.assertRaises(ValueError, m.write, b'abc')
def test_concat_repeat_exception(self):
m = mmap.mmap(-1, 16)
with self.assertRaises(TypeError):
m + m
with self.assertRaises(TypeError):
m * 2
def test_flush_return_value(self):
# mm.flush() should return None on success, raise an
# exception on error under all platforms.
mm = mmap.mmap(-1, 16)
self.addCleanup(mm.close)
mm.write(b'python')
result = mm.flush()
self.assertIsNone(result)
if (sys.platform.startswith(('linux', 'android'))
and not in_systemd_nspawn_sync_suppressed()):
# 'offset' must be a multiple of mmap.PAGESIZE on Linux.
# See bpo-34754 for details.
self.assertRaises(OSError, mm.flush, 1, len(b'python'))
def test_repr(self):
open_mmap_repr_pat = re.compile(
r"<mmap.mmap closed=False, "
r"access=(?P<access>\S+), "
r"length=(?P<length>\d+), "
r"pos=(?P<pos>\d+), "
r"offset=(?P<offset>\d+)>")
closed_mmap_repr_pat = re.compile(r"<mmap.mmap closed=True>")
mapsizes = (50, 100, 1_000, 1_000_000, 10_000_000)
offsets = tuple((mapsize // 2 // mmap.ALLOCATIONGRANULARITY)
* mmap.ALLOCATIONGRANULARITY for mapsize in mapsizes)
for offset, mapsize in zip(offsets, mapsizes):
data = b'a' * mapsize
length = mapsize - offset
accesses = ('ACCESS_DEFAULT', 'ACCESS_READ',
'ACCESS_COPY', 'ACCESS_WRITE')
positions = (0, length//10, length//5, length//4)
with open(TESTFN, "wb+") as fp:
fp.write(data)
fp.flush()
for access, pos in itertools.product(accesses, positions):
accint = getattr(mmap, access)
with mmap.mmap(fp.fileno(),
length,
access=accint,
offset=offset) as mm:
mm.seek(pos)
match = open_mmap_repr_pat.match(repr(mm))
self.assertIsNotNone(match)
self.assertEqual(match.group('access'), access)
self.assertEqual(match.group('length'), str(length))
self.assertEqual(match.group('pos'), str(pos))
self.assertEqual(match.group('offset'), str(offset))
match = closed_mmap_repr_pat.match(repr(mm))
self.assertIsNotNone(match)
@unittest.skipUnless(hasattr(mmap.mmap, 'madvise'), 'needs madvise')
def test_madvise(self):
size = 2 * PAGESIZE
m = mmap.mmap(-1, size)
with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
m.madvise(mmap.MADV_NORMAL, size)
with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
m.madvise(mmap.MADV_NORMAL, -1)
with self.assertRaisesRegex(ValueError, "madvise length invalid"):
m.madvise(mmap.MADV_NORMAL, 0, -1)
with self.assertRaisesRegex(OverflowError, "madvise length too large"):
m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize)
self.assertEqual(m.madvise(mmap.MADV_NORMAL), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None)
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_resize_up_when_mapped_to_pagefile(self):
"""If the mmap is backed by the pagefile ensure a resize up can happen
and that the original data is still in place
"""
start_size = PAGESIZE
new_size = 2 * start_size
data = bytes(random.getrandbits(8) for _ in range(start_size))
m = mmap.mmap(-1, start_size)
m[:] = data
m.resize(new_size)
self.assertEqual(len(m), new_size)
self.assertEqual(m[:start_size], data[:start_size])
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_resize_down_when_mapped_to_pagefile(self):
"""If the mmap is backed by the pagefile ensure a resize down up can happen
and that a truncated form of the original data is still in place
"""
start_size = PAGESIZE
new_size = start_size // 2
data = bytes(random.getrandbits(8) for _ in range(start_size))
m = mmap.mmap(-1, start_size)
m[:] = data
m.resize(new_size)
self.assertEqual(len(m), new_size)
self.assertEqual(m[:new_size], data[:new_size])
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_resize_fails_if_mapping_held_elsewhere(self):
"""If more than one mapping is held against a named file on Windows, neither
mapping can be resized
"""
start_size = 2 * PAGESIZE
reduced_size = PAGESIZE
f = open(TESTFN, 'wb+')
f.truncate(start_size)
try:
m1 = mmap.mmap(f.fileno(), start_size)
m2 = mmap.mmap(f.fileno(), start_size)
with self.assertRaises(OSError):
m1.resize(reduced_size)
with self.assertRaises(OSError):
m2.resize(reduced_size)
m2.close()
m1.resize(reduced_size)
self.assertEqual(m1.size(), reduced_size)
self.assertEqual(os.stat(f.fileno()).st_size, reduced_size)
finally:
f.close()
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
def test_resize_succeeds_with_error_for_second_named_mapping(self):
"""If a more than one mapping exists of the same name, none of them can
be resized: they'll raise an Exception and leave the original mapping intact
"""
start_size = 2 * PAGESIZE
reduced_size = PAGESIZE
tagname = random_tagname()
data_length = 8
data = bytes(random.getrandbits(8) for _ in range(data_length))
m1 = mmap.mmap(-1, start_size, tagname=tagname)
m2 = mmap.mmap(-1, start_size, tagname=tagname)
m1[:data_length] = data
self.assertEqual(m2[:data_length], data)
with self.assertRaises(OSError):
m1.resize(reduced_size)
self.assertEqual(m1.size(), start_size)
self.assertEqual(m1[:data_length], data)
self.assertEqual(m2[:data_length], data)
def test_mmap_closed_by_int_scenarios(self):
"""
gh-103987: Test that mmap objects raise ValueError
for closed mmap files
"""
class MmapClosedByIntContext:
def __init__(self, access) -> None:
self.access = access
def __enter__(self):
self.f = open(TESTFN, "w+b")
self.f.write(random.randbytes(100))
self.f.flush()
m = mmap.mmap(self.f.fileno(), 100, access=self.access)
class X:
def __index__(self):
m.close()
return 10
return (m, X)
def __exit__(self, exc_type, exc_value, traceback):
self.f.close()
read_access_modes = [
mmap.ACCESS_READ,
mmap.ACCESS_WRITE,
mmap.ACCESS_COPY,
mmap.ACCESS_DEFAULT,
]
write_access_modes = [
mmap.ACCESS_WRITE,
mmap.ACCESS_COPY,
mmap.ACCESS_DEFAULT,
]
for access in read_access_modes:
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m[X()]
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m[X() : 20]
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m[X() : 20 : 2]
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m[20 : X() : -2]
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m.read(X())
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m.find(b"1", 1, X())
for access in write_access_modes:
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m[X() : 20] = b"1" * 10
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m[X() : 20 : 2] = b"1" * 5
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m[20 : X() : -2] = b"1" * 5
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m.move(1, 2, X())
with MmapClosedByIntContext(access) as (m, X):
with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
m.write_byte(X())
@unittest.skipUnless(os.name == 'nt', 'requires Windows')
@unittest.skipUnless(hasattr(mmap.mmap, '_protect'), 'test needs debug build')
def test_access_violations(self):
from test.support.os_helper import TESTFN
code = textwrap.dedent("""
import faulthandler
import mmap
import os
import sys
from contextlib import suppress
# Prevent logging access violations to stderr.
faulthandler.disable()
PAGESIZE = mmap.PAGESIZE
PAGE_NOACCESS = 0x01
with open(sys.argv[1], 'bw+') as f:
f.write(b'A'* PAGESIZE)
f.flush()
m = mmap.mmap(f.fileno(), PAGESIZE)
m._protect(PAGE_NOACCESS, 0, PAGESIZE)
with suppress(OSError):
m.read(PAGESIZE)
assert False, 'mmap.read() did not raise'
with suppress(OSError):
m.read_byte()
assert False, 'mmap.read_byte() did not raise'
with suppress(OSError):
m.readline()
assert False, 'mmap.readline() did not raise'
with suppress(OSError):
m.write(b'A'* PAGESIZE)
assert False, 'mmap.write() did not raise'
with suppress(OSError):
m.write_byte(0)
assert False, 'mmap.write_byte() did not raise'
with suppress(OSError):
m[0] # test mmap_subscript
assert False, 'mmap.__getitem__() did not raise'
with suppress(OSError):
m[0:10] # test mmap_subscript
assert False, 'mmap.__getitem__() did not raise'
with suppress(OSError):
m[0:10:2] # test mmap_subscript
assert False, 'mmap.__getitem__() did not raise'
with suppress(OSError):
m[0] = 1
assert False, 'mmap.__setitem__() did not raise'
with suppress(OSError):
m[0:10] = b'A'* 10
assert False, 'mmap.__setitem__() did not raise'
with suppress(OSError):
m[0:10:2] = b'A'* 5
assert False, 'mmap.__setitem__() did not raise'
with suppress(OSError):
m.move(0, 10, 1)
assert False, 'mmap.move() did not raise'
with suppress(OSError):
list(m) # test mmap_item
assert False, 'mmap.__getitem__() did not raise'
with suppress(OSError):
m.find(b'A')
assert False, 'mmap.find() did not raise'
with suppress(OSError):
m.rfind(b'A')
assert False, 'mmap.rfind() did not raise'
""")
rt, stdout, stderr = assert_python_ok("-c", code, TESTFN)
self.assertEqual(stdout.strip(), b'')
self.assertEqual(stderr.strip(), b'')
class LargeMmapTests(unittest.TestCase):
def setUp(self):
unlink(TESTFN)
def tearDown(self):
unlink(TESTFN)
def _make_test_file(self, num_zeroes, tail):
if sys.platform[:3] == 'win' or is_apple:
requires('largefile',
'test requires %s bytes and a long time to run' % str(0x180000000))
f = open(TESTFN, 'w+b')
try:
f.seek(num_zeroes)
f.write(tail)
f.flush()
except (OSError, OverflowError, ValueError):
try:
f.close()
except (OSError, OverflowError):
pass
raise unittest.SkipTest("filesystem does not have largefile support")
return f
def test_large_offset(self):
with self._make_test_file(0x14FFFFFFF, b" ") as f:
with mmap.mmap(f.fileno(), 0, offset=0x140000000, access=mmap.ACCESS_READ) as m:
self.assertEqual(m[0xFFFFFFF], 32)
def test_large_filesize(self):
with self._make_test_file(0x17FFFFFFF, b" ") as f:
if sys.maxsize < 0x180000000:
# On 32 bit platforms the file is larger than sys.maxsize so
# mapping the whole file should fail -- Issue #16743
with self.assertRaises(OverflowError):
mmap.mmap(f.fileno(), 0x180000000, access=mmap.ACCESS_READ)
with self.assertRaises(ValueError):
mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
with mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ) as m:
self.assertEqual(m.size(), 0x180000000)
# Issue 11277: mmap() with large (~4 GiB) sparse files crashes on OS X.
def _test_around_boundary(self, boundary):
tail = b' DEARdear '
start = boundary - len(tail) // 2
end = start + len(tail)
with self._make_test_file(start, tail) as f:
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as m:
self.assertEqual(m[start:end], tail)
@unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems")
def test_around_2GB(self):
self._test_around_boundary(_2G)
@unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems")
def test_around_4GB(self):
self._test_around_boundary(_4G)
if __name__ == '__main__':
unittest.main()
View Git Blame Copy Permalink