GeezerVison

# HG changeset patch # Parent 5d8042ab3361978b7fd9493867277c286a46cec9 Issue #26720: Release temporary memory views at the end of their life diff -r 5d8042ab3361 Doc/library/codecs.rst --- a/Doc/library/codecs.rst Mon Apr 18 07:16:17 2016 +0000 +++ b/Doc/library/codecs.rst Mon Apr 18 13:23:55 2016 +0000 @@ -469,7 +469,7 @@ character set encoding to a string object. For text encodings and bytes-to-bytes codecs, - *input* must be a bytes object or one which provides the read-only + *input* must be a bytes object or an object which provides the read-only buffer interface -- for example, buffer objects and memory mapped files. The *errors* argument defines the error handling to apply. diff -r 5d8042ab3361 Doc/library/io.rst --- a/Doc/library/io.rst Mon Apr 18 07:16:17 2016 +0000 +++ b/Doc/library/io.rst Mon Apr 18 13:23:55 2016 +0000 @@ -406,6 +406,9 @@ returned if the raw stream is set not to block and no single byte could be readily written to it. + The caller may mutate *b* after this method returns, so the + implementation should only access *b* during the method call. + .. class:: BufferedIOBase diff -r 5d8042ab3361 Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Mon Apr 18 07:16:17 2016 +0000 +++ b/Lib/test/test_codecs.py Mon Apr 18 13:23:55 2016 +0000 @@ -1821,6 +1821,18 @@ self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) + def test_decode_buffer_lifetime(self): + class Codec(codecs.Codec): + def decode(self, input, errors=None): + self.view = input + return ('decoded', 0) + codec = Codec() + info = codecs.CodecInfo(codec.encode, codec.decode) + with support.swap_item(_TEST_CODECS, 'test', info): + b'abc'.decode('test') + with self.assertRaisesRegex(ValueError, 'released memoryview'): + codec.view.tobytes() + class StreamReaderTest(unittest.TestCase): diff -r 5d8042ab3361 Lib/test/test_io.py --- a/Lib/test/test_io.py Mon Apr 18 07:16:17 2016 +0000 +++ b/Lib/test/test_io.py Mon Apr 18 13:23:55 2016 +0000 @@ -1298,6 +1298,18 @@ self.assertRaises(ValueError, b.peek) self.assertRaises(ValueError, b.read1, 1) + def test_read_buffer_lifetime(self): + class Raw(self.MockRawIOWithoutRead): + def readinto(self, b): + self.view = b + b[:7] = b"abcdefg" + return 7 + reader = self.tp(Raw()) + self.assertEqual(reader.read(7), b"abcdefg") + if isinstance(reader.raw.view, memoryview): + with self.assertRaisesRegex(ValueError, "released memoryview"): + reader.raw.view.tobytes() + class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader @@ -1606,6 +1618,18 @@ self.assertRaises(OSError, b.close) # exception not swallowed self.assertTrue(b.closed) + def test_write_buffer_lifetime(self): + class Raw(self.MockRawIO): + def write(self, b): + self.view = b + return len(b) + writer = self.tp(Raw()) + writer.write(b"abc") + writer.flush() + if isinstance(writer.raw.view, memoryview): + with self.assertRaisesRegex(ValueError, "released memoryview"): + writer.raw.view.tobytes() + class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter diff -r 5d8042ab3361 Misc/NEWS --- a/Misc/NEWS Mon Apr 18 07:16:17 2016 +0000 +++ b/Misc/NEWS Mon Apr 18 13:23:55 2016 +0000 @@ -10,6 +10,11 @@ Core and Builtins ----------------- +- Issue #15994: When the interpreter passes a temporary memoryview object to + a Python method, it now releases the memoryview when the method returns. + This affects calls to RawIOBase.readinto() by BufferedReader, + RawIOBase.write() by BufferedWriter, and Codec.decode() by bytes.decode(). + - Issue #26659: Make the builtin slice type support cycle collection. - Issue #26718: super.__init__ no longer leaks memory if called multiple times. diff -r 5d8042ab3361 Modules/_io/bufferedio.c --- a/Modules/_io/bufferedio.c Mon Apr 18 07:16:17 2016 +0000 +++ b/Modules/_io/bufferedio.c Mon Apr 18 13:23:55 2016 +0000 @@ -1474,7 +1474,8 @@ Py_buffer buf; PyObject *memobj, *res; Py_ssize_t n; - /* NOTE: the buffer needn't be released as its object is NULL. */ + PyObject *exc, *val, *tb, *release_res; + /* The buffer will be released when raw.readinto() returns. */ if (PyBuffer_FillInfo(&buf, NULL, start, len, 0, PyBUF_CONTIG) == -1) return -1; memobj = PyMemoryView_FromBuffer(&buf); @@ -1488,7 +1489,15 @@ do { res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readinto, memobj, NULL); } while (res == NULL && _PyIO_trap_eintr()); + PyErr_Fetch(&exc, &val, &tb); + release_res = PyObject_CallMethod(memobj, "release", NULL); + _PyErr_ChainExceptions(exc, val, tb); Py_DECREF(memobj); + if (release_res == NULL) { + Py_XDECREF(res); + return -1; + } + Py_DECREF(release_res); if (res == NULL) return -1; if (res == Py_None) { @@ -1828,7 +1837,8 @@ PyObject *memobj, *res; Py_ssize_t n; int errnum; - /* NOTE: the buffer needn't be released as its object is NULL. */ + PyObject *exc, *val, *tb, *release_res; + /* The buffer will be released when raw.write() returns. */ if (PyBuffer_FillInfo(&buf, NULL, start, len, 1, PyBUF_CONTIG_RO) == -1) return -1; memobj = PyMemoryView_FromBuffer(&buf); @@ -1844,7 +1854,15 @@ res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_write, memobj, NULL); errnum = errno; } while (res == NULL && _PyIO_trap_eintr()); + PyErr_Fetch(&exc, &val, &tb); + release_res = PyObject_CallMethod(memobj, "release", NULL); + _PyErr_ChainExceptions(exc, val, tb); Py_DECREF(memobj); + if (release_res == NULL) { + Py_XDECREF(res); + return -1; + } + Py_DECREF(release_res); if (res == NULL) return -1; if (res == Py_None) { diff -r 5d8042ab3361 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Mon Apr 18 07:16:17 2016 +0000 +++ b/Objects/unicodeobject.c Mon Apr 18 13:23:55 2016 +0000 @@ -3142,6 +3142,7 @@ PyObject *buffer = NULL, *unicode; Py_buffer info; char lower[11]; /* Enough for any encoding shortcut */ + PyObject *exc, *val, *tb, *res; /* Shortcuts for common default encodings */ if (_Py_normalize_encoding(encoding, lower, sizeof(lower))) { @@ -3173,6 +3174,14 @@ if (buffer == NULL) goto onError; unicode = _PyCodec_DecodeText(buffer, encoding, errors); + PyErr_Fetch(&exc, &val, &tb); + res = PyObject_CallMethod(buffer, "release", NULL); + _PyErr_ChainExceptions(exc, val, tb); + if (res == NULL) { + Py_XDECREF(unicode); + goto onError; + } + Py_DECREF(res); if (unicode == NULL) goto onError; if (!PyUnicode_Check(unicode)) {