GeezerVison is showing github.com — gh-98393: os module reject bytes-like, only accept bytes by vstinner · Pull Request #98394 · python/cpython
gh-98393: os module reject bytes-like, only accept bytes
#98394
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement . We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Uh oh!
There was an error while loading. Please reload this page.
Show file tree
Hide file tree
Changes from all commits
Commits
File filter
Filter by extension
.c
(2)
.py
(4)
.rst
(3)
All 3 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Hide whitespace
Diff view
Hide whitespace
There are no files selected for viewing
Original file line number
Diff line number
Diff line change
Expand Up
@@ -515,6 +515,11 @@ Changes in the Python API
in Python 3.9.
(Contributed by Victor Stinner in :gh:`94352`.)
* The :mod:`os` module no longer accepts bytes-like paths, like
:class:`bytearray` and :class:`memoryview` types: only the exact
:class:`bytes` type is accepted for bytes strings.
(Contributed by Victor Stinner in :gh:`98393`.)
Build Changes
=============
Expand Down
Expand Up
@@ -631,6 +636,11 @@ Porting to Python 3.12
to traverse and clear their instance's dictionaries.
To clear weakrefs, call :c:func:`PyObject_ClearWeakRefs`, as before.
* The :c:func:`PyUnicode_FSDecoder` function no longer accepts bytes-like
paths, like :class:`bytearray` and :class:`memoryview` types: only the exact
:class:`bytes` type is accepted for bytes strings.
(Contributed by Victor Stinner in :gh:`98393`.)
Deprecated
----------
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -494,9 +494,8 @@ def test_compile_filename(self):
code = compile('pass', filename, 'exec')
self.assertEqual(code.co_filename, 'file.py')
for filename in bytearray(b'file.py'), memoryview(b'file.py'):
with self.assertWarns(DeprecationWarning):
code = compile('pass', filename, 'exec')
self.assertEqual(code.co_filename, 'file.py')
with self.assertRaises(TypeError):
compile('pass', filename, 'exec')
self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec')
@support.cpython_only
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -3860,18 +3860,18 @@ def test_oserror_filename(self):
for filenames, func, *func_args in funcs:
for name in filenames:
try :
if isinstance(name, (str, bytes) ):
if not isinstance(name, (str, bytes)) :
Comment thread
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more .
I would move these cases into separate test. It is not anymore about the filename attribute of OSError.
Sorry, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
with self.assertRaises(TypeError ):
func(name, *func_args)
else:
with self.assertWarnsRegex(DeprecationWarning, 'should be'):
func(name, *func_args)
except OSError as err:
self.assertIs(err.filename, name, str(func))
except UnicodeDecodeError:
pass
else:
self.fail("No exception thrown by {}".format(func))
try:
func(name, *func_args)
except OSError as err:
self.assertIs(err.filename, name, str(func))
except UnicodeDecodeError:
pass
else:
self.fail("No exception thrown by {}".format(func))
class CPUCountTests(unittest.TestCase):
def test_cpu_count(self):
Expand Down
Expand Up
@@ -4350,16 +4350,8 @@ def test_bytes_like(self):
for cls in bytearray, memoryview:
path_bytes = cls(os.fsencode(self.path))
with self.assertWarns(DeprecationWarning):
entries = list(os.scandir(path_bytes))
self.assertEqual(len(entries), 1, entries)
entry = entries[0]
self.assertEqual(entry.name, b'file.txt')
self.assertEqual(entry.path,
os.fsencode(os.path.join(self.path, 'file.txt')))
self.assertIs(type(entry.name), bytes)
self.assertIs(type(entry.path), bytes)
with self.assertRaises(TypeError):
list(os.scandir(path_bytes))
Comment thread
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more .
list() is redundant here.
Sorry, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
@unittest.skipUnless(os.listdir in os.supports_fd,
'fd support for listdir required for this test.')
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -634,7 +634,7 @@ def test_stat(self):
self.assertTrue(posix.stat(os_helper.TESTFN))
self.assertTrue(posix.stat(os.fsencode(os_helper.TESTFN)))
self.assertWarnsRegex(DeprecationWarning ,
self.assertRaisesRegex(TypeError ,
'should be string, bytes, os.PathLike or integer, not',
posix.stat, bytearray(os.fsencode(os_helper.TESTFN)))
self.assertRaisesRegex(TypeError,
Expand Down
Expand Up
@@ -841,11 +841,8 @@ def test_listdir_bytes(self):
def test_listdir_bytes_like(self):
for cls in bytearray, memoryview:
with self.assertWarns(DeprecationWarning):
names = posix.listdir(cls(b'.'))
self.assertIn(os.fsencode(os_helper.TESTFN), names)
for name in names:
self.assertIs(type(name), bytes)
with self.assertRaises(TypeError):
posix.listdir(cls(b'.'))
@unittest.skipUnless(posix.listdir in os.supports_fd,
"test needs fd support for posix.listdir()")
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -222,10 +222,9 @@ def checkfilename(brokencode, offset):
checkfilename("def f(x): foo)(", 14) # parse-time
checkfilename("def f(x): global x", 11) # symtable-build-time
symtable.symtable("pass", b"spam", "exec")
with self.assertWarns(DeprecationWarning), \
self.assertRaises(TypeError):
with self.assertRaises(TypeError):
symtable.symtable("pass", bytearray(b"spam"), "exec")
with self.assertWarns(DeprecationWarning ):
with self.assertRaises(TypeError ):
symtable.symtable("pass", memoryview(b"spam"), "exec")
with self.assertRaises(TypeError):
symtable.symtable("pass", list(b"spam"), "exec")
Expand Down
Original file line number
Diff line number
Diff line change
@@ -0,0 +1,3 @@
The :c:func:`PyUnicode_FSDecoder` function no longer accepts bytes-like
paths, like :class:`bytearray` and :class:`memoryview` types: only the exact
:class:`bytes` type is accepted for bytes strings. Patch by Victor Stinner.
Original file line number
Diff line number
Diff line change
@@ -0,0 +1,3 @@
The :mod:`os` module no longer accepts bytes-like paths, like
:class:`bytearray` and :class:`memoryview` types: only the exact
:class:`bytes` type is accepted for bytes strings. Patch by Victor Stinner.
Original file line number
Diff line number
Diff line change
Expand Up
@@ -1120,7 +1120,7 @@ path_converter(PyObject *o, void *p)
path_t *path = (path_t *)p;
PyObject *bytes = NULL;
Py_ssize_t length = 0;
int is_index, is_buffer, is_bytes, is_unicode;
int is_index, is_bytes, is_unicode;
const char *narrow;
#ifdef MS_WINDOWS
PyObject *wo = NULL;
Expand Down
Expand Up
@@ -1158,11 +1158,10 @@ path_converter(PyObject *o, void *p)
/* Only call this here so that we don't treat the return value of
os.fspath() as an fd or buffer. */
is_index = path->allow_fd && PyIndex_Check(o);
is_buffer = PyObject_CheckBuffer(o);
is_bytes = PyBytes_Check(o);
is_unicode = PyUnicode_Check(o);
if (!is_index && !is_buffer && ! is_unicode && !is_bytes) {
if (!is_index && !is_unicode && !is_bytes) {
/* Inline PyOS_FSPath() for better error messages. */
PyObject *func, *res;
Expand Down
Expand Up
@@ -1225,27 +1224,6 @@ path_converter(PyObject *o, void *p)
bytes = o;
Py_INCREF(bytes);
}
else if (is_buffer) {
/* XXX Replace PyObject_CheckBuffer with PyBytes_Check in other code
after removing support of non-bytes buffer objects. */
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
"%s%s%s should be %s, not %.200s",
path->function_name ? path->function_name : "",
path->function_name ? ": " : "",
path->argument_name ? path->argument_name : "path",
path->allow_fd && path->nullable ? "string, bytes, os.PathLike, "
"integer or None" :
path->allow_fd ? "string, bytes, os.PathLike or integer" :
path->nullable ? "string, bytes, os.PathLike or None" :
"string, bytes or os.PathLike",
_PyType_Name(Py_TYPE(o)))) {
goto error_exit;
}
bytes = PyBytes_FromObject(o);
if (!bytes) {
goto error_exit;
}
}
else if (is_index) {
if (!_fd_converter(o, &path->fd)) {
goto error_exit;
Expand Down
Expand Up
@@ -4126,8 +4104,8 @@ _posix_listdir(path_t *path, PyObject *list)
const char *name;
if (path->narrow) {
name = path->narrow;
/* only return bytes if they specified a bytes-like object */
return_str = !PyObject_CheckBuffer (path->object);
/* only return bytes if they specified a bytes object */
return_str = !PyBytes_Check (path->object);
}
else {
name = ".";
Expand Down
Expand Up
@@ -14049,7 +14027,7 @@ DirEntry_from_posix_info(PyObject *module, path_t *path, const char *name,
goto error;
}
if (!path->narrow || !PyObject_CheckBuffer (path->object)) {
if (!path->narrow || !PyBytes_Check (path->object)) {
entry->name = PyUnicode_DecodeFSDefaultAndSize(name, name_len);
if (joined_path)
entry->path = PyUnicode_DecodeFSDefault(joined_path);
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -3619,48 +3619,25 @@ PyUnicode_FSConverter(PyObject* arg, void* addr)
int
PyUnicode_FSDecoder(PyObject* arg, void* addr)
{
int is_buffer = 0;
PyObject *path = NULL;
PyObject *output = NULL;
if (arg == NULL) {
Py_DECREF(*(PyObject**)addr);
*(PyObject**)addr = NULL;
return 1;
}
is_buffer = PyObject_CheckBuffer(arg);
if (!is_buffer) {
path = PyOS_FSPath(arg);
if (path == NULL) {
return 0;
}
}
else {
path = arg;
Py_INCREF(arg);
PyObject *path = PyOS_FSPath(arg);
if (path == NULL) {
return 0;
}
PyObject *output = NULL;
if (PyUnicode_Check(path)) {
output = path;
}
else if (PyBytes_Check(path) || is_buffer) {
PyObject *path_bytes = NULL;
if (!PyBytes_Check(path) &&
PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
"path should be string, bytes, or os.PathLike, not %.200s",
Py_TYPE(arg)->tp_name)) {
Py_DECREF(path);
return 0;
}
path_bytes = PyBytes_FromObject(path);
else if (PyBytes_Check(path)) {
output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path),
PyBytes_GET_SIZE(path));
Py_DECREF(path);
if (!path_bytes) {
return 0;
}
output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path_bytes),
PyBytes_GET_SIZE(path_bytes));
Py_DECREF(path_bytes);
if (!output) {
return 0;
}
Expand All
@@ -3672,6 +3649,7 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr)
Py_DECREF(path);
return 0;
}
if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output),
PyUnicode_GET_LENGTH(output), 0, 1) >= 0) {
PyErr_SetString(PyExc_ValueError, "embedded null character");
Expand Down
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.