GeezerVison is showing github.com — gh-96670: Raise SyntaxError when parsing NULL bytes by pablogsal · Pull Request #97594 · python/cpython
gh-96670: Raise SyntaxError when parsing NULL bytes
#97594
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement . We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Uh oh!
There was an error while loading. Please reload this page.
Show file tree
Hide file tree
Changes from all commits
Commits
File filter
Filter by extension
.c
(3)
.h
(1)
.py
(4)
.rst
(2)
All 4 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Hide whitespace
Diff view
Hide whitespace
There are no files selected for viewing
Original file line number
Diff line number
Diff line change
Expand Up
@@ -86,6 +86,12 @@ Other Language Changes
* :class:`memoryview` now supports the half-float type (the "e" format code).
(Contributed by Dong-hee Na and Antoine Pitrou in :gh:`90751`.)
* The parser now raises :exc:`SyntaxError` when parsing source code containing
null bytes. (Contributed by Pablo Galindo in :gh:`96670`.)
* :func:`ast.parse` now raises :exc:`SyntaxError` instead of :exc:`ValueError`
when parsing source code containing null bytes. (Contributed by Pablo Galindo
in :gh:`96670`.)
New Modules
===========
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -3,6 +3,7 @@
#endif
PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
PyAPI_FUNC(char *) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*);
/* The std printer acts as a preliminary sys.stderr until the new io
infrastructure is in place. */
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -844,6 +844,10 @@ def check_limit(prefix, repeated):
check_limit("a", "[0]")
check_limit("a", "*a")
def test_null_bytes(self):
with self.assertRaises(SyntaxError,
msg="source code string cannot contain null bytes"):
ast.parse("a\0b")
class ASTHelpers_Test(unittest.TestCase):
maxDiff = None
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -334,11 +334,10 @@ def test_compile(self):
self.assertRaises(TypeError, compile)
self.assertRaises(ValueError, compile, 'print(42)\n', '<string>', 'badmode')
self.assertRaises(ValueError, compile, 'print(42)\n', '<string>', 'single', 0xff)
self.assertRaises(ValueError, compile, chr(0), 'f', 'exec')
self.assertRaises(TypeError, compile, 'pass', '?', 'exec',
mode='eval', source='0', filename='tmp')
compile('print("\xe5")\n', '', 'exec')
self.assertRaises(ValueError , compile, chr(0), 'f', 'exec')
self.assertRaises(SyntaxError , compile, chr(0), 'f', 'exec')
self.assertRaises(ValueError, compile, str('a = 1'), 'f', 'bad')
# test the optimize argument
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -657,6 +657,18 @@ def test_syntaxerror_invalid_escape_sequence_multi_line(self):
],
)
def test_syntaxerror_null_bytes(self):
script = "x = '\0' nothing to see here\n';import os;os.system('echo pwnd')\n"
with os_helper.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, 'script', script)
exitcode, stdout, stderr = assert_python_failure(script_name)
self.assertEqual(
stderr.splitlines()[-2:],
[ b" x = '",
b'SyntaxError: source code cannot contain null bytes'
],
)
def test_consistent_sys_path_for_direct_execution(self):
# This test case ensures that the following all give the same
# sys.path configuration:
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -544,7 +544,7 @@ def test_particularly_evil_undecodable(self):
with open(fn, "wb") as fp:
fp.write(src)
res = script_helper.run_python_until_end(fn)[0]
self.assertIn(b"Non-UTF-8 ", res.err)
self.assertIn(b"source code cannot contain null bytes ", res.err)
def test_yet_more_evil_still_undecodable(self):
# Issue #25388
Expand All
@@ -554,7 +554,7 @@ def test_yet_more_evil_still_undecodable(self):
with open(fn, "wb") as fp:
fp.write(src)
res = script_helper.run_python_until_end(fn)[0]
self.assertIn(b"Non-UTF-8 ", res.err)
self.assertIn(b"source code cannot contain null bytes ", res.err)
@support.cpython_only
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
Expand Down
Expand Up
@@ -591,9 +591,9 @@ def check_limit(prefix, repeated, mode="single"):
def test_null_terminated(self):
# The source code is null-terminated internally, but bytes-like
# objects are accepted, which could be not terminated.
with self.assertRaisesRegex(ValueError , "cannot contain null"):
with self.assertRaisesRegex(SyntaxError , "cannot contain null"):
compile("123\x00", "<dummy>", "eval")
with self.assertRaisesRegex(ValueError , "cannot contain null"):
with self.assertRaisesRegex(SyntaxError , "cannot contain null"):
compile(memoryview(b"123\x00"), "<dummy>", "eval")
code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval")
self.assertEqual(eval(code), 23)
Expand Down
Original file line number
Diff line number
Diff line change
@@ -0,0 +1,2 @@
The parser now raises :exc:`SyntaxError` when parsing source code containing
null bytes. Patch by Pablo Galindo
Original file line number
Diff line number
Diff line change
Expand Up
@@ -230,16 +230,8 @@ _PyLong_FileDescriptor_Converter(PyObject *o, void *ptr)
return 1;
}
/*
** Py_UniversalNewlineFgets is an fgets variation that understands
** all of \r, \n and \r\n conventions.
** The stream should be opened in binary mode.
** The fobj parameter exists solely for legacy reasons and must be NULL.
** Note that we need no error handling: fgets() treats error and eof
** identically.
*/
char *
Py_UniversalNewlineFgets (char *buf, int n, FILE *stream, PyObject *fobj)
_Py_UniversalNewlineFgetsWithSize (char *buf, int n, FILE *stream, PyObject *fobj, size_t* size )
Comment thread
pablogsal marked this conversation as resolved.
Resolved
Uh oh!
There was an error while loading. Please reload this page.
{
char *p = buf;
int c;
Expand All
@@ -265,11 +257,28 @@ Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
}
FUNLOCKFILE(stream);
*p = '\0';
if (p == buf)
if (p == buf) {
return NULL;
}
*size = p - buf;
return buf;
}
/*
** Py_UniversalNewlineFgets is an fgets variation that understands
** all of \r, \n and \r\n conventions.
** The stream should be opened in binary mode.
** The fobj parameter exists solely for legacy reasons and must be NULL.
** Note that we need no error handling: fgets() treats error and eof
** identically.
*/
char *
Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj) {
size_t size;
return _Py_UniversalNewlineFgetsWithSize(buf, n, stream, fobj, &size);
}
/* **************************** std printer ****************************
* The stdprinter is used during the boot strapping phase as a preliminary
* file like object for sys.stderr.
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -378,6 +378,11 @@ tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
return 1;
}
static inline int
contains_null_bytes(const char* str, size_t size) {
return memchr(str, 0, size) != NULL;
}
static int
tok_readline_recode(struct tok_state *tok) {
PyObject *line;
Expand Down
Expand Up
@@ -829,17 +834,17 @@ tok_readline_raw(struct tok_state *tok)
if (!tok_reserve_buf(tok, BUFSIZ)) {
return 0;
}
char *line = Py_UniversalNewlineFgets( tok->inp,
(int)(tok->end - tok->inp),
tok->fp, NULL);
int n_chars = (int)( tok->end - tok-> inp);
size_t line_size = 0;
char *line = _Py_UniversalNewlineFgetsWithSize(tok->inp, n_chars, tok->fp, NULL, &line_size );
if (line == NULL) {
return 1;
}
if (tok->fp_interactive &&
tok_concatenate_interactive_new_line(tok, line) == -1) {
return 0;
}
tok->inp = strchr(tok->inp, '\0') ;
tok->inp += line_size ;
if (tok->inp == tok->buf) {
return 0;
}
Expand Down
Expand Up
@@ -1075,6 +1080,12 @@ tok_nextc(struct tok_state *tok)
return EOF;
}
tok->line_start = tok->cur;
if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
syntaxerror(tok, "source code cannot contain null bytes");
tok->cur = tok->inp;
return EOF;
}
}
Py_UNREACHABLE();
}
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -1858,7 +1858,7 @@ _Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyComp
}
if (strlen(str) != (size_t)size) {
PyErr_SetString(PyExc_ValueError ,
PyErr_SetString(PyExc_SyntaxError ,
"source code string cannot contain null bytes");
Py_CLEAR(*cmd_copy);
return NULL;
Expand Down
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.