GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — gh-153569: move tokenizer state to source offsets · python/cpython@1753605

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • Lib/test
    • test_codeop.py
    • test_eof.py
    • test_free_threading
      • test_tokenize.py
    • test_fstring.py
    • test_syntax.py
    • test_tokenize.py
  • Makefile.pre.in
  • Modules/_testinternalcapi
    • tokenizer.c
  • PCbuild
    • _freeze_module.vcxproj.filters
    • _freeze_module.vcxproj
    • pythoncore.vcxproj.filters
    • pythoncore.vcxproj
  • Parser
    • action_helpers.c
    • lexer
      • buffer.c
      • buffer.h
      • lexer.c
      • lexer.h
      • lexer_internal.h
      • number.c
      • state.c
      • state.h
      • string.c
    • pegen.c
    • pegen.h
    • pegen_errors.c
    • string_parser.c
    • tokenizer
      • api.c
      • cursor.c
      • cursor.h
      • decoder.c
      • errors.h
      • helpers.c
      • helpers.h
      • reader.c
      • reader.h
      • reader_internal.h
      • source.c
      • source.h
      • tokenizer.h
      • tokenizer_errors.c
  • Python
    • Python-tokenize.c
    • errors.c
    • traceback.c
  • Tools
    • c-analyzer/cpython
      • ignored.tsv
    • peg_generator/pegen
      • build.py

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Dismiss banner
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎Lib/test/test_codeop.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_codeop.py
+72-1Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,19 @@
22
Test cases for codeop.py
33
Nick Mathewson
44
"""
5+
import builtins
56
import unittest
67
import warnings
78
from test.support import subTests, warnings_helper
89
from textwrap import dedent
910
import functools
1011

1112
from codeop import compile_command, CommandCompiler, Compile
12-
from codeop import PyCF_DONT_IMPLY_DEDENT, PyCF_ONLY_AST
13+
from codeop import (
14+
PyCF_ALLOW_INCOMPLETE_INPUT,
15+
PyCF_DONT_IMPLY_DEDENT,
16+
PyCF_ONLY_AST,
17+
)
1318
import ast
1419

1520

@@ -248,6 +253,72 @@ def test_incomplete(self, compiler):
248253
ai('a = f"""')
249254
ai('a = \\')
250255

256+
def test_tokenizer_incomplete_input_classification(self):
257+
cases = [
258+
(
259+
"x = 'abc",
260+
"single",
261+
builtins._IncompleteInputError,
262+
("incomplete input", 1, 5, 1, -1),
263+
),
264+
(
265+
'f"""abc',
266+
"single",
267+
builtins._IncompleteInputError,
268+
("incomplete input", 1, 1, 1, -1),
269+
),
270+
(
271+
"x = \\\n",
272+
"single",
273+
builtins._IncompleteInputError,
274+
("incomplete input", 1, 6, 1, -1),
275+
),
276+
(
277+
"x = 'abc",
278+
"exec",
279+
SyntaxError,
280+
(
281+
"unterminated string literal (detected at line 1)",
282+
1, 5, 1, 5,
283+
),
284+
),
285+
(
286+
'f"abc',
287+
"single",
288+
SyntaxError,
289+
(
290+
"unterminated f-string literal (detected at line 1)",
291+
1, 1, 1, 1,
292+
),
293+
),
294+
(
295+
"x = \\",
296+
"single",
297+
SyntaxError,
298+
(
299+
"unexpected character after line continuation character",
300+
1, 5, 1, 0,
301+
),
302+
),
303+
]
304+
305+
for source, mode, exception_type, expected in cases:
306+
with self.subTest(source=source, mode=mode):
307+
with self.assertRaises(exception_type) as caught:
308+
compile(
309+
source,
310+
"<test>",
311+
mode,
312+
PyCF_ALLOW_INCOMPLETE_INPUT,
313+
)
314+
self.assertIs(type(caught.exception), exception_type)
315+
error = caught.exception
316+
self.assertEqual(
317+
(error.msg, error.lineno, error.offset,
318+
error.end_lineno, error.end_offset),
319+
expected,
320+
)
321+
251322
@subTests('compiler', COMPILERS)
252323
def test_invalid(self, compiler):
253324
ai = functools.partial(self.assertInvalid, compiler=compiler)
Collapse file

‎Lib/test/test_eof.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_eof.py
-1Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,6 @@ def test_line_continuation_EOF(self):
126126
@unittest.skipIf(not sys.executable, "sys.executable required")
127127
@force_not_colorized
128128
def test_line_continuation_EOF_from_file_bpo2180(self):
129-
"""Ensure tok_nextc() does not add too many ending newlines."""
130129
with os_helper.temp_dir() as temp_dir:
131130
file_name = script_helper.make_script(temp_dir, 'foo', '\\')
132131
rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name)
Collapse file

‎Lib/test/test_free_threading/test_tokenize.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_free_threading/test_tokenize.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def next_token(it):
4545
tokenize.TokenInfo(type=1, string='pass', start=(2, 2), end=(2, 6), line=' pass'),
4646
tokenize.TokenInfo(type=4, string='', start=(2, 6), end=(2, 6), line=' pass'),
4747
tokenize.TokenInfo(type=6, string='', start=(2, -1), end=(2, -1), line=' pass'),
48-
tokenize.TokenInfo(type=0, string='', start=(2, -1), end=(2, -1), line=' pass'),
48+
tokenize.TokenInfo(type=0, string='', start=(3, 0), end=(3, 0), line=''),
4949
]
5050

5151
tokens.sort()
Collapse file

‎Lib/test/test_fstring.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_fstring.py
+23-8Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -710,14 +710,16 @@ def test_double_braces(self):
710710
])
711711

712712
def test_double_brace_ast_location_covers_both_source_braces(self):
713-
value = ast.parse('f"a{{"').body[0].value.values[0]
714-
self.assertIsInstance(value, ast.Constant)
715-
self.assertEqual(value.value, "a{")
716-
self.assertEqual(
717-
(value.lineno, value.col_offset, value.end_lineno,
718-
value.end_col_offset),
719-
(1, 2, 1, 5),
720-
)
713+
for source, expected in [('f"a{{"', "a{"), ('f"a}}"', "a}")]:
714+
with self.subTest(source=source):
715+
value = ast.parse(source).body[0].value.values[0]
716+
self.assertIsInstance(value, ast.Constant)
717+
self.assertEqual(value.value, expected)
718+
self.assertEqual(
719+
(value.lineno, value.col_offset, value.end_lineno,
720+
value.end_col_offset),
721+
(1, 2, 1, 5),
722+
)
721723

722724
def test_compile_time_concat(self):
723725
x = 'def'
@@ -1301,6 +1303,13 @@ def test_nested_fstrings(self):
13011303
self.assertEqual(f'{f"{0}"*3}', '000')
13021304
self.assertEqual(f'{f"{y}"*3}', '555')
13031305

1306+
def test_deeply_nested_fstrings_with_leading_text(self):
1307+
source = "0"
1308+
for depth in range(20):
1309+
quote = '"' if depth % 2 == 0 else "'"
1310+
source = f"f{quote}x{{{source}}}{quote}"
1311+
self.assertEqual(eval(source), "x" * 20 + "0")
1312+
13041313
def test_invalid_string_prefixes(self):
13051314
single_quote_cases = ["fu''",
13061315
"uf''",
@@ -1679,6 +1688,12 @@ def __repr__(self):
16791688

16801689
self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
16811690
self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
1691+
self.assertEqual(f'''{" \" # nooo \" " # real comment
1692+
=}''', '" \\" # nooo \\" " \n=\' " # nooo " \'')
1693+
self.assertEqual(f'{"""a" # inside"""=}',
1694+
'"""a" # inside"""=\'a" # inside\'')
1695+
self.assertEqual(f"{'''a' # inside'''=}",
1696+
"'''a' # inside'''=\"a' # inside\"")
16821697

16831698
self.assertEqual(f'{ # some comment goes here
16841699
"""hello"""=}', ' \n """hello"""=\'hello\'')
Collapse file

‎Lib/test/test_syntax.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_syntax.py
+36Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3088,6 +3088,42 @@ def test_expression_with_assignment(self):
30883088
def test_curly_brace_after_primary_raises_immediately(self):
30893089
self._check_error("f{}", "invalid syntax", mode="single")
30903090

3091+
def test_tokenizer_eof_error_offsets_after_non_ascii(self):
3092+
self._check_error(
3093+
"é + (",
3094+
re.escape("'(' was never closed"),
3095+
lineno=1,
3096+
offset=5,
3097+
end_lineno=1,
3098+
end_offset=0,
3099+
)
3100+
self._check_error(
3101+
"é + \\\n",
3102+
"unexpected EOF while parsing",
3103+
lineno=1,
3104+
offset=6,
3105+
end_lineno=1,
3106+
end_offset=-1,
3107+
)
3108+
3109+
def test_fstring_error_text_excludes_previous_newline(self):
3110+
cases = [
3111+
('f"{a:{\n0\n\'\'=(\n="0("', (2, 1, "0", 3, 2)),
3112+
("f'{x!=[\n]a\n}(]==!", (2, 3, "]a", 2, 2)),
3113+
]
3114+
for source, expected in cases:
3115+
with self.subTest(source=source):
3116+
with self.assertRaises(SyntaxError) as caught:
3117+
compile(source, "<testcase>", "exec")
3118+
actual = (
3119+
caught.exception.lineno,
3120+
caught.exception.offset,
3121+
caught.exception.text,
3122+
caught.exception.end_lineno,
3123+
caught.exception.end_offset,
3124+
)
3125+
self.assertEqual(actual, expected)
3126+
30913127
def test_assign_call(self):
30923128
self._check_error("f() = 1", "assign")
30933129

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.