GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — [3.15] gh-151464: exclude '<>' token from tokenize output (GH-154854)… · python/cpython@2168416

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • Grammar
    • python.gram
  • Lib/test
    • test_syntax.py
    • test_tokenize.py
  • Misc/NEWS.d/next/Core_and_Builtins
    • 2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst
  • Parser
    • action_helpers.c
    • lexer
      • lexer.c
      • state.h
    • parser.c
    • pegen.c
    • pegen.h
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎Grammar/python.gram‎

Copy file name to clipboardExpand all lines: Grammar/python.gram
+8Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -794,6 +794,7 @@ compare_op_bitwise_or_pair[CmpopExprPair*]:
794794
| eq_bitwise_or
795795
| noteq_bitwise_or
796796
| lte_bitwise_or
797+
| invalid_noteq
797798
| lt_bitwise_or
798799
| gte_bitwise_or
799800
| gt_bitwise_or
@@ -1643,3 +1644,10 @@ invalid_type_params:
16431644
RAISE_SYNTAX_ERROR_STARTING_FROM(
16441645
token,
16451646
"Type parameter list cannot be empty")}
1647+
1648+
invalid_noteq:
1649+
| a='<' b='>' {
1650+
_PyPegen_tokens_are_adjacent(a, b)
1651+
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '!=' instead of '<>'?")
1652+
: NULL
1653+
}
Collapse file

‎Lib/test/test_syntax.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_syntax.py
+24Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3592,6 +3592,30 @@ def test_ifexp_body_stmt_else_stmt(self):
35923592
]:
35933593
self._check_error(f"x = {lhs_stmt} if 1 else {rhs_stmt}", msg)
35943594

3595+
def test_diamond_operator(self):
3596+
self._check_error(
3597+
"1<>2",
3598+
r"Maybe you meant '!=' instead of '<>'\?",
3599+
lineno=1,
3600+
end_lineno=1,
3601+
offset=2,
3602+
end_offset=4,
3603+
)
3604+
3605+
def test_diamond_operator_barry_as_flufl(self):
3606+
compile(
3607+
"from __future__ import barry_as_FLUFL\n1<>2",
3608+
"<test>", "exec",
3609+
)
3610+
self._check_error(
3611+
"from __future__ import barry_as_FLUFL\na != b",
3612+
"with Barry as BDFL, use '<>' instead of '!='",
3613+
lineno=2,
3614+
end_lineno=2,
3615+
offset=3,
3616+
end_offset=5,
3617+
)
3618+
35953619

35963620
class LazyImportRestrictionTestCase(SyntaxErrorTestCase):
35973621
"""Test syntax restrictions for lazy imports."""
Collapse file

‎Lib/test/test_tokenize.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_tokenize.py
+14Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1233,6 +1233,20 @@ def test_multiline_non_ascii_fstring_with_expr(self):
12331233
FSTRING_END \'"\' (2, 2) (2, 3)
12341234
""")
12351235

1236+
def test_ineq_tokens(self):
1237+
self.check_tokenize("1 != 2", """\
1238+
NUMBER '1' (1, 0) (1, 1)
1239+
OP '!=' (1, 2) (1, 4)
1240+
NUMBER '2' (1, 5) (1, 6)
1241+
""")
1242+
self.check_tokenize("1 <> 2", """\
1243+
NUMBER '1' (1, 0) (1, 1)
1244+
OP '<' (1, 2) (1, 3)
1245+
OP '>' (1, 3) (1, 4)
1246+
NUMBER '2' (1, 5) (1, 6)
1247+
""")
1248+
1249+
12361250
class GenerateTokensTest(TokenizeTest):
12371251
def check_tokenize(self, s, expected):
12381252
# Format the tokens in s in a table format.
Collapse file

‎Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst‎

Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Exclude invalid token ``<>`` from :mod:`tokenize` output. :exc:`SyntaxError`
2+
for ``<>`` now suggests ``!=``.
Collapse file

‎Parser/action_helpers.c‎

Copy file name to clipboardExpand all lines: Parser/action_helpers.c
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include "pegen.h"
88
#include "string_parser.h" // _PyPegen_decode_string()
9+
#include "lexer/state.h" // tok_state
910

1011

1112
void *
@@ -2095,6 +2096,7 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name,
20952096
alias_ty alias = asdl_seq_GET(names, i);
20962097
if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) {
20972098
p->flags |= PyPARSE_BARRY_AS_BDFL;
2099+
p->tok->barry_as_bdfl = 1;
20982100
}
20992101
}
21002102
}
Collapse file

‎Parser/lexer/lexer.c‎

Copy file name to clipboardExpand all lines: Parser/lexer/lexer.c
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1245,6 +1245,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
12451245
{
12461246
int c2 = tok_nextc(tok);
12471247
int current_token = _PyToken_TwoChars(c, c2);
1248+
if (c == '<' && c2 == '>' && !tok->barry_as_bdfl) {
1249+
current_token = OP;
1250+
}
12481251
if (current_token != OP) {
12491252
int c3 = tok_nextc(tok);
12501253
int current_token3 = _PyToken_ThreeChars(c, c2, c3);
Collapse file

‎Parser/lexer/state.h‎

Copy file name to clipboardExpand all lines: Parser/lexer/state.h
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ struct tok_state {
138138
#ifdef Py_DEBUG
139139
int debug;
140140
#endif
141+
int barry_as_bdfl;
141142
};
142143

143144
int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.