GeezerVison

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\

_PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))

&& !(tok->cur > ignore_end

&& ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));

int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;

int start_col_offset = is_type_ignore

? ignore_end_col_offset : current_starting_col_offset;

p_end = tok->cur;

if (is_type_ignore) {

p_start = ignore_end;

p_end = tok->cur;

/* If this type ignore is the only thing on the line, consume the newline also. */

if (blankline) {

tok_nextc(tok);

tok->atbol = 1;

}

return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);

} else {

p_start = type_start;

p_end = tok->cur;

return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);

}

_PyLexer_token_setup(tok, token, type, p_start, p_end);

token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};

token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset};

return type;

}

}

if (tok->tok_extra_tokens) {

int _PyTokenizer_Get(struct tok_state *, struct token *);

static inline const char *

_PyToken_TextView(const struct tok_state *tok, const struct token *token,

Py_ssize_t *length)

{

assert(length != NULL);

if (token->span.start < 0) {

assert(token->span.start == -1 && token->span.end == -1);

*length = 0;

return "";

}

assert(_PyTok_SpanIsValid(token->span));

assert(tok->buf != NULL);

assert(tok->inp >= tok->buf);

assert(token->span.start >= tok->buf_offset);

assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);

*length = token->span.end - token->span.start;

return tok->buf + (token->span.start - tok->buf_offset);

}

_PyToken_Init(struct token *token) {

#ifdef Py_DEBUG

token->span = (_PyTok_Span){-1, -1};

token->start_loc = (_PyTok_Loc){-1, -1};

token->end_loc = (_PyTok_Loc){-1, -1};

token->metadata = NULL;

}

_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,

int end_col_offset, const char *start, const char *end)

static inline _PyTok_Span

buffer_span(const struct tok_state *tok, const char *start, const char *end)

{

token->level = tok->level;

token->lineno = token->end_lineno = tok->lineno;

token->col_offset = col_offset;

token->end_col_offset = end_col_offset;

token->start = start;

token->end = end;

return type;

if (start == NULL) {

assert(end == NULL);

return (_PyTok_Span){-1, -1};

}

assert(end != NULL);

const char *base = tok->buf;

assert(base != NULL);

assert(tok->inp >= base);

Py_ssize_t start_offset = start - base;

Py_ssize_t end_offset = end - base;

assert(start_offset >= 0 && start_offset <= end_offset);

assert(end_offset <= tok->inp - base);

assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);

return _PyTok_SpanFromBounds(

tok->buf_offset + start_offset, tok->buf_offset + end_offset);

}

_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)

{

assert((start == NULL && end == NULL) || (start != NULL && end != NULL));

token->level = tok->level;

if (ISSTRINGLIT(type)) {

token->lineno = tok->first_lineno;

}

else {

token->lineno = tok->lineno;

}

token->end_lineno = tok->lineno;

token->col_offset = token->end_col_offset = -1;

token->start = start;

token->end = end;

token->span = buffer_span(tok, start, end);

int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;

token->start_loc = (_PyTok_Loc){lineno, -1};

token->end_loc = (_PyTok_Loc){tok->lineno, -1};

if (start != NULL && end != NULL) {

token->col_offset = tok->starting_col_offset;

token->end_col_offset = tok->col_offset;

token->start_loc.byte_col = tok->starting_col_offset;

token->end_loc.byte_col = tok->col_offset;

}

return type;

}

struct token {

int level;

int lineno, col_offset, end_lineno, end_col_offset;

const char *start, *end;

_PyTok_Span span;

_PyTok_Loc start_loc;

_PyTok_Loc end_loc;

PyObject *metadata;

};

struct tok_state {

/* Input state; buf <= cur <= inp */

/* NB an entire line is held in the buffer */

char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */

char *buf; /* Owned for file/readline input; source-backed otherwise. */

char *cur; /* Next character in buffer */

char *inp; /* End of data in buffer */

_PyTok_Off buf_offset; /* Logical offset of buf[0]. */

};

int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,

int end_col_offset, const char *start, const char *end);

int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);

struct tok_state *_PyTokenizer_tok_new(void);

}

static int

_get_keyword_or_name_type(Parser *p, struct token *new_token)

_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length)

{

Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset;

assert(name_len > 0);

assert(length > 0);

if (name_len >= p->n_keyword_lists ||

p->keywords[name_len] == NULL ||

p->keywords[name_len]->type == -1) {

if (length >= p->n_keyword_lists ||

p->keywords[length] == NULL ||

p->keywords[length]->type == -1) {

return NAME;

}

for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {

if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) {

for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) {

if (memcmp(k->str, text, (size_t)length) == 0) {

return k->type;

}

}

initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {

assert(parser_token != NULL);

parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type;

parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start);

Py_ssize_t length;

const char *text = _PyToken_TextView(p->tok, new_token, &length);

parser_token->type = token_type == NAME

? _get_keyword_or_name_type(p, text, length) : token_type;

parser_token->bytes = PyBytes_FromStringAndSize(text, length);

if (parser_token->bytes == NULL) {

return -1;

}

}

parser_token->level = new_token->level;

parser_token->lineno = new_token->lineno;

parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset

: new_token->col_offset;

parser_token->end_lineno = new_token->end_lineno;

parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset

: new_token->end_col_offset;

parser_token->lineno = new_token->start_loc.lineno;

parser_token->col_offset = p->tok->lineno == p->starting_lineno

? p->starting_col_offset + new_token->start_loc.byte_col

: new_token->start_loc.byte_col;

parser_token->end_lineno = new_token->end_loc.lineno;

parser_token->end_col_offset = p->tok->lineno == p->starting_lineno

? p->starting_col_offset + new_token->end_loc.byte_col

: new_token->end_loc.byte_col;

p->fill += 1;

// Record and skip '# type: ignore' comments

while (type == TYPE_IGNORE) {

Py_ssize_t len = new_token.end_col_offset - new_token.col_offset;

Py_ssize_t len;

const char *text = _PyToken_TextView(p->tok, &new_token, &len);

char *tag = PyMem_Malloc((size_t)len + 1);

if (tag == NULL) {

PyErr_NoMemory();

goto error;

}

strncpy(tag, new_token.start, (size_t)len);

memcpy(tag, text, (size_t)len);

tag[len] = '\0';

// Ownership of tag passes to the growable array

if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {

typedef Py_ssize_t _PyTok_Off;

typedef struct {

_PyTok_Off start;

_PyTok_Off end;

}

static void

_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start,

PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno,

_get_col_offsets(tokenizeriterobject *it, const struct token *token,

const char *token_start, const char *line_start,

PyObject *line, int line_changed,

Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)

{

_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);

const char *token_end = token_start == NULL

? NULL : token_start + token->span.end - token->span.start;

Py_ssize_t lineno = token->start_loc.lineno;

Py_ssize_t end_lineno = token->end_loc.lineno;

Py_ssize_t byte_offset = -1;

if (token.start != NULL && token.start >= line_start) {

byte_offset = token.start - line_start;

if (token_start != NULL && token_start >= line_start) {

byte_offset = token_start - line_start;

if (line_changed) {

*col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);

it->byte_col_offset_diff = byte_offset - *col_offset;

}

}

if (token.end != NULL && token.end >= it->tok->line_start) {

Py_ssize_t end_byte_offset = token.end - it->tok->line_start;

if (token_end != NULL && token_end >= it->tok->line_start) {

Py_ssize_t end_byte_offset = token_end - it->tok->line_start;

if (lineno == end_lineno) {

// If the whole token is at the same line, we can just use the token.start

// buffer for figuring out the new column offset, since using line is not

// performant for very long lines.

// Avoid rescanning the prefix of a very long line.

Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);

*end_col_offset = *col_offset + token_col_offset;

it->byte_col_offset_diff += token.end - token.start - token_col_offset;

it->byte_col_offset_diff += token_end - token_start - token_col_offset;

}

else {

*end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);

it->done = 1;

goto exit;

}

PyObject *str = NULL;

if (token.start == NULL || token.end == NULL) {

const char *token_start = NULL;

PyObject *str;

if (token.span.start < 0) {

assert(token.span.start == -1 && token.span.end == -1);

str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);

}

else {

str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);

Py_ssize_t token_length;

token_start = _PyToken_TextView(

it->tok, &token, &token_length);

str = PyUnicode_FromStringAndSize(token_start, token_length);

}

if (str == NULL) {

goto exit;

goto exit;

}

Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;

Py_ssize_t end_lineno = it->tok->lineno;

Py_ssize_t lineno = token.start_loc.lineno;

Py_ssize_t end_lineno = token.end_loc.lineno;

Py_ssize_t col_offset = -1;

Py_ssize_t end_col_offset = -1;

_get_col_offsets(it, token, line_start, line, line_changed,

lineno, end_lineno, &col_offset, &end_col_offset);

_get_col_offsets(it, &token, token_start, line_start, line, line_changed,

&col_offset, &end_col_offset);

if (it->tok->tok_extra_tokens) {

if (is_trailing_token) {

else if (type == NEWLINE) {

Py_DECREF(str);

if (!it->tok->implicit_newline) {

if (it->tok->start[0] == '\r') {

assert(token_start != NULL);

if (token_start[0] == '\r') {

str = PyUnicode_FromString("\r\n");

} else {

str = PyUnicode_FromString("\n");