GeezerVisonThe World Wide Web, restored to 800 pixels of glory. |
| Home|About|Gaming cabinet|Archives|How it works|Starter sites |
GeezerVison is showing github.com — cpython/Parser/tokenizer.h at master · python-emscripten/cpython Expand file tree/ Copy pathtokenizer.hMore file actions 88 lines (77 loc) · 3.75 KB / Copy pathtokenizer.hFile metadata and controls88 lines (77 loc) · 3.75 KB 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 #ifndef Py_TOKENIZER_H #define Py_TOKENIZER_H #ifdef __cplusplus extern "C" { #endif #include "object.h" /* Tokenizer interface */ #include "token.h" /* For token types */ #define MAXINDENT 100 /* Max indentation level */ #define MAXLEVEL 200 /* Max parentheses level */ enum decoding_state { STATE_INIT, STATE_RAW, STATE_NORMAL /* have a codec associated with input */ }; /* Tokenizer state */ struct tok_state { /* Input state; buf <= cur <= inp <= end */ /* NB an entire line is held in the buffer */ char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ const char *end; /* End of input buffer if buf != NULL */ const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ int tabsize; /* Tab spacing */ int indent; /* Current indentation index */ int indstack[MAXINDENT]; /* Stack of indents */ int atbol; /* Nonzero if at begin of new line */ int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ const char *prompt, *nextprompt; /* For interactive prompting */ int lineno; /* Current line number */ int first_lineno; /* First line of a single line or multi line string expression (cf. issue 16806) */ int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; int parenlinenostack[MAXLEVEL]; PyObject *filename; /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ enum decoding_state decoding_state; int decoding_erred; /* whether erred in decoding */ int read_coding_spec; /* whether 'coding:...' has been read */ char *encoding; /* Source encoding. */ int cont_line; /* whether we are in a continuation line. */ const char* line_start; /* pointer to start of current line */ const char* multi_line_start; /* pointer to start of first line of a single line or multi line string expression (cf. issue 16806) */ PyObject *decoding_readline; /* open(...).readline */ PyObject *decoding_buffer; const char* enc; /* Encoding for the current str. */ char* str; char* input; /* Tokenizer's newline translated copy of the string. */ int type_comments; /* Whether to look for type comments */ /* async/await related fields (still needed depending on feature_version) */ int async_hacks; /* =1 if async/await aren't always keywords */ int async_def; /* =1 if tokens are inside an 'async def' body. */ int async_def_indent; /* Indentation level of the outermost 'async def'. */ int async_def_nl; /* =1 if the outermost 'async def' had at least one NEWLINE token after it. */ }; extern struct tok_state *PyTokenizer_FromString(const char *, int); extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*, const char *, const char *); extern void PyTokenizer_Free(struct tok_state *); extern int PyTokenizer_Get(struct tok_state *, const char **, const char **); #define tok_dump _Py_tok_dump #ifdef __cplusplus } #endif #endif /* !Py_TOKENIZER_H */ |
| You are visitor 01402520 — GeezerVison does not store the pages you look up. Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required. |