Skip to content

Commit 0c5a521

Browse files
pablogsalpablogsal
committed
gh-153569: group indentation and logical-line state
1 parent 324737d commit 0c5a521

12 files changed

Lines changed: 241 additions & 178 deletions

File tree

‎Makefile.pre.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,7 @@ PEGEN_OBJS= \
395395

396396
TOKENIZER_OBJS= \
397397
Parser/lexer/lexer.o \
398+
Parser/lexer/layout.o \
398399
Parser/lexer/number.o \
399400
Parser/lexer/state.o \
400401
Parser/lexer/string.o \

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@
183183
<ClCompile Include="..\Parser\token.c" />
184184
<ClCompile Include="..\Parser\lexer\state.c" />
185185
<ClCompile Include="..\Parser\lexer\lexer.c" />
186+
<ClCompile Include="..\Parser\lexer\layout.c" />
186187
<ClCompile Include="..\Parser\lexer\number.c" />
187188
<ClCompile Include="..\Parser\lexer\string.c" />
188189
<ClCompile Include="..\Parser\tokenizer\decoder.c" />

‎PCbuild/_freeze_module.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,9 @@
463463
<ClCompile Include="..\Parser\lexer\lexer.c">
464464
<Filter>Source Files</Filter>
465465
</ClCompile>
466+
<ClCompile Include="..\Parser\lexer\layout.c">
467+
<Filter>Source Files</Filter>
468+
</ClCompile>
466469
<ClCompile Include="..\Parser\lexer\number.c">
467470
<Filter>Source Files</Filter>
468471
</ClCompile>

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,7 @@
590590
<ClCompile Include="..\Parser\myreadline.c" />
591591
<ClCompile Include="..\Parser\lexer\state.c" />
592592
<ClCompile Include="..\Parser\lexer\lexer.c" />
593+
<ClCompile Include="..\Parser\lexer\layout.c" />
593594
<ClCompile Include="..\Parser\lexer\number.c" />
594595
<ClCompile Include="..\Parser\lexer\string.c" />
595596
<ClCompile Include="..\Parser\tokenizer\cursor.c" />

‎PCbuild/pythoncore.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,6 +1349,9 @@
13491349
<ClCompile Include="..\Parser\lexer\lexer.c">
13501350
<Filter>Parser</Filter>
13511351
</ClCompile>
1352+
<ClCompile Include="..\Parser\lexer\layout.c">
1353+
<Filter>Parser</Filter>
1354+
</ClCompile>
13521355
<ClCompile Include="..\Parser\lexer\number.c">
13531356
<Filter>Parser</Filter>
13541357
</ClCompile>

‎Parser/lexer/layout.c‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
#include "Python.h"
2+
#include "pycore_token.h"
3+
4+
#include "lexer_internal.h"
5+
#include "../tokenizer/helpers.h"
6+
#include "../tokenizer/reader.h"
7+
8+
#define TABSIZE 8
9+
#define ALTTABSIZE 1
10+
11+
int
12+
_PyLexer_ContinueLine(struct tok_state *tok)
13+
{
14+
int c = tok_nextc(tok);
15+
if (c == '\r') {
16+
c = tok_nextc(tok);
17+
}
18+
if (c != '\n') {
19+
tok->done = E_LINECONT;
20+
return -1;
21+
}
22+
c = tok_nextc(tok);
23+
if (c == EOF) {
24+
tok->done = E_EOF;
25+
tok->cur = tok->inp;
26+
return -1;
27+
} else {
28+
tok_backup(tok, c);
29+
}
30+
return c;
31+
}
32+
33+
34+
static int
35+
update_indentation(struct tok_state *tok, int col, int altcol)
36+
{
37+
lexer_layout_state *layout = &tok->layout;
38+
if (col == layout->stack[layout->depth].column) {
39+
if (altcol != layout->stack[layout->depth].alternate_column) {
40+
_PyTokenizer_indenterror(tok);
41+
return -1;
42+
}
43+
}
44+
else if (col > layout->stack[layout->depth].column) {
45+
if (layout->depth + 1 >= MAXINDENT) {
46+
tok->done = E_TOODEEP;
47+
tok->cur = tok->inp;
48+
return -1;
49+
}
50+
if (altcol <= layout->stack[layout->depth].alternate_column) {
51+
_PyTokenizer_indenterror(tok);
52+
return -1;
53+
}
54+
layout->pending++;
55+
layout->stack[++layout->depth] = (indentation_level){col, altcol};
56+
}
57+
else {
58+
while (layout->depth > 0 &&
59+
col < layout->stack[layout->depth].column) {
60+
layout->pending--;
61+
layout->depth--;
62+
}
63+
if (col != layout->stack[layout->depth].column) {
64+
tok->done = E_DEDENT;
65+
tok->cur = tok->inp;
66+
return -1;
67+
}
68+
if (altcol != layout->stack[layout->depth].alternate_column) {
69+
_PyTokenizer_indenterror(tok);
70+
return -1;
71+
}
72+
}
73+
return 0;
74+
}
75+
76+
int
77+
_PyLexer_BeginLine(struct tok_state *tok)
78+
{
79+
assert(tok->layout.at_bol);
80+
int c;
81+
int blankline = 0;
82+
int col = 0;
83+
int altcol = 0;
84+
tok->layout.at_bol = 0;
85+
int cont_line_col = 0;
86+
for (;;) {
87+
c = tok_nextc(tok);
88+
if (c == ' ') {
89+
col++, altcol++;
90+
}
91+
else if (c == '\t') {
92+
col = (col / TABSIZE + 1) * TABSIZE;
93+
altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
94+
}
95+
else if (c == '\014') {/* Control-L (formfeed) */
96+
col = altcol = 0; /* For Emacs users */
97+
}
98+
else if (c == '\\') {
99+
// Indentation cannot be split over multiple physical lines
100+
// using backslashes. This means that if we found a backslash
101+
// preceded by whitespace, **the first one we find** determines
102+
// the level of indentation of whatever comes next.
103+
cont_line_col = cont_line_col ? cont_line_col : col;
104+
if ((c = _PyLexer_ContinueLine(tok)) == -1) {
105+
return -1;
106+
}
107+
}
108+
else if (c == EOF && PyErr_Occurred()) {
109+
return -1;
110+
}
111+
else {
112+
break;
113+
}
114+
}
115+
tok_backup(tok, c);
116+
if (c == '#' || c == '\n' || c == '\r') {
117+
int interactive = _PyTok_ReaderIsInteractive(tok);
118+
/* Lines with only whitespace and/or comments
119+
shouldn't affect the indentation and are
120+
not passed to the parser as NEWLINE tokens,
121+
except *totally* empty lines in interactive
122+
mode, which signal the end of a command group. */
123+
if (col == 0 && c == '\n' && interactive) {
124+
blankline = 0; /* Let it through */
125+
}
126+
else if (interactive && tok->lineno == 1) {
127+
/* In interactive mode, if the first line contains
128+
only spaces and/or a comment, let it through. */
129+
blankline = 0;
130+
col = altcol = 0;
131+
}
132+
else {
133+
blankline = 1; /* Ignore completely */
134+
}
135+
}
136+
if (!blankline && tok->level == 0) {
137+
col = cont_line_col ? cont_line_col : col;
138+
altcol = cont_line_col ? cont_line_col : altcol;
139+
if (update_indentation(tok, col, altcol) < 0) {
140+
return -1;
141+
}
142+
}
143+
return blankline;
144+
}
145+
146+
int
147+
_PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
148+
{
149+
assert(tok->layout.pending != 0);
150+
const char *p_start = NULL;
151+
const char *p_end = NULL;
152+
if (tok->layout.pending < 0) {
153+
if (tok->tok_extra_tokens) {
154+
p_start = tok->cur;
155+
p_end = tok->cur;
156+
}
157+
tok->layout.pending++;
158+
return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
159+
}
160+
else {
161+
if (tok->tok_extra_tokens) {
162+
p_start = tok->buf;
163+
p_end = tok->cur;
164+
}
165+
tok->layout.pending--;
166+
return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
167+
}
168+
}
169+
170+
int
171+
_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
172+
{
173+
tok->layout.at_bol = 1;
174+
if (blankline || tok->level > 0) {
175+
if (!tok->tok_extra_tokens) {
176+
return 0;
177+
}
178+
}
179+
else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) {
180+
return _PyLexer_token_setup(tok, token, NEWLINE,
181+
tok->start, tok->cur - 1);
182+
}
183+
tok->layout.comment_newline = 0;
184+
return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
185+
}
186+
187+
void
188+
_PyLexer_ImplyDedents(struct tok_state *tok)
189+
{
190+
if (tok->layout.depth != 0) {
191+
tok->layout.pending = -tok->layout.depth;
192+
tok->layout.depth = 0;
193+
}
194+
}

0 commit comments

Comments
 (0)