GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — [improvement](parser) Skip hidden tokens in internal SQL parsing (#67… · apache/doris@d145a5d

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • fe
    • fe-core/src
      • main/java/org/apache/doris/nereids/parser
        • NereidsParser.java
      • test/java/org/apache/doris/nereids/parser
        • LeanTokenModeTest.java
    • fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark
      • LeanTokenModeBenchmark.java
    • fe-sql-parser/src
      • main/antlr4/org/apache/doris/nereids
        • DorisLexer.g4
      • test/java/org/apache/doris/sqlparser
        • DorisLexerLeanTokenModeTest.java
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java‎

Copy file name to clipboardExpand all lines: fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
+33-5Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.antlr.v4.runtime.Token;
5050
import org.antlr.v4.runtime.TokenSource;
5151
import org.antlr.v4.runtime.atn.PredictionMode;
52+
import org.antlr.v4.runtime.misc.Interval;
5253
import org.antlr.v4.runtime.misc.ParseCancellationException;
5354
import org.antlr.v4.runtime.tree.TerminalNode;
5455
import org.apache.commons.collections4.CollectionUtils;
@@ -343,7 +344,7 @@ private <T> T parse(String sql, Function<DorisParser, ParserRuleContext> parseFu
343344

344345
private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
345346
Function<DorisParser, ParserRuleContext> parseFunction) {
346-
CommonTokenStream tokenStream = parseAllTokens(sql);
347+
CommonTokenStream tokenStream = parseLeanTokens(sql);
347348
ParserRuleContext tree = toAst(tokenStream, parseFunction);
348349
LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null
349350
? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint))
@@ -352,15 +353,15 @@ private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
352353
}
353354

354355
public LogicalPlan parseForCreateView(String sql) {
355-
CommonTokenStream tokenStream = parseAllTokens(sql);
356+
CommonTokenStream tokenStream = parseLeanTokens(sql);
356357
ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
357358
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView(
358359
getHintMap(sql, tokenStream, DorisParser::selectHint));
359360
return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
360361
}
361362

362363
public LogicalPlan parseForEncryption(String sql, Map<Pair<Integer, Integer>, String> indexInSqlToString) {
363-
CommonTokenStream tokenStream = parseAllTokens(sql);
364+
CommonTokenStream tokenStream = parseLeanTokens(sql);
364365
ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
365366
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForEncryption(
366367
getHintMap(sql, tokenStream, DorisParser::selectHint), indexInSqlToString);
@@ -369,7 +370,7 @@ public LogicalPlan parseForEncryption(String sql, Map<Pair<Integer, Integer>, St
369370

370371
/** parseForSyncMv */
371372
public Optional<String> parseForSyncMv(String sql) {
372-
CommonTokenStream tokenStream = parseAllTokens(sql);
373+
CommonTokenStream tokenStream = parseLeanTokens(sql);
373374
ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
374375
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
375376
getHintMap(sql, tokenStream, DorisParser::selectHint));
@@ -466,10 +467,37 @@ public static String removeCommentAndTrimBlank(String sql) {
466467
}
467468

468469
private static CommonTokenStream parseAllTokens(String sql) {
470+
return parseTokens(sql, false);
471+
}
472+
473+
private static CommonTokenStream parseLeanTokens(String sql) {
474+
return parseTokens(sql, true);
475+
}
476+
477+
private static CommonTokenStream parseTokens(String sql, boolean leanTokenMode) {
469478
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
470479
lexer.isNoBackslashEscapes = SqlModeHelper.hasNoBackSlashEscapes();
471-
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
480+
lexer.isLeanTokenMode = leanTokenMode;
481+
CommonTokenStream tokenStream = leanTokenMode
482+
? new LeanTokenStream(lexer)
483+
: new CommonTokenStream(lexer);
472484
tokenStream.fill();
473485
return tokenStream;
474486
}
487+
488+
/** Preserve source text used by ANTLR diagnostics without allocating hidden tokens. */
489+
private static final class LeanTokenStream extends CommonTokenStream {
490+
private LeanTokenStream(TokenSource tokenSource) {
491+
super(tokenSource);
492+
}
493+
494+
@Override
495+
public String getText(Token start, Token stop) {
496+
if (start == null || stop == null || start.getType() == Token.EOF) {
497+
return "";
498+
}
499+
return getTokenSource().getInputStream().getText(
500+
Interval.of(start.getStartIndex(), stop.getStopIndex()));
501+
}
502+
}
475503
}
Collapse file

‎fe/fe-core/src/test/java/org/apache/doris/nereids/parser/LeanTokenModeTest.java‎

Copy file name to clipboard
+123Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.parser;
19+
20+
import org.apache.doris.common.Pair;
21+
import org.apache.doris.nereids.DorisLexer;
22+
import org.apache.doris.nereids.analyzer.UnboundSlot;
23+
import org.apache.doris.nereids.exceptions.ParseException;
24+
import org.apache.doris.nereids.properties.SelectHintOrdered;
25+
import org.apache.doris.nereids.trees.plans.Plan;
26+
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
27+
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
28+
import org.apache.doris.qe.ConnectContext;
29+
import org.apache.doris.qe.GlobalVariable;
30+
import org.apache.doris.qe.SqlModeHelper;
31+
import org.apache.doris.sqlparser.DorisSqlParser;
32+
33+
import org.antlr.v4.runtime.Token;
34+
import org.antlr.v4.runtime.TokenSource;
35+
import org.junit.jupiter.api.Assertions;
36+
import org.junit.jupiter.api.Test;
37+
38+
import java.util.ArrayList;
39+
import java.util.List;
40+
41+
class LeanTokenModeTest extends ParserTestBase {
42+
private final NereidsParser parser = new NereidsParser();
43+
44+
@Test
45+
void preservesHintsInDirectAstParsing() {
46+
LogicalPlan plan = parser.parseSingle("SELECT /*+ ORDERED */ a -- ordinary\nFROM t");
47+
List<LogicalSelectHint<Plan>> hintPlans = plan.collectToList(LogicalSelectHint.class::isInstance);
48+
49+
Assertions.assertEquals(1, hintPlans.size());
50+
Assertions.assertTrue(hintPlans.get(0).getHints().stream()
51+
.anyMatch(SelectHintOrdered.class::isInstance));
52+
}
53+
54+
@Test
55+
void keepsFullTokensForScanAndCommentNormalization() {
56+
String sql = "SELECT /* ordinary */ /*+ ORDERED */ a -- tail\nFROM t";
57+
List<Integer> tokenTypes = new ArrayList<>();
58+
TokenSource tokenSource = NereidsParser.scan(sql);
59+
Token token;
60+
do {
61+
token = tokenSource.nextToken();
62+
tokenTypes.add(token.getType());
63+
} while (token.getType() != Token.EOF);
64+
65+
Assertions.assertTrue(tokenTypes.contains(DorisLexer.WS));
66+
Assertions.assertTrue(tokenTypes.contains(DorisLexer.SIMPLE_COMMENT));
67+
Assertions.assertTrue(tokenTypes.contains(DorisLexer.BRACKETED_COMMENT));
68+
Assertions.assertEquals("SELECT /*+ ORDERED */ a FROM t",
69+
NereidsParser.removeCommentAndTrimBlank(sql));
70+
}
71+
72+
@Test
73+
void preservesCharacterIntervalsAcrossSkippedTokens() {
74+
String sql = "SELECT /* leading */ db -- between\n . tbl . col FROM t";
75+
LogicalPlan plan = parser.parseForCreateView(sql);
76+
UnboundSlot slot = plan.<LogicalPlan>collectToList(ignored -> true).stream()
77+
.flatMap(node -> node.getExpressions().stream())
78+
.flatMap(expression -> expression.<UnboundSlot>collectToList(
79+
UnboundSlot.class::isInstance).stream())
80+
.filter(unboundSlot -> unboundSlot.getNameParts().equals(List.of("db", "tbl", "col")))
81+
.findFirst()
82+
.orElseThrow();
83+
84+
Assertions.assertEquals(Pair.of(sql.indexOf("db"), sql.indexOf("col") + 2),
85+
slot.getIndexInSqlString().orElseThrow());
86+
Assertions.assertEquals(sql, parser.parseForSyncMv(
87+
"CREATE MATERIALIZED VIEW mv AS " + sql).orElseThrow());
88+
}
89+
90+
@Test
91+
void producesSameErrorsWithoutHiddenTokens() {
92+
boolean previousAnsi = GlobalVariable.enable_ansi_query_organization_behavior;
93+
long previousSqlMode = ConnectContext.get().getSessionVariable().getSqlMode();
94+
try {
95+
for (boolean noBackslashEscapes : new boolean[] {false, true}) {
96+
long sqlMode = noBackslashEscapes
97+
? previousSqlMode | SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES
98+
: previousSqlMode & ~SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES;
99+
ConnectContext.get().getSessionVariable().setSqlMode(sqlMode);
100+
for (boolean ansi : new boolean[] {false, true}) {
101+
GlobalVariable.enable_ansi_query_organization_behavior = ansi;
102+
for (String sql : List.of(
103+
"SELECT a\n-- skipped comment\nFROM t WHERE )",
104+
"SELECT 1,\n-- missing expression\nFROM t",
105+
"SELECT * FROM -- missing relation\r\n",
106+
"SELECT /*+ ORDERED */ FROM t",
107+
"CREATE TABLE t ( k1 BOOL )",
108+
"SELECT 'a\\' FROM t WHERE )")) {
109+
ParseException fullException = Assertions.assertThrows(ParseException.class,
110+
() -> new DorisSqlParser(noBackslashEscapes, ansi).parseStatement(sql));
111+
ParseException leanException = Assertions.assertThrows(ParseException.class,
112+
() -> parser.parseSingle(sql));
113+
Assertions.assertEquals(fullException.getClass(), leanException.getClass());
114+
Assertions.assertEquals(fullException.getMessage(), leanException.getMessage());
115+
}
116+
}
117+
}
118+
} finally {
119+
GlobalVariable.enable_ansi_query_organization_behavior = previousAnsi;
120+
ConnectContext.get().getSessionVariable().setSqlMode(previousSqlMode);
121+
}
122+
}
123+
}
Collapse file

‎fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/LeanTokenModeBenchmark.java‎

Copy file name to clipboard
+104Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.sqlparser.benchmark;
19+
20+
import org.apache.doris.nereids.DorisLexer;
21+
import org.apache.doris.nereids.DorisParser;
22+
import org.apache.doris.sqlparser.DorisSqlParser;
23+
24+
import org.antlr.v4.runtime.CommonTokenStream;
25+
import org.antlr.v4.runtime.atn.PredictionMode;
26+
import org.openjdk.jmh.annotations.Benchmark;
27+
import org.openjdk.jmh.annotations.BenchmarkMode;
28+
import org.openjdk.jmh.annotations.Fork;
29+
import org.openjdk.jmh.annotations.Level;
30+
import org.openjdk.jmh.annotations.Measurement;
31+
import org.openjdk.jmh.annotations.Mode;
32+
import org.openjdk.jmh.annotations.OutputTimeUnit;
33+
import org.openjdk.jmh.annotations.Param;
34+
import org.openjdk.jmh.annotations.Scope;
35+
import org.openjdk.jmh.annotations.Setup;
36+
import org.openjdk.jmh.annotations.State;
37+
import org.openjdk.jmh.annotations.Warmup;
38+
39+
import java.util.concurrent.TimeUnit;
40+
import java.util.stream.Collectors;
41+
import java.util.stream.IntStream;
42+
43+
/** Measures whitespace and ordinary-comment token allocation in full and lean modes. */
44+
@BenchmarkMode(Mode.AverageTime)
45+
@OutputTimeUnit(TimeUnit.MICROSECONDS)
46+
@Fork(value = 3, jvmArgsAppend = {"-Xms1g", "-Xmx1g"})
47+
@Warmup(iterations = 4, time = 300, timeUnit = TimeUnit.MILLISECONDS)
48+
@Measurement(iterations = 7, time = 400, timeUnit = TimeUnit.MILLISECONDS)
49+
@State(Scope.Thread)
50+
public class LeanTokenModeBenchmark {
51+
@Param({"false", "true"})
52+
public boolean leanTokenMode;
53+
54+
@Param({"typical", "commentHeavy", "wideSelect", "hinted"})
55+
public String workload;
56+
57+
private final DorisSqlParser facade = new DorisSqlParser();
58+
private String sql;
59+
60+
@Setup(Level.Trial)
61+
public void setUp() {
62+
switch (workload) {
63+
case "typical":
64+
sql = "SELECT a, b, c FROM t WHERE a > 1 AND b < 10 ORDER BY c LIMIT 20";
65+
break;
66+
case "commentHeavy":
67+
sql = IntStream.range(0, 32)
68+
.mapToObj(index -> "c" + index + " -- column " + index + "\n")
69+
.collect(Collectors.joining(", ", "SELECT ", "FROM t WHERE c0 > 0"));
70+
break;
71+
case "wideSelect":
72+
sql = IntStream.range(0, 128)
73+
.mapToObj(index -> "c" + index + " AS alias_" + index)
74+
.collect(Collectors.joining(", ", "SELECT ", " FROM wide_table"));
75+
break;
76+
case "hinted":
77+
sql = "SELECT /*+ ORDERED */ a, b FROM t1 JOIN t2 ON t1.id = t2.id WHERE a > 1";
78+
break;
79+
default:
80+
throw new IllegalArgumentException("Unknown workload: " + workload);
81+
}
82+
}
83+
84+
@Benchmark
85+
public Object tokenize() {
86+
DorisLexer lexer = newLexer();
87+
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
88+
tokenStream.fill();
89+
return tokenStream.getTokens();
90+
}
91+
92+
@Benchmark
93+
public Object parseStatement() {
94+
DorisParser parser = facade.newParser(newLexer());
95+
parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
96+
return parser.singleStatement();
97+
}
98+
99+
private DorisLexer newLexer() {
100+
DorisLexer lexer = facade.newLexer(sql);
101+
lexer.isLeanTokenMode = leanTokenMode;
102+
return lexer;
103+
}
104+
}
Collapse file

‎fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4‎

Copy file name to clipboardExpand all lines: fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+9-2Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ lexer grammar DorisLexer;
2121

2222
@members {
2323
public boolean isNoBackslashEscapes = false;
24+
public boolean isLeanTokenMode = false;
25+
26+
private void skipInLeanTokenMode() {
27+
if (isLeanTokenMode) {
28+
skip();
29+
}
30+
}
2431

2532
/**
2633
* Verify whether current token is a valid decimal token (which contains dot).
@@ -748,7 +755,7 @@ fragment LETTER
748755
;
749756

750757
SIMPLE_COMMENT
751-
: '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
758+
: '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? {skipInLeanTokenMode();} -> channel(HIDDEN)
752759
;
753760

754761
BRACKETED_COMMENT
@@ -757,7 +764,7 @@ BRACKETED_COMMENT
757764

758765

759766
WS
760-
: [ \r\n\t]+ -> channel(HIDDEN)
767+
: [ \r\n\t]+ {skipInLeanTokenMode();} -> channel(HIDDEN)
761768
;
762769

763770
// Catch-all for anything we can't recognize.

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.