Line data Source code
1 : //! The SQL scanner.
2 : //!
3 : //! The rules of every flex-style start condition are a logos lexer: the longest match wins, and
4 : //! logos priorities break ties. The lexer of the current start condition matches one token at a
5 : //! time, so that actions can switch the start condition and put text back with `yyless`. See
6 : //! docs/design/core_quirks.md for the quirks of the scanner.
7 : //!
8 : //! [`scan`] then applies a lookahead filter, which turns `NOT`, `NULLS`, `WITH` and `.` into their
9 : //! `*_LA` variants depending on the following tokens.
10 :
11 : use crate::keywords::Keyword;
12 : use crate::names::{NameId, NameRegistry};
13 : use crate::span::TextSpan;
14 : use crate::{symbol_kind::*, SymbolKind};
15 : use logos::Logos;
16 : use std::collections::VecDeque;
17 :
18 : /// The semantic value of a scanned symbol
19 : #[derive(Clone, Copy, Debug, PartialEq, Eq)]
20 : pub enum SymbolValue {
21 : None,
22 : /// An identifier
23 : Name(NameId),
24 : /// The text of a keyword
25 : Text(&'static str),
26 : }
27 :
28 : /// A scanned symbol
29 : #[derive(Clone, Copy, Debug, PartialEq, Eq)]
30 : pub struct Symbol {
31 : /// The terminal symbol kind
32 : pub kind: SymbolKind,
33 : /// The byte span in the text
34 : pub location: TextSpan,
35 : /// The semantic value
36 : pub value: SymbolValue,
37 : }
38 :
39 : impl Symbol {
40 150905 : fn new(kind: SymbolKind, location: TextSpan) -> Self {
41 150905 : Self { kind, location, value: SymbolValue::None }
42 150905 : }
43 : }
44 :
45 : /// A literal that Hyper replaced with a typed marker comment, e.g. `/*Integer(4888)*/`
46 : #[derive(Clone, Debug, PartialEq, Eq)]
47 : pub struct ObfuscatedLiteral {
48 : /// The location of the marker
49 : pub location: TextSpan,
50 : /// An executable literal derived from the marker hash, e.g. `18568`
51 : pub replacement: String,
52 : }
53 :
54 : /// A scanner error
55 : #[derive(Clone, Debug, PartialEq, Eq)]
56 : pub struct ScannerError {
57 : /// The byte span of the error
58 : pub location: TextSpan,
59 : /// The error message
60 : pub message: String,
61 : }
62 :
63 : /// The output of the scanner
64 : #[derive(Clone, Debug, Default)]
65 : pub struct ScannedScript {
66 : /// The origin id
67 : pub external_id: u32,
68 : /// The text version
69 : pub text_version: u64,
70 : /// The text
71 : pub text: String,
72 : /// The scanner errors
73 : pub errors: Vec<ScannerError>,
74 : /// The line breaks (`\n`, `\r\n` or `\r`), including those in comments and literals
75 : pub line_breaks: Vec<TextSpan>,
76 : /// The comments
77 : pub comments: Vec<TextSpan>,
78 : /// The obfuscated literals, ordered by offset
79 : pub obfuscated_literals: Vec<ObfuscatedLiteral>,
80 : /// The names
81 : pub names: NameRegistry,
82 : /// The symbols, ending with EOF
83 : pub symbols: Vec<Symbol>,
84 : }
85 :
86 : /// Scan a script.
87 7465 : pub fn scan(text: &str, text_version: u64, external_id: u32) -> ScannedScript {
88 7465 : let mut out = ScannedScript {
89 7465 : external_id,
90 7465 : text_version,
91 7465 : text: text.to_string(),
92 7465 : symbols: Vec::with_capacity(text.len() / 4 + 8),
93 7465 : ..Default::default()
94 7465 : };
95 7465 : out.line_breaks = line_breaks(text.as_bytes());
96 7465 : let mut lexer = Lexer::new(text.as_bytes());
97 7465 : let mut lookahead: VecDeque<Symbol> = VecDeque::new();
98 : loop {
99 340856 : let symbol = next_symbol(&mut lexer, &mut out, &mut lookahead);
100 340856 : out.symbols.push(symbol);
101 340856 : if symbol.kind == S_YYEOF {
102 7465 : break;
103 333391 : }
104 : }
105 7465 : out
106 7465 : }
107 :
108 : /// Find the line breaks of a text, `\r\n` is one line break.
109 7465 : fn line_breaks(text: &[u8]) -> Vec<TextSpan> {
110 7465 : let mut breaks = Vec::new();
111 7465 : let mut i = 0;
112 1970699 : while i < text.len() {
113 1963234 : match text[i] {
114 2 : b'\r' if text.get(i + 1) == Some(&b'\n') => {
115 1 : breaks.push(Lexer::span(i, 2));
116 1 : i += 1;
117 1 : }
118 38867 : b'\n' | b'\r' => breaks.push(Lexer::span(i, 1)),
119 1924366 : _ => {}
120 : }
121 1963234 : i += 1;
122 : }
123 7465 : breaks
124 7465 : }
125 :
126 : /// The lookahead filter of [`scan`].
127 340856 : fn next_symbol(lexer: &mut Lexer, out: &mut ScannedScript, lookahead: &mut VecDeque<Symbol>) -> Symbol {
128 340856 : let current = match lookahead.pop_front() {
129 14533 : Some(symbol) => symbol,
130 326323 : None => lexer.lex(out),
131 : };
132 340856 : match current.kind {
133 14533 : S_NOT | S_NULLS_P | S_WITH | S_DOT => {}
134 326323 : _ => return current,
135 : }
136 14533 : let next = lexer.lex(out);
137 14533 : lookahead.push_back(next);
138 14533 : match (current.kind, next.kind) {
139 90 : (S_NOT, S_BETWEEN | S_IN_P | S_LIKE | S_ILIKE | S_GLOB | S_SIMILAR) => Symbol::new(S_NOT_LA, current.location),
140 1457 : (S_NULLS_P, S_FIRST_P | S_LAST_P) => Symbol::new(S_NULLS_LA, current.location),
141 36 : (S_WITH, S_TIME | S_ORDINALITY) => Symbol::new(S_WITH_LA, current.location),
142 26 : (S_DOT, S_STAR) => Symbol::new(S_DOT_STAR_LA, current.location),
143 12924 : _ => current,
144 : }
145 340856 : }
146 :
147 : /// The flex start conditions
148 : #[derive(Clone, Copy, Debug, PartialEq, Eq)]
149 : enum State {
150 : Initial,
151 : /// Bit string literal
152 : Xb,
153 : /// Backtick-quoted identifier
154 : Xbt,
155 : /// C-style comment
156 : Xc,
157 : /// Double-quoted identifier
158 : Xd,
159 : /// Hexadecimal string literal
160 : Xh,
161 : /// Quoted string
162 : Xq,
163 : }
164 :
165 : // The rules of every start condition form a logos lexer. Logos takes the longest match like flex,
166 : // and resolves a tie by priority. The default priority of a pattern is 2 for every byte or class it
167 : // has to match, which already ranks `{comment}`, `{xcstart}` and the two-character operators above
168 : // `{operator}`. The explicit priorities rank the remaining ties.
169 : //
170 : // Like flex, the lexers match bytes, not characters (`utf8 = false` and byte-string patterns), so
171 : // `\x80-\xFF` are identifier bytes.
172 :
173 : /// The rules of INITIAL
174 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
175 : #[logos(utf8 = false)]
176 : #[logos(subpattern space = br"[ \t\n\r\f]")]
177 : #[logos(subpattern op_chars = br"[~!@#^&|+\-*/%<>=]")]
178 : #[logos(subpattern integer = br"[0-9]+")]
179 : #[logos(subpattern decimal = br"[0-9]*\.[0-9]+|[0-9]+\.[0-9]*")]
180 : enum Initial {
181 : /// `{space}+`
182 : #[regex(br"(?&space)+")]
183 : Space,
184 : /// `{comment}`
185 : #[regex(br"--[^\n\r]*", allow_greedy = true)]
186 : Comment,
187 : /// The Hyper obfuscation markers, case-sensitive unlike the other rules
188 : #[regex(br"/\*String\([0-9A-F]{4}\)\*/", |_| S_SCONST)]
189 : #[regex(br"/\*Integer\([0-9A-F]{4}\)\*/", |_| S_ICONST)]
190 : #[regex(br"/\*Float\([0-9A-F]{4}\)\*/", |_| S_FCONST)]
191 : #[regex(br"/\*HexString\([0-9A-F]{4}\)\*/", |_| S_XCONST)]
192 : #[regex(br"/\*BitString\([0-9A-F]{4}\)\*/", |_| S_BCONST)]
193 : ObfuscatedLiteral(SymbolKind),
194 : /// `{xcstart}`: matches as much as `{operator}` would, the action puts the operator back
195 : #[regex(br"/\*(?&op_chars)*")]
196 : CommentStart,
197 : /// `{xbstart}`, `{xhstart}`, `{xqstart}`, `{xdstart}`, `{xbtstart}`
198 : #[regex(b"[bB]'", |_| State::Xb)]
199 : #[regex(b"[xX]'", |_| State::Xh)]
200 : #[token("'", |_| State::Xq)]
201 : #[token("\"", |_| State::Xd)]
202 : #[token("`", |_| State::Xbt)]
203 : Begin(State),
204 : /// The operators and punctuation with a token of their own
205 : #[token("::", |_| S_TYPECAST)]
206 : #[token("..", |_| S_DOT_DOT)]
207 : #[token(".", |_| S_DOT)]
208 : #[token(":=", |_| S_COLON_EQUALS)]
209 : #[token("=>", |_| S_EQUALS_GREATER)]
210 : #[token("<=", |_| S_LESS_EQUALS)]
211 : #[token(">=", |_| S_GREATER_EQUALS)]
212 : #[token("<>", |_| S_NOT_EQUALS)]
213 : #[token("!=", |_| S_NOT_EQUALS)]
214 : #[token(",", |_| S_COMMA)]
215 : #[token("(", |_| S_LRB)]
216 : #[token(")", |_| S_RRB)]
217 : #[token("[", |_| S_LSB)]
218 : #[token("]", |_| S_RSB)]
219 : #[token("{", |_| S_LCB)]
220 : #[token("}", |_| S_RCB)]
221 : #[token(";", |_| S_SEMICOLON)]
222 : #[token(":", |_| S_COLON)]
223 : #[token("+", |_| S_PLUS)]
224 : #[token("-", |_| S_MINUS)]
225 : #[token("/", |_| S_DIVIDE)]
226 : #[token("*", |_| S_STAR)]
227 : #[token("%", |_| S_MODULO)]
228 : #[token("^", |_| S_CIRCUMFLEX)]
229 : #[token("&", |_| S_AMPERSAND)]
230 : #[token("|", |_| S_PIPE)]
231 : #[token("!", |_| S_EXCLAMATION_MARK)]
232 : #[token("<", |_| S_LESS_THAN)]
233 : #[token(">", |_| S_GREATER_THAN)]
234 : #[token("=", |_| S_EQUALS)]
235 : #[token("$", |_| S_DOLLAR)]
236 : Token(SymbolKind),
237 : /// `{raw_dot_trailing}`, the action puts the space back
238 : #[regex(br"\.(?&space)")]
239 : DotTrailing,
240 : /// `{raw_question_mark}`, the action checks the trailing context
241 : #[token("?")]
242 : QuestionMark,
243 : /// `{operator}`, below the single-character tokens
244 : #[regex(br"(?&op_chars)+", priority = 1)]
245 : Operator,
246 : /// `{integer}`
247 : #[regex(br"(?&integer)")]
248 : Integer,
249 : /// `{decimal}`, `{real}`
250 : #[regex(br"(?&decimal)")]
251 : #[regex(br"((?&integer)|(?&decimal))[Ee][-+]?[0-9]+")]
252 : Float,
253 : /// `{decimalfail}`
254 : #[regex(br"(?&integer)\.\.")]
255 : DecimalFail,
256 : /// `{realfail1}`
257 : #[regex(br"((?&integer)|(?&decimal))[Ee]")]
258 : RealFail1,
259 : /// `{realfail2}`
260 : #[regex(br"((?&integer)|(?&decimal))[Ee][-+]")]
261 : RealFail2,
262 : /// `{identifier}`
263 : #[regex(br"[A-Za-z\x80-\xFF_][A-Za-z\x80-\xFF_0-9$]*")]
264 : Identifier,
265 : /// `{other}`, below everything else
266 : #[regex(br"[^\n]", priority = 0)]
267 : Other,
268 : }
269 :
270 : /// The rules of xc
271 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
272 : #[logos(utf8 = false)]
273 : enum Xc {
274 : /// `{xcstart}`
275 : #[regex(br"/\*[~!@#^&|+\-*/%<>=]*")]
276 : CommentStart,
277 : /// `{xcstop}`
278 : #[regex(br"\*+/")]
279 : CommentStop,
280 : /// `{xcinside}`, `{op_chars}`, `\*+`
281 : #[regex(br"[^*/]+")]
282 : #[regex(br"\*+")]
283 : #[token("/")]
284 : Inside,
285 : }
286 :
287 : /// The rules of xb and xh
288 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
289 : #[logos(utf8 = false)]
290 : enum XbXh {
291 : /// The closing quote, the action checks for a continuation (`{quotecontinue}`)
292 : #[token("'")]
293 : QuoteStop,
294 : /// `{xbinside}`, `{xhinside}`
295 : #[regex(br"[^']+")]
296 : Inside,
297 : }
298 :
299 : /// The rules of xq
300 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
301 : #[logos(utf8 = false)]
302 : enum Xq {
303 : /// The closing quote
304 : #[token("'")]
305 : QuoteStop,
306 : /// `{xqdouble}`, `{xqinside}`
307 : #[token("''")]
308 : #[regex(br"[^']+")]
309 : Inside,
310 : }
311 :
312 : /// The rules of xd
313 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
314 : #[logos(utf8 = false)]
315 : enum Xd {
316 : /// `{xddouble}`, `{xdinside}`
317 : #[token("\"\"")]
318 : #[regex(br#"[^"]+"#)]
319 : Inside,
320 : /// `{xdstop}`
321 : #[token("\"")]
322 : Stop,
323 : }
324 :
325 : /// The rules of xbt
326 : #[derive(Logos, Clone, Copy, Debug, PartialEq, Eq)]
327 : #[logos(utf8 = false)]
328 : enum Xbt {
329 : /// `{xbtdouble}`, `{xbtinside}`
330 : #[token("``")]
331 : #[regex(br"[^`]+")]
332 : Inside,
333 : /// `{xbtstop}`
334 : #[token("`")]
335 : Stop,
336 : }
337 :
338 : /// `{ident_cont}`
339 50 : fn is_ident_cont(c: u8) -> bool {
340 50 : c.is_ascii_alphanumeric() || c == b'_' || c == b'$' || c >= 0x80
341 50 : }
342 :
343 : /// The token of a single special character
344 54 : fn special_character(c: u8) -> Option<SymbolKind> {
345 54 : Some(match c {
346 0 : b',' => S_COMMA,
347 0 : b'(' => S_LRB,
348 0 : b')' => S_RRB,
349 0 : b'[' => S_LSB,
350 0 : b']' => S_RSB,
351 0 : b'{' => S_LCB,
352 0 : b'}' => S_RCB,
353 0 : b'.' => S_DOT,
354 0 : b';' => S_SEMICOLON,
355 0 : b':' => S_COLON,
356 17 : b'+' => S_PLUS,
357 0 : b'-' => S_MINUS,
358 3 : b'*' => S_STAR,
359 2 : b'/' => S_DIVIDE,
360 0 : b'%' => S_MODULO,
361 0 : b'?' => S_QUESTION_MARK,
362 0 : b'^' => S_CIRCUMFLEX,
363 0 : b'&' => S_AMPERSAND,
364 0 : b'|' => S_PIPE,
365 0 : b'!' => S_EXCLAMATION_MARK,
366 27 : b'<' => S_LESS_THAN,
367 2 : b'>' => S_GREATER_THAN,
368 3 : b'=' => S_EQUALS,
369 0 : _ => return None,
370 : })
371 54 : }
372 :
373 : /// The two-character operator tokens that also match `{operator}`
374 15 : fn two_char_operator(a: u8, b: u8) -> Option<SymbolKind> {
375 15 : Some(match (a, b) {
376 5 : (b'=', b'>') => S_EQUALS_GREATER,
377 2 : (b'<', b'=') => S_LESS_EQUALS,
378 6 : (b'>', b'=') => S_GREATER_EQUALS,
379 2 : (b'<', b'>') | (b'!', b'=') => S_NOT_EQUALS,
380 0 : _ => return None,
381 : })
382 15 : }
383 :
384 : /// The scanner, switches between the lexers of the start conditions like the flex scanner
385 : struct Lexer<'a> {
386 : /// The script text
387 : text: &'a [u8],
388 : /// The byte offset of the next match
389 : pos: usize,
390 : /// The current start condition
391 : state: State,
392 : /// The start of the current extended token (quoted literal, identifier or comment)
393 : ext_begin: TextSpan,
394 : /// The nesting depth of C-style comments
395 : ext_depth: usize,
396 : /// A buffer for lower-casing and unescaping names
397 : buffer: Vec<u8>,
398 : }
399 :
400 : impl<'a> Lexer<'a> {
401 7465 : fn new(text: &'a [u8]) -> Self {
402 7465 : Self {
403 7465 : text,
404 7465 : pos: 0,
405 7465 : state: State::Initial,
406 7465 : ext_begin: TextSpan::default(),
407 7465 : ext_depth: 0,
408 7465 : buffer: Vec::with_capacity(64),
409 7465 : }
410 7465 : }
411 :
412 620903 : fn span(begin: usize, length: usize) -> TextSpan {
413 620903 : TextSpan::new(begin as u32, length as u32)
414 620903 : }
415 :
416 : /// The empty location at the end of the text
417 7496 : fn eof_location(&self) -> TextSpan {
418 7496 : Self::span(self.text.len(), 0)
419 7496 : }
420 :
421 : /// Match the rules of a start condition at the current position and consume the match.
422 574538 : fn next_match<T>(&mut self) -> (T, TextSpan)
423 574538 : where
424 574538 : T: Logos<'a, Source = [u8], Extras = ()>,
425 : {
426 574538 : let start = self.pos;
427 574538 : let mut lexer = T::lexer(&self.text[start..]);
428 : // Every byte starts a match in every start condition.
429 574538 : let Some(Ok(token)) = lexer.next() else {
430 0 : unreachable!("no rule matches at {start}")
431 : };
432 574538 : let length = lexer.span().end;
433 574538 : self.pos = start + length;
434 574538 : (token, Self::span(start, length))
435 574538 : }
436 :
437 : /// Scan the next raw token (before the lookahead filter)
438 340856 : fn lex(&mut self, out: &mut ScannedScript) -> Symbol {
439 : loop {
440 582034 : if self.pos >= self.text.len() {
441 7496 : let eof = self.eof_location();
442 7496 : let message = match self.state {
443 7465 : State::Initial => return Symbol::new(S_YYEOF, eof),
444 5 : State::Xc => "unterminated comment",
445 2 : State::Xb => "unterminated bit string literal",
446 2 : State::Xh => "unterminated hexadecimal bit string literal",
447 18 : State::Xq => "unterminated quoted string",
448 2 : State::Xd => "unterminated delimited identifier",
449 2 : State::Xbt => "unterminated backtick-quoted identifier",
450 : };
451 31 : out.errors.push(ScannerError {
452 31 : location: self.ext_begin.cover(eof),
453 31 : message: message.to_string(),
454 31 : });
455 31 : self.state = State::Initial;
456 31 : continue;
457 574538 : }
458 574538 : let symbol = match self.state {
459 : State::Initial => {
460 550592 : let (token, loc) = self.next_match::<Initial>();
461 550592 : self.initial(out, token, loc)
462 : }
463 1778 : State::Xc => match self.next_match::<Xc>() {
464 15 : (Xc::CommentStart, mut loc) => {
465 15 : self.yyless(&mut loc, 2);
466 15 : self.ext_depth += 1;
467 15 : None
468 : }
469 496 : (Xc::CommentStop, loc) => {
470 496 : self.ext_depth -= 1;
471 496 : if self.ext_depth == 0 {
472 481 : out.comments.push(self.ext_begin.cover(loc));
473 481 : self.state = State::Initial;
474 481 : }
475 496 : None
476 : }
477 1267 : (Xc::Inside, _) => None,
478 : },
479 68 : State::Xb | State::Xh => match self.next_match::<XbXh>() {
480 32 : (XbXh::QuoteStop, loc) => {
481 32 : if self.quote_continue(out) {
482 1 : None
483 : } else {
484 31 : Some(self.quote_stop(loc))
485 : }
486 : }
487 36 : (XbXh::Inside, _) => None,
488 : },
489 19912 : State::Xq => match self.next_match::<Xq>() {
490 9912 : (Xq::QuoteStop, loc) => Some(self.quote_stop(loc)),
491 10000 : (Xq::Inside, _) => None,
492 : },
493 1688 : State::Xd => match self.next_match::<Xd>() {
494 824 : (Xd::Stop, loc) => Some(self.identifier_stop(out, loc, b'"', "zero-length delimited identifier")),
495 864 : (Xd::Inside, _) => None,
496 : },
497 500 : State::Xbt => match self.next_match::<Xbt>() {
498 238 : (Xbt::Stop, loc) => {
499 238 : Some(self.identifier_stop(out, loc, b'`', "zero-length backtick-quoted identifier"))
500 : }
501 262 : (Xbt::Inside, _) => None,
502 : },
503 : };
504 574538 : if let Some(symbol) = symbol {
505 333391 : return symbol;
506 241147 : }
507 : }
508 340856 : }
509 :
510 : /// Put back all but the first `n` characters of the current match (`yyless(n)`).
511 591 : fn yyless(&mut self, loc: &mut TextSpan, n: usize) {
512 591 : self.pos = loc.offset as usize + n;
513 591 : loc.length = n as u32;
514 591 : }
515 :
516 : /// Run the action of an INITIAL rule. Returns a symbol if the action returns one.
517 550592 : fn initial(&mut self, out: &mut ScannedScript, token: Initial, mut loc: TextSpan) -> Option<Symbol> {
518 550592 : match token {
519 216123 : Initial::Space => None,
520 : Initial::Comment => {
521 566 : out.comments.push(loc);
522 566 : None
523 : }
524 24 : Initial::ObfuscatedLiteral(kind) => {
525 24 : out.obfuscated_literals.push(obfuscated_literal(self.text, loc));
526 24 : Some(Symbol::new(kind, loc))
527 : }
528 : Initial::CommentStart => {
529 486 : self.yyless(&mut loc, 2);
530 486 : self.ext_begin = loc;
531 486 : self.ext_depth += 1;
532 486 : self.state = State::Xc;
533 486 : None
534 : }
535 11031 : Initial::Begin(state) => {
536 11031 : self.ext_begin = loc;
537 11031 : self.state = state;
538 11031 : None
539 : }
540 112896 : Initial::Token(kind) => Some(Symbol::new(kind, loc)),
541 : Initial::DotTrailing => {
542 17 : self.yyless(&mut loc, 1);
543 17 : Some(Symbol::new(S_DOT_TRAILING, loc))
544 : }
545 : // A standalone ? is an unnamed parameter. ?name, ?1 and ?"name" are not parameters of Hyper,
546 : // the grammar has no rule for QUESTION_MARK, so they are syntax errors.
547 52 : Initial::QuestionMark => match self.text.get(self.pos) {
548 50 : Some(&c) if is_ident_cont(c) || c == b'"' => Some(Symbol::new(S_QUESTION_MARK, loc)),
549 48 : _ => Some(Symbol::new(S_PARAM, loc)),
550 : },
551 225 : Initial::Operator => Some(self.operator(out, loc)),
552 17729 : Initial::Integer => Some(self.number(loc)),
553 939 : Initial::Float => Some(Symbol::new(S_FCONST, loc)),
554 : Initial::DecimalFail | Initial::RealFail1 | Initial::RealFail2 => {
555 : // Put back the dots, or the exponent marker and its sign
556 4 : let putback = if token == Initial::RealFail1 { 1 } else { 2 };
557 4 : let keep = loc.length as usize - putback;
558 4 : self.yyless(&mut loc, keep);
559 4 : Some(self.number(loc))
560 : }
561 190498 : Initial::Identifier => Some(self.read_identifier(out, loc)),
562 2 : Initial::Other => Some(Symbol::new(S_RAW_CHAR, loc)),
563 : }
564 550592 : }
565 :
566 : /// A number literal: integers that do not fit into an `i64` are FCONST, like in Postgres
567 17733 : fn number(&self, loc: TextSpan) -> Symbol {
568 17733 : let text = &self.text[loc.offset as usize..loc.end() as usize];
569 17733 : let is_integer = utf8(text).parse::<i64>().is_ok();
570 17733 : Symbol::new(if is_integer { S_ICONST } else { S_FCONST }, loc)
571 17733 : }
572 :
573 : /// The closing quote of a string, bit string or hex string literal
574 9943 : fn quote_stop(&mut self, loc: TextSpan) -> Symbol {
575 9943 : let kind = self.literal_kind();
576 9943 : self.state = State::Initial;
577 9943 : Symbol::new(kind, self.ext_begin.cover(loc))
578 9943 : }
579 :
580 : /// `{quotecontinue}`: a bit or hex string continues after whitespace with a line break and
581 : /// another quote. Consumes the whitespace and the quote and records the comments in it.
582 32 : fn quote_continue(&mut self, out: &mut ScannedScript) -> bool {
583 32 : let mut i = self.pos;
584 32 : let mut comments = Vec::new();
585 32 : let mut has_line_break = false;
586 49 : while i < self.text.len() {
587 43 : match self.text[i] {
588 15 : b' ' | b'\t' | b'\x0C' => i += 1,
589 1 : b'\n' | b'\r' => {
590 1 : has_line_break = true;
591 1 : i += 1;
592 1 : }
593 11 : b'-' if self.text.get(i + 1) == Some(&b'-') => {
594 5 : let length = self.text[i..].iter().position(|&c| c == b'\n' || c == b'\r');
595 1 : let length = length.unwrap_or(self.text.len() - i);
596 1 : comments.push(Self::span(i, length));
597 1 : i += length;
598 : }
599 1 : b'\'' if has_line_break => {
600 1 : out.comments.extend(comments);
601 1 : self.pos = i + 1;
602 1 : return true;
603 : }
604 25 : _ => break,
605 : }
606 : }
607 31 : false
608 32 : }
609 :
610 : /// The `{xdstop}` and `{xbtstop}` actions
611 1062 : fn identifier_stop(&mut self, out: &mut ScannedScript, loc: TextSpan, quote: u8, message: &str) -> Symbol {
612 1062 : let loc = self.ext_begin.cover(loc);
613 1062 : if loc.length == 2 {
614 3 : out.errors.push(ScannerError {
615 3 : location: loc,
616 3 : message: message.to_string(),
617 3 : });
618 1059 : }
619 1062 : self.state = State::Initial;
620 1062 : self.read_quoted_identifier(out, loc, quote)
621 1062 : }
622 :
623 : /// The token kind of the literal that the current start condition scans
624 9943 : fn literal_kind(&self) -> SymbolKind {
625 9943 : match self.state {
626 18 : State::Xb => S_BCONST,
627 13 : State::Xh => S_XCONST,
628 9912 : _ => S_SCONST,
629 : }
630 9943 : }
631 :
632 1062 : fn read_quoted_identifier(&mut self, out: &mut ScannedScript, loc: TextSpan, quote: u8) -> Symbol {
633 1062 : let text = &self.text[loc.offset as usize..loc.end() as usize];
634 1062 : let inner = &text[1..text.len() - 1];
635 1062 : let name = if inner.contains("e) {
636 33 : let name = &mut self.buffer;
637 33 : name.clear();
638 33 : let mut i = 0;
639 222 : while i < inner.len() {
640 189 : name.push(inner[i]);
641 189 : if inner[i] == quote && i + 1 < inner.len() && inner[i + 1] == quote {
642 33 : i += 1;
643 156 : }
644 189 : i += 1;
645 : }
646 33 : &self.buffer[..]
647 : } else {
648 1029 : inner
649 : };
650 1062 : let id = out.names.register(utf8(name), loc);
651 1062 : Symbol {
652 1062 : kind: S_IDENT,
653 1062 : location: loc,
654 1062 : value: SymbolValue::Name(id),
655 1062 : }
656 1062 : }
657 :
658 190498 : fn read_identifier(&mut self, out: &mut ScannedScript, loc: TextSpan) -> Symbol {
659 190498 : let text = &self.text[loc.offset as usize..loc.end() as usize];
660 190498 : self.buffer.clear();
661 190498 : self.buffer.extend(text.iter().map(u8::to_ascii_lowercase));
662 190498 : let lower = &self.buffer[..];
663 190498 : if let Some(keyword) = Keyword::find(lower) {
664 91539 : return Symbol {
665 91539 : kind: keyword.symbol,
666 91539 : location: loc,
667 91539 : value: SymbolValue::Text(keyword.name),
668 91539 : };
669 98959 : }
670 98959 : let id = out.names.register(utf8(lower), loc);
671 98959 : Symbol {
672 98959 : kind: S_IDENT,
673 98959 : location: loc,
674 98959 : value: SymbolValue::Name(id),
675 98959 : }
676 190498 : }
677 :
678 : /// The `{operator}` action
679 225 : fn operator(&mut self, out: &mut ScannedScript, mut loc: TextSpan) -> Symbol {
680 225 : let start = loc.offset as usize;
681 225 : let yyleng = loc.length as usize;
682 225 : let text = &self.text[start..start + yyleng];
683 :
684 : // Embedded slash-star or dash-dash start a comment, the operator stops there.
685 832 : let find = |pattern: &[u8]| text.windows(2).position(|w| w == pattern);
686 225 : let mut nchars = yyleng;
687 225 : if let Some(i) = [find(b"/*"), find(b"--")].into_iter().flatten().min() {
688 8 : nchars = i;
689 217 : }
690 : // '+' and '-' cannot end a multi-char operator unless it contains chars that are not in
691 : // SQL operators.
692 294 : while nchars > 1 && matches!(text[nchars - 1], b'+' | b'-') {
693 110 : if text[..nchars - 1].iter().any(|c| b"~!@#^&|?%".contains(c)) {
694 10 : break;
695 69 : }
696 69 : nchars -= 1;
697 : }
698 225 : if nchars < yyleng {
699 69 : self.yyless(&mut loc, nchars);
700 69 : if nchars == 1 {
701 54 : if let Some(kind) = special_character(text[0]) {
702 54 : return Symbol::new(kind, loc);
703 0 : }
704 15 : }
705 15 : if nchars == 2 {
706 15 : if let Some(kind) = two_char_operator(text[0], text[1]) {
707 15 : return Symbol::new(kind, loc);
708 0 : }
709 0 : }
710 156 : }
711 156 : if nchars >= 64 {
712 2 : out.errors.push(ScannerError {
713 2 : location: loc,
714 2 : message: "operator too long: operators longer than 64 bytes are not supported".to_string(),
715 2 : });
716 154 : }
717 156 : Symbol::new(S_Op, loc)
718 225 : }
719 : }
720 :
721 : /// Derive an executable literal from the hash of a marker.
722 24 : fn obfuscated_literal(text: &[u8], location: TextSpan) -> ObfuscatedLiteral {
723 24 : let marker = &text[location.offset as usize..location.end() as usize];
724 240 : let open = marker.iter().position(|&c| c == b'(').unwrap();
725 24 : let kind = &marker[2..open];
726 24 : let hash = std::str::from_utf8(&marker[open + 1..open + 5]).unwrap();
727 24 : let value = u32::from_str_radix(hash, 16).unwrap();
728 24 : let replacement = match kind {
729 24 : b"String" => format!("'{hash}'"),
730 22 : b"Integer" => value.to_string(),
731 3 : b"Float" => format!("{value}.0"),
732 2 : b"HexString" => format!("X'{hash}'"),
733 : _ => {
734 16 : let bits: String = (0..16).rev().map(|bit| if (value >> bit) & 1 == 1 { '1' } else { '0' }).collect();
735 1 : format!("B'{bits}'")
736 : }
737 : };
738 24 : ObfuscatedLiteral { location, replacement }
739 24 : }
740 :
741 : /// Names are cut out of valid UTF-8 at ASCII boundaries, so they are valid UTF-8 themselves.
742 117754 : fn utf8(bytes: &[u8]) -> &str {
743 117754 : std::str::from_utf8(bytes).expect("names are cut at ASCII boundaries")
744 117754 : }
745 :
746 : #[cfg(test)]
747 : mod tests {
748 : use super::*;
749 :
750 17 : fn kinds(text: &str) -> Vec<SymbolKind> {
751 17 : scan(text, 0, 0).symbols.iter().map(|s| s.kind).collect()
752 17 : }
753 :
754 4 : fn spans(text: &str) -> Vec<(u32, u32)> {
755 15 : scan(text, 0, 0).symbols.iter().map(|s| (s.location.offset, s.location.length)).collect()
756 4 : }
757 :
758 : #[test]
759 1 : fn scans_keywords_and_identifiers() {
760 1 : let out = scan("SELECT Foo, foo", 0, 0);
761 1 : let kinds: Vec<_> = out.symbols.iter().map(|s| s.kind).collect();
762 1 : assert_eq!(kinds, vec![S_SELECT, S_IDENT, S_COMMA, S_IDENT, S_YYEOF]);
763 1 : assert_eq!(out.symbols[0].value, SymbolValue::Text("select"));
764 1 : assert_eq!(out.symbols[1].value, SymbolValue::Name(0));
765 1 : assert_eq!(out.symbols[3].value, SymbolValue::Name(0));
766 1 : assert_eq!(out.names.get(0).text, "foo");
767 1 : assert_eq!(out.names.get(0).occurrences, 2);
768 1 : assert_eq!(out.symbols[4].location, TextSpan::new(15, 0));
769 1 : }
770 :
771 : #[test]
772 1 : fn scans_numbers() {
773 1 : assert_eq!(kinds("1..2"), vec![S_ICONST, S_DOT_DOT, S_ICONST, S_YYEOF]);
774 1 : assert_eq!(spans("1..2"), vec![(0, 1), (1, 2), (3, 1), (4, 0)]);
775 1 : assert_eq!(kinds("1e5 1.5 .5 1. 1e"), vec![S_FCONST, S_FCONST, S_FCONST, S_FCONST, S_ICONST, S_IDENT, S_YYEOF]);
776 1 : assert_eq!(kinds("1.5e+"), vec![S_FCONST, S_IDENT, S_PLUS, S_YYEOF]);
777 : // Integers that do not fit into an i64 are FCONST
778 1 : assert_eq!(kinds("9223372036854775807 9223372036854775808"), vec![S_ICONST, S_FCONST, S_YYEOF]);
779 1 : }
780 :
781 : #[test]
782 1 : fn scans_trailing_dots() {
783 1 : assert_eq!(kinds("a. b"), vec![S_IDENT, S_DOT_TRAILING, S_IDENT, S_YYEOF]);
784 1 : assert_eq!(spans("a. b")[1], (1, 1));
785 1 : let out = scan("a.\n", 0, 0);
786 1 : assert_eq!(out.line_breaks, vec![TextSpan::new(2, 1)]);
787 1 : }
788 :
789 : #[test]
790 1 : fn trims_trailing_signs_and_comments_from_operators() {
791 : // Like Postgres, the operator keeps the characters before the comment start or trailing sign.
792 1 : assert_eq!(kinds("+-- foo"), vec![S_PLUS, S_YYEOF]);
793 1 : assert_eq!(spans("+-- foo")[0], (0, 1));
794 1 : assert_eq!(kinds("=- <=> !="), vec![S_EQUALS, S_MINUS, S_Op, S_NOT_EQUALS, S_YYEOF]);
795 1 : assert_eq!(kinds("a>=-1"), vec![S_IDENT, S_GREATER_EQUALS, S_MINUS, S_ICONST, S_YYEOF]);
796 1 : assert_eq!(spans("a>=-1")[1], (1, 2));
797 1 : assert_eq!(kinds("a=>-1"), vec![S_IDENT, S_EQUALS_GREATER, S_MINUS, S_ICONST, S_YYEOF]);
798 1 : assert_eq!(kinds("1 +/* c */ 2"), vec![S_ICONST, S_PLUS, S_ICONST, S_YYEOF]);
799 1 : assert_eq!(kinds("*/*"), vec![S_STAR, S_YYEOF]);
800 1 : }
801 :
802 : #[test]
803 1 : fn applies_the_lookahead_filter() {
804 1 : assert_eq!(kinds("not in"), vec![S_NOT_LA, S_IN_P, S_YYEOF]);
805 1 : assert_eq!(kinds("not glob"), vec![S_NOT_LA, S_GLOB, S_YYEOF]);
806 1 : assert_eq!(kinds("not foo"), vec![S_NOT, S_IDENT, S_YYEOF]);
807 1 : assert_eq!(kinds("a.*"), vec![S_IDENT, S_DOT_STAR_LA, S_STAR, S_YYEOF]);
808 1 : }
809 :
810 : #[test]
811 1 : fn scans_literals_and_comments() {
812 1 : let out = scan("'a' --c\n-1", 0, 0);
813 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 3));
814 1 : assert_eq!(out.line_breaks, vec![TextSpan::new(7, 1)]);
815 1 : assert_eq!(out.comments, vec![TextSpan::new(4, 3)]);
816 1 : let out = scan("'abc' -- c", 0, 0);
817 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 5));
818 1 : assert_eq!(out.comments, vec![TextSpan::new(7, 4)]);
819 1 : let out = scan("'a''b' ", 0, 0);
820 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 6));
821 1 : let out = scan("/* a /* b */ c */ 1", 0, 0);
822 1 : assert_eq!(out.comments, vec![TextSpan::new(0, 17)]);
823 1 : let out = scan("'unterminated", 0, 0);
824 1 : assert_eq!(out.errors[0].location, TextSpan::new(0, 13));
825 1 : assert_eq!(out.errors[0].message, "unterminated quoted string");
826 1 : }
827 :
828 : #[test]
829 1 : fn continues_bit_and_hex_strings() {
830 1 : let out = scan("x'AB' -- c\n 'CD' 1", 0, 0);
831 1 : assert_eq!(out.symbols.iter().map(|s| s.kind).collect::<Vec<_>>(), vec![S_XCONST, S_ICONST, S_YYEOF]);
832 1 : assert_eq!(out.symbols[0].location, TextSpan::new(0, 17));
833 1 : assert_eq!(out.comments, vec![TextSpan::new(6, 4)]);
834 : // Without a line break, the next quote starts another literal
835 1 : assert_eq!(kinds("b'01' '1'"), vec![S_BCONST, S_SCONST, S_YYEOF]);
836 : // Strings do not continue
837 1 : assert_eq!(kinds("'a'\n'b'"), vec![S_SCONST, S_SCONST, S_YYEOF]);
838 1 : }
839 :
840 : #[test]
841 1 : fn records_line_breaks() {
842 1 : let out = scan("a\r\nb\rc\n'd\ne' -- f\n", 0, 0);
843 5 : let breaks: Vec<_> = out.line_breaks.iter().map(|s| (s.offset, s.length)).collect();
844 1 : assert_eq!(breaks, vec![(1, 2), (4, 1), (6, 1), (9, 1), (17, 1)]);
845 1 : }
846 :
847 : #[test]
848 1 : fn unescapes_quoted_identifiers() {
849 1 : let out = scan(r#"select "trailing""", """leading", "a""b", `trailing```, `` `a``b`"#, 0, 0);
850 5 : for name in ["trailing\"", "\"leading", "a\"b", "trailing`", "a`b"] {
851 5 : assert!(out.names.find(name).is_some(), "{name}");
852 : }
853 1 : }
854 :
855 : #[test]
856 1 : fn records_obfuscated_literals() {
857 1 : let out = scan("/*Integer(1A2B)*/", 0, 0);
858 1 : assert_eq!(out.symbols[0].kind, S_ICONST);
859 1 : assert_eq!(out.obfuscated_literals[0].replacement, "6699");
860 1 : }
861 : }
|
