Line data Source code
1 : //! FlatBuffer packing of scanned, parsed and analyzed scripts, tokens, cursors and completions.
2 : //!
3 : //! The parsed script leaves out empty vectors and strings, the analyzed script always writes its
4 : //! vectors.
5 :
6 : // Bison names some symbol kinds in mixed case (S_Op)
7 : #![allow(non_upper_case_globals)]
8 :
9 : use crate::analyzer::*;
10 : use crate::catalog::*;
11 : use dashql_buffers::algebra::{
12 : BinaryExpressionT, ColumnRefExpressionT, ComparisonT, ConstIntervalCastT, ExpressionSubTypeT, ExpressionT,
13 : FunctionCallExpressionT, LiteralT, ResolvedColumnT,
14 : };
15 : use dashql_buffers::analyzer::{
16 : AnalyzedScriptT, AnalyzerErrorT, ConstantExpressionT, FunctionDeclarationT, FunctionParamT, IndexedColumnReferenceT,
17 : IndexedTableReferenceT, InferredColumnT, InferredTableSchemaT, InsertStatementT, InsertTargetColumnT, NameScopeT,
18 : ParameterKind as PackedParameterKind, ParameterT, SqlTypeKind as PackedSqlTypeKind, SqlTypeT,
19 : QualifiedColumnNameT, QualifiedFunctionNameT, QualifiedTableNameT, ResolvedTableT, TableColumnT,
20 : TableReferenceRole, TableReferenceT, TableT, VisualizationSpecT,
21 : };
22 : use dashql_buffers::parser::{
23 : AttachDatabaseStatementT, AttributeKey, CommentBlockT, ErrorT, Node, NodeT, NodeType,
24 : ParsedScriptT, ScannedScriptT, ScannerTokenType, ScannerTokensT, StatementT, StatementTargetT, StatementTargetType,
25 : StatementType, SymbolSpan, SymbolSpanT, TextSpanT,
26 : };
27 : use dashql_buffers::visualization::{DashboardSpecT, TableSpecT};
28 : use dashql_parser::keywords::Keyword;
29 : use dashql_parser::parser::context::resolve_text_span;
30 : use dashql_parser::parser::ParsedScript;
31 : use dashql_parser::scanner::{ScannedScript, Symbol};
32 : use dashql_parser::span::TextSpan;
33 : use dashql_parser::symbol_kind::*;
34 :
35 : /// Build a FlatBuffers object API value, which are non-exhaustive: start from the default and
36 : /// assign the listed fields
37 : macro_rules! build {
38 : ($ty:ident { $($field:ident $(: $value:expr)?),* $(,)? }) => {{
39 : let mut v = $ty::default();
40 : $( build!(@set v, $field $(, $value)?); )*
41 : v
42 : }};
43 : (@set $v:ident, $field:ident, $value:expr) => { $v.$field = $value; };
44 : (@set $v:ident, $field:ident) => { $v.$field = $field; };
45 : }
46 :
47 : /// A vector, left out when empty
48 1933 : fn non_empty<T>(v: Vec<T>) -> Option<Vec<T>> {
49 1933 : (!v.is_empty()).then_some(v)
50 1933 : }
51 :
52 : /// A string, left out when empty
53 3577 : fn non_empty_str(s: &str) -> Option<String> {
54 3577 : (!s.is_empty()).then(|| s.to_string())
55 3577 : }
56 :
57 99 : fn text_span(span: TextSpan) -> TextSpanT {
58 99 : build!(TextSpanT { offset: span.offset, length: span.length })
59 99 : }
60 :
61 396 : fn symbol_span(span: SymbolSpan) -> SymbolSpanT {
62 396 : build!(SymbolSpanT { offset: span.offset(), length: span.length() })
63 396 : }
64 :
65 : /// The index of a node in its AST
66 1129 : fn n_index(nodes: &[Node], n: &Node) -> usize {
67 1129 : (n as *const Node as usize - nodes.as_ptr() as usize) / std::mem::size_of::<Node>()
68 1129 : }
69 :
70 6 : fn comment_block(b: &dashql_parser::comment_blocks::CommentBlock) -> CommentBlockT {
71 6 : build!(CommentBlockT {
72 6 : block_type: b.block_type,
73 6 : text_span: text_span(b.text_span),
74 6 : tag_span: text_span(b.tag_span),
75 6 : comments_begin: b.comments_begin,
76 6 : comments_count: b.comments_count,
77 6 : lines_begin: b.lines_begin,
78 6 : lines_count: b.lines_count,
79 : })
80 6 : }
81 :
82 178 : fn node(n: &Node) -> NodeT {
83 178 : n.unpack()
84 178 : }
85 :
86 : /// Pack a scanned script
87 6 : pub fn pack_scanned_script(scanned: &ScannedScript) -> ScannedScriptT {
88 6 : build!(ScannedScriptT {
89 6 : external_id: scanned.external_id,
90 6 : line_breaks: non_empty(scanned.line_breaks.iter().map(|&s| text_span(s)).collect()),
91 6 : errors: non_empty(
92 6 : scanned
93 6 : .errors
94 6 : .iter()
95 6 : .map(|e| build!(ErrorT {
96 2 : text_span: Some(text_span(e.location)),
97 2 : symbol_span: None,
98 2 : message: non_empty_str(&e.message),
99 2 : hint: None,
100 : }))
101 6 : .collect(),
102 : ),
103 6 : comments: non_empty(scanned.comments.iter().map(|&s| text_span(s)).collect()),
104 : })
105 6 : }
106 :
107 : /// The token type of a scanner symbol
108 2254 : fn token_type(symbol: &Symbol, text: &str) -> ScannerTokenType {
109 2254 : match symbol.kind {
110 0 : S_NULLS_LA | S_NOT_LA | S_WITH_LA => ScannerTokenType::KEYWORD,
111 2254 : kind if Keyword::name_of(kind).is_some() => ScannerTokenType::KEYWORD,
112 18 : S_SCONST => ScannerTokenType::LITERAL_STRING,
113 77 : S_ICONST => ScannerTokenType::LITERAL_INTEGER,
114 3 : S_FCONST => ScannerTokenType::LITERAL_FLOAT,
115 3 : S_BCONST => ScannerTokenType::LITERAL_BINARY,
116 3 : S_XCONST => ScannerTokenType::LITERAL_HEX,
117 767 : S_IDENT => ScannerTokenType::IDENTIFIER,
118 2 : S_Op | S_EQUALS_GREATER | S_GREATER_EQUALS | S_LESS_EQUALS | S_NOT_EQUALS => ScannerTokenType::OPERATOR,
119 147 : S_DOT | S_DOT_STAR_LA => ScannerTokenType::DOT,
120 2 : S_DOT_TRAILING => ScannerTokenType::DOT_TRAILING,
121 : _ => {
122 430 : if symbol.location.length == 1 && text.as_bytes().get(symbol.location.offset as usize) == Some(&b'=') {
123 23 : ScannerTokenType::OPERATOR
124 : } else {
125 407 : ScannerTokenType::NONE
126 : }
127 : }
128 : }
129 2254 : }
130 :
131 : /// Pack the tokens for highlighting
132 441 : pub fn pack_tokens(parsed: &ParsedScript) -> ScannerTokensT {
133 441 : let scanned = &parsed.scanned;
134 441 : let symbol_count = scanned.symbols.len();
135 : // Keywords that the parser used as names are identifiers
136 441 : let mut name_overrides = vec![false; symbol_count];
137 3095 : for n in &parsed.nodes {
138 3095 : if n.node_type() == NodeType::NAME && n.symbol_span().length() == 1 {
139 671 : if let Some(slot) = name_overrides.get_mut(n.symbol_span().offset() as usize) {
140 671 : *slot = true;
141 671 : }
142 2424 : }
143 : }
144 441 : let count = symbol_count.saturating_sub(1);
145 441 : let mut offsets = Vec::with_capacity(count);
146 441 : let mut lengths = Vec::with_capacity(count);
147 441 : let mut types = Vec::with_capacity(count);
148 2256 : for (id, symbol) in scanned.symbols[..count].iter().enumerate() {
149 2254 : offsets.push(symbol.location.offset);
150 2254 : lengths.push(symbol.location.length);
151 2254 : let mut t = token_type(symbol, &scanned.text);
152 2254 : if t == ScannerTokenType::KEYWORD && name_overrides[id] {
153 13 : t = ScannerTokenType::IDENTIFIER;
154 2241 : }
155 2254 : types.push(t);
156 : }
157 : // The token index of every line break
158 441 : let mut breaks = Vec::with_capacity(scanned.line_breaks.len());
159 441 : let mut oi = 0;
160 441 : for lb in &scanned.line_breaks {
161 166 : while oi < offsets.len() && offsets[oi] < lb.offset {
162 104 : oi += 1;
163 104 : }
164 62 : breaks.push(oi as u32);
165 : }
166 441 : build!(ScannerTokensT {
167 441 : token_offsets: non_empty(offsets),
168 441 : token_lengths: non_empty(lengths),
169 441 : token_types: non_empty(types),
170 441 : token_breaks: non_empty(breaks),
171 : })
172 441 : }
173 :
174 : /// Pack a statement with its target
175 1141 : fn pack_statement(parsed: &ParsedScript, statement_id: usize) -> StatementT {
176 1141 : let statement = &parsed.statements[statement_id];
177 1141 : let nodes = &parsed.nodes;
178 1141 : let mut out = build!(StatementT {
179 1141 : statement_type: statement.statement_type,
180 1141 : root_node: statement.root,
181 1141 : nodes_begin: statement.nodes_begin,
182 1141 : node_count: statement.node_count,
183 : });
184 1141 : let root = &nodes[statement.root as usize];
185 2286 : fn children<'a>(nodes: &'a [Node], n: &Node) -> &'a [Node] {
186 2286 : let begin = n.children_begin_or_value() as usize;
187 2286 : &nodes[begin..begin + n.children_count() as usize]
188 2286 : }
189 1153 : let find_attribute = |key: AttributeKey| children(nodes, root).iter().find(|c| c.attribute_key() == key);
190 : // Follow the left inputs of set operations
191 1141 : let find_select_attribute = |key: AttributeKey| -> Option<&Node> {
192 16 : let mut current = root;
193 17 : while current.node_type() == NodeType::OBJECT_SQL_SELECT {
194 17 : let mut next = None;
195 34 : for child in children(nodes, current) {
196 34 : if child.attribute_key() == key {
197 16 : return Some(child);
198 18 : }
199 18 : if child.attribute_key() == AttributeKey::SQL_COMBINE_INPUT
200 1 : && child.node_type() == NodeType::ARRAY
201 1 : && child.children_count() > 0
202 1 : {
203 1 : next = Some(&nodes[child.children_begin_or_value() as usize]);
204 17 : }
205 : }
206 1 : current = next?;
207 : }
208 0 : None
209 16 : };
210 1282 : let read_name = |source: Option<&Node>| -> String {
211 1280 : match source {
212 1280 : Some(n) if n.node_type() == NodeType::NAME => {
213 1280 : parsed.scanned.names.get(n.children_begin_or_value()).text.clone()
214 : }
215 0 : _ => String::new(),
216 : }
217 1280 : };
218 1141 : let read_target = |out: &mut StatementT, source: Option<&Node>| {
219 1129 : let mut source: Option<&Node> = source.map(|n| &nodes[n_index(nodes, n)]);
220 1129 : if let Some(s) = source.filter(|s| s.node_type() == NodeType::OBJECT_SQL_TABLEREF) {
221 2 : if let Some(name) = children(nodes, s).iter().find(|c| c.attribute_key() == AttributeKey::SQL_TABLEREF_NAME)
222 2 : {
223 2 : source = Some(name);
224 2 : }
225 1127 : }
226 1129 : let Some(source) = source.filter(|s| s.node_type() == NodeType::ARRAY) else { return };
227 1129 : let names = children(nodes, source);
228 1129 : if names.is_empty() || names.len() > 3 {
229 0 : return;
230 1129 : }
231 1129 : let relation_name = read_name(names.last());
232 1129 : let schema_name = if names.len() >= 2 { read_name(Some(&names[names.len() - 2])) } else { String::new() };
233 1129 : let database_name = if names.len() == 3 { read_name(Some(&names[0])) } else { String::new() };
234 1129 : if !relation_name.is_empty() {
235 1129 : out.target = Some(Box::new(build!(StatementTargetT {
236 1129 : database_name: non_empty_str(&database_name),
237 1129 : schema_name: non_empty_str(&schema_name),
238 1129 : relation_name: non_empty_str(&relation_name),
239 1129 : })));
240 1129 : }
241 1129 : };
242 1141 : let set_target = |out: &mut StatementT, target_type: StatementTargetType, key: AttributeKey| {
243 1113 : out.target_type = target_type;
244 1113 : read_target(out, find_attribute(key));
245 1113 : };
246 1141 : match statement.statement_type {
247 : StatementType::CREATE_TABLE => {
248 901 : set_target(&mut out, StatementTargetType::TABLE, AttributeKey::SQL_CREATE_TABLE_NAME)
249 : }
250 : StatementType::CREATE_TABLE_AS => {
251 65 : set_target(&mut out, StatementTargetType::TABLE, AttributeKey::SQL_CREATE_AS_NAME)
252 : }
253 41 : StatementType::CREATE_VIEW => set_target(&mut out, StatementTargetType::VIEW, AttributeKey::SQL_VIEW_NAME),
254 89 : StatementType::DROP_TABLE => set_target(&mut out, StatementTargetType::TABLE, AttributeKey::SQL_DROP_NAME),
255 15 : StatementType::DROP_VIEW => set_target(&mut out, StatementTargetType::VIEW, AttributeKey::SQL_DROP_NAME),
256 : StatementType::SELECT_INTO => {
257 16 : if let Some(into) =
258 16 : find_select_attribute(AttributeKey::SQL_SELECT_INTO).filter(|n| n.node_type() != NodeType::NONE)
259 : {
260 16 : out.target_type = StatementTargetType::TABLE;
261 16 : if let Some(name) =
262 32 : children(nodes, into).iter().find(|c| c.attribute_key() == AttributeKey::SQL_TEMP_NAME)
263 16 : {
264 16 : read_target(&mut out, Some(name));
265 16 : }
266 0 : }
267 : }
268 3 : StatementType::ATTACH_DATABASE => {
269 3 : out.attach_database = Some(Box::new(build!(AttachDatabaseStatementT {
270 3 : path: non_empty_str(&read_name(find_attribute(AttributeKey::SQL_ATTACH_DATABASE_PATH))),
271 3 : alias: non_empty_str(&read_name(find_attribute(AttributeKey::SQL_ATTACH_DATABASE_ALIAS))),
272 3 : local: find_attribute(AttributeKey::SQL_ATTACH_DATABASE_LOCAL).is_some(),
273 3 : })));
274 3 : }
275 1 : StatementType::INSERT => set_target(&mut out, StatementTargetType::TABLE, AttributeKey::SQL_INSERT_TARGET),
276 1 : StatementType::DELETE => set_target(&mut out, StatementTargetType::TABLE, AttributeKey::SQL_DELETE_TARGET),
277 9 : _ => {}
278 : }
279 1141 : out
280 1141 : }
281 :
282 : /// The statement of a parsed script with its target
283 1121 : pub fn pack_parsed_statement(parsed: &ParsedScript, statement_id: usize) -> StatementT {
284 1121 : pack_statement(parsed, statement_id)
285 1121 : }
286 :
287 : /// Pack a parsed script
288 12 : pub fn pack_parsed_script(parsed: &ParsedScript) -> ParsedScriptT {
289 12 : let scanned = &parsed.scanned;
290 12 : let descriptions = parsed.associate_descriptions();
291 12 : let statements = (0..parsed.statements.len())
292 21 : .map(|i| {
293 20 : let mut statement = pack_statement(parsed, i);
294 20 : let metadata = &descriptions[i];
295 20 : statement.statement_span = Some(text_span(metadata.statement_span));
296 20 : statement.description_begin = metadata.description_begin;
297 20 : statement.description_count = metadata.description_count;
298 20 : statement.description_blocks_begin = metadata.description_blocks_begin;
299 20 : statement.description_blocks_count = metadata.description_blocks_count;
300 20 : statement
301 20 : })
302 12 : .collect();
303 12 : let scanner_errors = scanned
304 12 : .errors
305 12 : .iter()
306 12 : .map(|e| {
307 1 : build!(ErrorT {
308 1 : text_span: Some(text_span(e.location)),
309 1 : symbol_span: None,
310 1 : message: non_empty_str(&e.message),
311 1 : hint: None,
312 : })
313 1 : })
314 12 : .collect();
315 12 : let parser_errors = parsed
316 12 : .errors
317 12 : .iter()
318 12 : .map(|e| {
319 3 : build!(ErrorT {
320 3 : symbol_span: Some(symbol_span(e.location)),
321 3 : text_span: Some(text_span(resolve_text_span(&scanned.text, &scanned.symbols, e.location))),
322 3 : message: non_empty_str(&e.message),
323 3 : hint: non_empty_str(&e.hint),
324 : })
325 3 : })
326 12 : .collect();
327 12 : let blocks = &parsed.comment_blocks;
328 12 : build!(ParsedScriptT {
329 12 : external_id: scanned.external_id,
330 12 : nodes: non_empty(parsed.nodes.iter().map(node).collect()),
331 12 : statements: non_empty(statements),
332 12 : scanner_errors: non_empty(scanner_errors),
333 12 : parser_errors: non_empty(parser_errors),
334 12 : tokens: Some(Box::new(pack_tokens(parsed))),
335 12 : line_breaks: non_empty(scanned.line_breaks.iter().map(|&s| text_span(s)).collect()),
336 12 : comments: non_empty(scanned.comments.iter().map(|&s| text_span(s)).collect()),
337 12 : comment_blocks: non_empty(blocks.blocks.iter().map(comment_block).collect()),
338 12 : comment_lines: non_empty(blocks.lines.iter().map(|&s| text_span(s)).collect()),
339 : })
340 12 : }
341 :
342 : /// A packed name, absent if empty
343 172 : fn name(n: &Name) -> Option<String> {
344 172 : non_empty_str(&n.text)
345 172 : }
346 :
347 40 : fn qualified_table_name(n: &QualifiedTableName) -> QualifiedTableNameT {
348 40 : build!(QualifiedTableNameT {
349 40 : ast_node_id: n.ast_node_id.unwrap_or(PROTO_NULL_U32),
350 40 : database_name: name(&n.database_name),
351 40 : schema_name: name(&n.schema_name),
352 40 : table_name: name(&n.table_name),
353 : })
354 40 : }
355 :
356 27 : fn qualified_column_name(n: &QualifiedColumnName) -> QualifiedColumnNameT {
357 27 : build!(QualifiedColumnNameT {
358 27 : ast_node_id: n.ast_node_id.unwrap_or(PROTO_NULL_U32),
359 27 : table_alias: n.table_alias.as_ref().and_then(name),
360 27 : column_name: name(&n.column_name),
361 : })
362 27 : }
363 :
364 7 : fn qualified_function_name(n: &QualifiedFunctionName) -> QualifiedFunctionNameT {
365 7 : build!(QualifiedFunctionNameT {
366 7 : ast_node_id: n.ast_node_id.unwrap_or(PROTO_NULL_U32),
367 7 : database_name: name(&n.database_name),
368 7 : schema_name: name(&n.schema_name),
369 7 : function_name: name(&n.function_name),
370 : })
371 7 : }
372 :
373 : /// Pack a function declaration
374 3 : pub fn pack_function_declaration(f: &FunctionDeclaration) -> FunctionDeclarationT {
375 3 : build!(FunctionDeclarationT {
376 3 : ast_node_id: f.ast_node_id.unwrap_or(PROTO_NULL_U32),
377 3 : ast_statement_id: f.ast_statement_id.unwrap_or(PROTO_NULL_U32),
378 3 : is_aggregate: f.is_aggregate,
379 3 : function_name: Some(Box::new(qualified_function_name(&f.function_name))),
380 : params: Some(
381 3 : f.params
382 3 : .iter()
383 3 : .map(|p| build!(FunctionParamT {
384 2 : ast_node_id: p.ast_node_id.unwrap_or(PROTO_NULL_U32),
385 2 : param_name: Some(p.param_name.text.to_string()),
386 2 : param_type: Some(p.param_type.to_string()),
387 : }))
388 3 : .collect(),
389 : ),
390 3 : return_type: non_empty_str(&f.return_type),
391 3 : location: f.location.map(text_span),
392 : })
393 3 : }
394 :
395 : /// Pack a table declaration
396 14 : pub fn pack_table(table: &TableDeclaration) -> TableT {
397 14 : let (db, schema) = table.catalog_schema_id.unpack_schema_id();
398 14 : build!(TableT {
399 14 : catalog_table_id: table.table_id().pack(),
400 14 : catalog_database_id: db,
401 14 : catalog_schema_id: schema,
402 14 : ast_node_id: table.ast_node_id.unwrap_or(PROTO_NULL_U32),
403 14 : ast_statement_id: table.ast_statement_id.unwrap_or(PROTO_NULL_U32),
404 14 : ast_scope_root: table.ast_scope_root.unwrap_or(PROTO_NULL_U32),
405 14 : table_name: Some(Box::new(qualified_table_name(&table.table_name))),
406 : // The column names are always written, the column AST nodes are not
407 : table_columns: Some(
408 14 : table
409 14 : .table_columns
410 14 : .iter()
411 20 : .map(|c| build!(TableColumnT {
412 20 : column_name: Some(c.column_name.text.to_string()),
413 20 : location: c.location.map(text_span),
414 : }))
415 14 : .collect(),
416 : ),
417 14 : location: table.location.map(text_span),
418 : })
419 14 : }
420 :
421 18 : fn resolved_column(ids: &ResolvedColumnIds) -> ResolvedColumnT {
422 18 : let (db, schema) = ids.catalog_schema_id.unpack_schema_id();
423 18 : let (table_id, column) = ids.catalog_table_column_id.unpack_table_column_id();
424 18 : build!(ResolvedColumnT {
425 18 : catalog_database_id: db,
426 18 : catalog_schema_id: schema,
427 18 : catalog_table_id: table_id.pack(),
428 18 : column_id: column,
429 18 : referenced_catalog_version: ids.referenced_catalog_version,
430 : })
431 18 : }
432 :
433 59 : fn pack_expression(script: &AnalyzedScript, expr: &Expression) -> ExpressionT {
434 59 : let scanned = &script.parsed.scanned;
435 59 : let inner = match &expr.inner {
436 0 : ExpressionInner::None => ExpressionSubTypeT::NONE,
437 27 : ExpressionInner::ColumnRef(c) => {
438 27 : ExpressionSubTypeT::ColumnRefExpression(Box::new(build!(ColumnRefExpressionT {
439 27 : ast_scope_root: c.ast_scope_root.unwrap_or(u32::MAX),
440 27 : column_name: Some(Box::new(qualified_column_name(&c.column_name))),
441 27 : resolved_column: c.resolved_ids.as_ref().map(|ids| Box::new(resolved_column(ids))),
442 : })))
443 : }
444 25 : ExpressionInner::Literal(l) => {
445 25 : let begin = (l.raw_value.offset as usize).min(scanned.text.len());
446 25 : let end = (begin + l.raw_value.length as usize).min(scanned.text.len());
447 25 : ExpressionSubTypeT::Literal(Box::new(build!(LiteralT {
448 25 : literal_type: l.literal_type,
449 25 : // Always written, even if empty
450 25 : raw_value: Some(scanned.text[begin..end].to_string()),
451 25 : })))
452 : }
453 1 : ExpressionInner::Comparison(c) => ExpressionSubTypeT::Comparison(Box::new(build!(ComparisonT {
454 1 : func: c.func,
455 1 : left_child: c.left_expression_id,
456 1 : right_child: c.right_expression_id,
457 1 : }))),
458 1 : ExpressionInner::BinaryExpression(b) => {
459 1 : ExpressionSubTypeT::BinaryExpression(Box::new(build!(BinaryExpressionT {
460 1 : func: b.func,
461 1 : left_child: b.left_expression_id,
462 1 : right_child: b.right_expression_id,
463 1 : })))
464 : }
465 4 : ExpressionInner::FunctionCall(f) => {
466 4 : let arguments = match f.arguments {
467 1 : FunctionArguments::None => None,
468 3 : FunctionArguments::Generic { begin, count } => Some(
469 3 : script.function_arguments[begin as usize..(begin + count) as usize]
470 3 : .iter()
471 3 : .filter_map(|a| a.expression_id)
472 3 : .collect(),
473 : ),
474 : };
475 4 : ExpressionSubTypeT::FunctionCallExpression(Box::new(build!(FunctionCallExpressionT {
476 4 : func_name: match &f.function_name {
477 4 : FunctionName::Qualified(n) => Some(Box::new(qualified_function_name(n))),
478 0 : FunctionName::Known(_) => None,
479 : },
480 4 : func_call_modifiers: f.function_call_modifiers,
481 : arguments,
482 : })))
483 : }
484 1 : ExpressionInner::ConstIntervalCast(c) => {
485 1 : let mut out = build!(ConstIntervalCastT { value_expression: c.value_expression_id });
486 1 : if let Some(interval) = &c.interval {
487 1 : out.interval_type = interval.interval_type;
488 1 : if let Some(p) = interval.precision_expression {
489 0 : out.interval_precision = p as u32;
490 1 : }
491 0 : }
492 1 : ExpressionSubTypeT::ConstIntervalCast(Box::new(out))
493 : }
494 : };
495 59 : build!(ExpressionT {
496 59 : ast_node_id: expr.ast_node_id,
497 59 : ast_statement_id: expr.ast_statement_id.unwrap_or(PROTO_NULL_U32),
498 59 : symbol_span: expr.location.map(symbol_span),
499 : inner,
500 : })
501 59 : }
502 :
503 : /// The renderer-specific packing of a visualization spec: (vegalite json, markdown spec)
504 : pub type VisualizationPacker = dyn Fn(
505 : &AnalyzedScript,
506 : &VisualizationSpec,
507 : ) -> (
508 : Option<String>,
509 : Option<Box<dashql_buffers::visualization::MarkdownSpecT>>,
510 : );
511 :
512 : /// Pack an analyzed script. `visualize` packs the renderer-specific specs.
513 20 : pub fn pack_analyzed_script(script: &AnalyzedScript, visualize: &VisualizationPacker) -> AnalyzedScriptT {
514 20 : let entry = &script.entry;
515 20 : let tables = entry.table_declarations.iter().map(pack_table).collect();
516 20 : let table_references = script
517 20 : .table_references
518 20 : .iter()
519 21 : .map(|r| {
520 13 : let relation = r.inner.as_ref().expect("relation expression");
521 13 : build!(TableReferenceT {
522 13 : ast_node_id: r.ast_node_id,
523 13 : ast_statement_id: r.ast_statement_id.unwrap_or(PROTO_NULL_U32),
524 13 : symbol_span: r.location.map(symbol_span),
525 13 : table_name: Some(Box::new(qualified_table_name(&relation.table_name))),
526 13 : alias_name: r.alias.as_ref().map(|(a, _)| a.text.to_string()),
527 13 : resolved_table: relation.resolved_table.as_ref().map(|t| {
528 9 : let (db, schema) = t.catalog_schema_id.unpack_schema_id();
529 9 : Box::new(build!(ResolvedTableT {
530 9 : catalog_database_id: db,
531 9 : catalog_schema_id: schema,
532 9 : catalog_table_id: t.catalog_table_id.unpack_table_id().pack(),
533 9 : referenced_catalog_version: t.referenced_catalog_version,
534 9 : table_name: Some(Box::new(qualified_table_name(&t.table_name))),
535 : }))
536 9 : }),
537 13 : role: if r.role == TableRole::Write { TableReferenceRole::WRITE } else { TableReferenceRole::READ },
538 : })
539 13 : })
540 20 : .collect();
541 20 : let insert_statements = script
542 20 : .insert_statements
543 20 : .iter()
544 20 : .map(|insert| {
545 1 : build!(InsertStatementT {
546 1 : ast_node_id: insert.ast_node_id,
547 1 : ast_statement_id: insert.ast_statement_id.unwrap_or(PROTO_NULL_U32),
548 1 : target_table_reference_id: script.table_references[insert.target as usize].table_reference_id.object,
549 1 : source_ast_node_id: insert.source_ast_node_id.unwrap_or(PROTO_NULL_U32),
550 1 : default_values: insert.default_values,
551 1 : returning: insert.returning,
552 : target_columns: Some(
553 1 : insert
554 1 : .target_columns
555 1 : .iter()
556 2 : .map(|c| build!(InsertTargetColumnT {
557 2 : ast_node_id: c.ast_node_id,
558 2 : column_name: Some(c.column_name.text.to_string()),
559 2 : resolved_column: c.resolved.and_then(|loc| {
560 2 : let table = script.table(loc.table)?;
561 2 : let ids = ResolvedColumnIds {
562 2 : catalog_schema_id: table.catalog_schema_id,
563 2 : catalog_table_column_id: table.table_columns[loc.column as usize].object_id,
564 2 : referenced_catalog_version: table.catalog_version,
565 2 : };
566 2 : Some(Box::new(resolved_column(&ids)))
567 2 : }),
568 : }))
569 1 : .collect(),
570 : ),
571 : })
572 1 : })
573 20 : .collect();
574 59 : let expressions = script.expressions.iter().map(|e| pack_expression(script, e)).collect();
575 :
576 : // Index the resolved table and column refs by catalog id, then by their own id
577 20 : let mut table_refs_by_id: Vec<IndexedTableReferenceT> = script
578 20 : .table_references
579 20 : .iter()
580 20 : .enumerate()
581 21 : .filter_map(|(i, r)| {
582 13 : let resolved = r.inner.as_ref()?.resolved_table.as_ref()?;
583 9 : let (db, schema) = resolved.catalog_schema_id.unpack_schema_id();
584 9 : Some(build!(IndexedTableReferenceT {
585 9 : catalog_database_id: db,
586 9 : catalog_schema_id: schema,
587 9 : catalog_table_id: resolved.catalog_table_id.unpack_table_id().pack(),
588 9 : table_reference_id: i as u32,
589 9 : }))
590 13 : })
591 20 : .collect();
592 21 : table_refs_by_id.sort_unstable_by_key(|r| {
593 4 : (r.catalog_database_id, r.catalog_schema_id, r.catalog_table_id, r.table_reference_id)
594 4 : });
595 20 : let mut column_refs_by_id: Vec<IndexedColumnReferenceT> = script
596 20 : .expressions
597 20 : .iter()
598 20 : .enumerate()
599 59 : .filter_map(|(i, e)| {
600 59 : let c = e.column_ref()?;
601 27 : if !c.resolved.is_resolved() {
602 11 : return None;
603 16 : }
604 16 : let ids = c.resolved_ids.as_ref()?;
605 16 : let (db, schema) = ids.catalog_schema_id.unpack_schema_id();
606 16 : let (table_id, column) = ids.catalog_table_column_id.unpack_table_column_id();
607 16 : Some(build!(IndexedColumnReferenceT {
608 16 : catalog_database_id: db,
609 16 : catalog_schema_id: schema,
610 16 : catalog_table_id: table_id.pack(),
611 16 : table_column_id: column,
612 16 : expression_id: i as u32,
613 16 : }))
614 59 : })
615 20 : .collect();
616 33 : column_refs_by_id.sort_unstable_by_key(|r| {
617 24 : (r.catalog_database_id, r.catalog_schema_id, r.catalog_table_id, r.table_column_id, r.expression_id)
618 24 : });
619 :
620 20 : let name_scopes = script
621 20 : .name_scopes
622 20 : .iter()
623 20 : .enumerate()
624 30 : .map(|(i, scope)| {
625 30 : build!(NameScopeT {
626 30 : scope_id: i as u32,
627 30 : ast_node_id: scope.ast_node_id,
628 30 : ast_statement_id: scope.ast_statement_id.unwrap_or(PROTO_NULL_U32),
629 30 : child_scopes: Some(scope.child_scopes.clone()),
630 30 : expressions: Some(scope.expressions.clone()),
631 30 : table_references: Some(scope.table_references.clone()),
632 : })
633 30 : })
634 20 : .collect();
635 20 : let constant_expressions = script
636 20 : .constant_expressions
637 20 : .iter()
638 34 : .map(|c| {
639 29 : let root = &script.expressions[c.root as usize];
640 29 : build!(ConstantExpressionT {
641 29 : ast_node_id: root.ast_node_id,
642 29 : ast_statement_id: root.ast_statement_id.unwrap_or(PROTO_NULL_U32),
643 29 : symbol_span: symbol_span(root.location.unwrap_or_default()),
644 29 : expression_id: root.expression_id,
645 : })
646 29 : })
647 20 : .collect();
648 20 : let parameters = script
649 20 : .parameters
650 20 : .iter()
651 23 : .map(|p| {
652 4 : build!(ParameterT {
653 4 : ast_node_id: p.ast_node_id,
654 4 : ast_statement_id: p.ast_statement_id.unwrap_or(PROTO_NULL_U32),
655 4 : symbol_span: Some(symbol_span(p.location)),
656 4 : kind: match p.kind {
657 3 : ParameterKind::Scalar => PackedParameterKind::SCALAR,
658 1 : ParameterKind::Table => PackedParameterKind::TABLE,
659 : },
660 4 : name: p.name.as_deref().map(str::to_string),
661 4 : cast_type: p.cast_type.as_ref().map(|t| Box::new(pack_sql_type(t))),
662 : })
663 4 : })
664 20 : .collect();
665 20 : let function_declarations = entry.function_declarations.iter().map(pack_function_declaration).collect();
666 20 : let visualization_specs = script
667 20 : .visualization_specs
668 20 : .iter()
669 20 : .map(|spec| {
670 3 : let (vegalite_spec, markdown_spec) = visualize(script, spec);
671 3 : build!(VisualizationSpecT {
672 3 : ast_node_id: spec.ast_node_id,
673 3 : ast_statement_id: spec.ast_statement_id.unwrap_or(PROTO_NULL_U32),
674 3 : renderer: spec.renderer.clone(),
675 : vegalite_spec,
676 : markdown_spec,
677 3 : dashboard: Some(Box::new(build!(DashboardSpecT {
678 3 : row: spec.dashboard.row,
679 3 : column: spec.dashboard.column,
680 3 : width: spec.dashboard.width,
681 3 : height: spec.dashboard.height,
682 3 : }))),
683 3 : table_spec: (spec.renderer.as_deref() == Some("table")).then(|| Box::new(TableSpecT::default())),
684 : })
685 3 : })
686 20 : .collect();
687 20 : let inferred_table_schemas = script
688 20 : .inferred_table_schemas
689 20 : .iter()
690 20 : .map(|schema| {
691 4 : build!(InferredTableSchemaT {
692 4 : table_name: Some(Box::new(qualified_table_name(&schema.table_name))),
693 4 : confidence: dashql_buffers::analyzer::InferenceConfidence(schema.confidence as u8),
694 : columns: Some(
695 4 : schema
696 4 : .columns
697 4 : .iter()
698 7 : .map(|c| build!(InferredColumnT {
699 7 : column_name: Some(c.column_name.text.to_string()),
700 7 : confidence: dashql_buffers::analyzer::InferenceConfidence(c.confidence as u8),
701 7 : candidate_table_names: non_empty(
702 7 : c.candidate_table_names.iter().map(qualified_table_name).collect()
703 : ),
704 : }))
705 4 : .collect(),
706 : ),
707 : })
708 4 : })
709 20 : .collect();
710 20 : let errors = script
711 20 : .errors
712 20 : .iter()
713 20 : .map(|e| {
714 4 : build!(AnalyzerErrorT {
715 4 : error_type: e.error_type,
716 4 : severity: e.severity,
717 4 : ast_node_id: e.ast_node_id.unwrap_or(PROTO_NULL_U32),
718 4 : symbol_span: e.symbol_span.map(symbol_span),
719 4 : text_span: e.text_span.map(text_span),
720 4 : message: Some(e.message.clone()),
721 : })
722 4 : })
723 20 : .collect();
724 20 : build!(AnalyzedScriptT {
725 20 : catalog_entry_id: entry.catalog_entry_id,
726 20 : errors: non_empty(errors),
727 20 : tables: Some(tables),
728 20 : table_references: Some(table_references),
729 20 : insert_statements: Some(insert_statements),
730 20 : resolved_table_references_by_id: Some(table_refs_by_id),
731 20 : expressions: Some(expressions),
732 20 : resolved_column_references_by_id: Some(column_refs_by_id),
733 20 : constant_expressions: Some(constant_expressions),
734 20 : name_scopes: Some(name_scopes),
735 20 : function_declarations: Some(function_declarations),
736 20 : visualization_specs: Some(visualization_specs),
737 20 : inferred_table_schemas: Some(inferred_table_schemas),
738 20 : parameters: non_empty(parameters),
739 : })
740 20 : }
741 :
742 : /// Pack a SQL type
743 21 : pub fn pack_sql_type(t: &SqlType) -> SqlTypeT {
744 21 : let opt = |v: &Option<u32>| v.map_or(-1, |v| v as i32);
745 21 : let mut out = SqlTypeT::default();
746 21 : out.array_dimensions = t.array_dimensions;
747 21 : out.sql = Some(t.to_string());
748 21 : out.kind = match &t.kind {
749 1 : SqlTypeKind::Bool => PackedSqlTypeKind::BOOL,
750 1 : SqlTypeKind::Int1 => PackedSqlTypeKind::INT1,
751 1 : SqlTypeKind::Int2 => PackedSqlTypeKind::INT2,
752 1 : SqlTypeKind::Int4 => PackedSqlTypeKind::INT4,
753 1 : SqlTypeKind::Int8 => PackedSqlTypeKind::INT8,
754 1 : SqlTypeKind::Float4 => PackedSqlTypeKind::FLOAT4,
755 1 : SqlTypeKind::Float8 => PackedSqlTypeKind::FLOAT8,
756 3 : SqlTypeKind::Numeric { precision, scale } => {
757 3 : out.precision = opt(precision);
758 3 : out.scale = opt(scale);
759 3 : PackedSqlTypeKind::NUMERIC
760 : }
761 1 : SqlTypeKind::Varchar { length } => {
762 1 : out.length = opt(length);
763 1 : PackedSqlTypeKind::VARCHAR
764 : }
765 1 : SqlTypeKind::Char { length } => {
766 1 : out.length = opt(length);
767 1 : PackedSqlTypeKind::CHAR
768 : }
769 1 : SqlTypeKind::Text => PackedSqlTypeKind::TEXT,
770 1 : SqlTypeKind::Bit { length, varying } => {
771 1 : out.length = opt(length);
772 1 : out.varying = *varying;
773 1 : PackedSqlTypeKind::BIT
774 : }
775 1 : SqlTypeKind::Date => PackedSqlTypeKind::DATE,
776 1 : SqlTypeKind::Time { precision, with_time_zone } => {
777 1 : out.precision = opt(precision);
778 1 : out.with_time_zone = *with_time_zone;
779 1 : PackedSqlTypeKind::TIME
780 : }
781 1 : SqlTypeKind::Timestamp { precision, with_time_zone } => {
782 1 : out.precision = opt(precision);
783 1 : out.with_time_zone = *with_time_zone;
784 1 : PackedSqlTypeKind::TIMESTAMP
785 : }
786 2 : SqlTypeKind::Interval { fields, precision } => {
787 2 : out.precision = opt(precision);
788 2 : out.interval_fields = fields.map_or(-1, |f| f.0 as i16);
789 2 : PackedSqlTypeKind::INTERVAL
790 : }
791 2 : SqlTypeKind::Generic { name, modifiers } => {
792 2 : out.name = Some(name.to_string());
793 2 : out.modifiers = non_empty(modifiers.iter().map(|m| m.to_string()).collect());
794 2 : PackedSqlTypeKind::GENERIC
795 : }
796 : };
797 21 : out
798 21 : }
799 :
800 : /// Pack a cursor
801 3 : pub fn pack_cursor(cursor: &crate::cursor::ScriptCursor) -> dashql_buffers::cursor::ScriptCursorT {
802 : use crate::cursor::CursorContext;
803 : use dashql_buffers::cursor::{
804 : RelativeSymbolPosition, ScriptCursorColumnRefContextT, ScriptCursorContextT, ScriptCursorT,
805 : ScriptCursorTableRefContextT,
806 : };
807 3 : let mut out = ScriptCursorT::default();
808 3 : out.text_offset = cursor.text_offset as u32;
809 3 : match &cursor.scanner_location {
810 3 : Some(location) => {
811 3 : let target = &location.current;
812 3 : out.scanner_symbol_id = target.symbol_id;
813 3 : out.scanner_relative_position = target.relative_pos;
814 3 : out.scanner_symbol_offset = target.symbol.location.offset;
815 3 : out.scanner_symbol_kind = target.symbol.kind as u32;
816 3 : out.scanner_symbol_completable = crate::completion::is_symbol_kind_completable(target.symbol.kind);
817 3 : }
818 0 : None => {
819 0 : out.scanner_symbol_id = u32::MAX;
820 0 : out.scanner_relative_position = RelativeSymbolPosition::AFTER_SYMBOL;
821 0 : out.scanner_symbol_offset = 0;
822 0 : out.scanner_symbol_kind = 0;
823 0 : out.scanner_symbol_completable = false;
824 0 : }
825 : }
826 3 : out.statement_id = cursor.statement_id.unwrap_or(u32::MAX);
827 3 : out.ast_node_id = cursor.ast_node_id.unwrap_or(u32::MAX);
828 3 : out.ast_path_to_root = non_empty(cursor.ast_path_to_root.clone());
829 3 : out.name_scopes = non_empty(cursor.name_scopes.clone());
830 3 : out.context = match cursor.context {
831 1 : CursorContext::None => ScriptCursorContextT::NONE,
832 1 : CursorContext::TableRef { table_reference_id, .. } => {
833 1 : let mut ctx = ScriptCursorTableRefContextT::default();
834 1 : ctx.table_reference_id = table_reference_id;
835 1 : ScriptCursorContextT::ScriptCursorTableRefContext(Box::new(ctx))
836 : }
837 1 : CursorContext::ColumnRef { expression_id } => {
838 1 : let mut ctx = ScriptCursorColumnRefContextT::default();
839 1 : ctx.expression_id = expression_id;
840 1 : ScriptCursorContextT::ScriptCursorColumnRefContext(Box::new(ctx))
841 : }
842 : // A column reference without an expression
843 : CursorContext::AliasDot => {
844 0 : let mut ctx = ScriptCursorColumnRefContextT::default();
845 0 : ctx.expression_id = PROTO_NULL_U32;
846 0 : ScriptCursorContextT::ScriptCursorColumnRefContext(Box::new(ctx))
847 : }
848 : };
849 3 : out
850 3 : }
851 :
852 : /// Pack a completion
853 12 : pub fn pack_completion(
854 12 : completion: &crate::completion::Completion,
855 12 : upper_case_keywords: bool,
856 12 : ) -> dashql_buffers::completion::CompletionT {
857 : use crate::completion::CandidateObjectRef;
858 : use crate::utils::quote_identifier;
859 : use dashql_buffers::completion::{
860 : CompletionCandidateObjectT, CompletionCandidateObjectType, CompletionCandidateT, CompletionT, CompletionTableParameter,
861 : };
862 : use crate::table_parameters::TableParameterSource;
863 :
864 12 : let candidates = completion
865 12 : .top_candidates
866 12 : .iter()
867 143 : .map(|candidate| {
868 143 : let upper_keyword = upper_case_keywords && candidate.keyword_symbol.is_some();
869 143 : let display_text = if upper_keyword {
870 11 : candidate.completion_text.to_ascii_uppercase()
871 : } else {
872 132 : candidate.completion_text.to_string()
873 : };
874 143 : let mut completion_text = if upper_keyword || candidate.completion_text_is_verbatim {
875 19 : display_text.clone()
876 : } else {
877 124 : quote_identifier(&candidate.completion_text)
878 : };
879 143 : let mut is_function = false;
880 143 : let catalog_objects = candidate
881 143 : .catalog_objects
882 143 : .iter()
883 143 : .map(|&co| {
884 7 : let object = &completion.candidate_objects[co];
885 7 : let mut o = CompletionCandidateObjectT::default();
886 7 : o.object_type = CompletionCandidateObjectType(object.object.object_type() as u8);
887 7 : o.candidate_tags = object.candidate_tags;
888 7 : o.score = object.score;
889 9 : o.qualified_name = Some(object.qualified_name.iter().map(|n| quote_identifier(n)).collect());
890 7 : o.qualified_name_target_idx = object.qualified_name_target_idx as u32;
891 7 : o.prefer_qualified = object.prefer_qualified;
892 7 : match object.object {
893 1 : CandidateObjectRef::Database(id) => o.catalog_database_id = id.unpack_database_id(),
894 1 : CandidateObjectRef::Schema(id) => {
895 1 : let (db, schema) = id.unpack_schema_id();
896 1 : o.catalog_database_id = db;
897 1 : o.catalog_schema_id = schema;
898 1 : }
899 3 : CandidateObjectRef::Table(loc) => {
900 3 : o.table_parameter = match completion.table_parameter_source(loc) {
901 0 : Some(TableParameterSource::Script(_)) => CompletionTableParameter::SCRIPT,
902 0 : Some(TableParameterSource::Attachment(_)) => CompletionTableParameter::ATTACHMENT,
903 3 : None => CompletionTableParameter::NONE,
904 : };
905 3 : if let Some(table) = completion.table(loc) {
906 3 : let (db, schema) = table.catalog_schema_id.unpack_schema_id();
907 3 : o.catalog_database_id = db;
908 3 : o.catalog_schema_id = schema;
909 3 : o.catalog_table_id = table.table_id().pack();
910 3 : o.referenced_catalog_version = table.catalog_version;
911 3 : }
912 : }
913 0 : CandidateObjectRef::Column(loc) => {
914 0 : if let Some(table) = completion.table(loc.table) {
915 0 : let (db, schema) = table.catalog_schema_id.unpack_schema_id();
916 0 : let (table_id, column) =
917 0 : table.table_columns[loc.column as usize].object_id.unpack_table_column_id();
918 0 : o.catalog_database_id = db;
919 0 : o.catalog_schema_id = schema;
920 0 : o.catalog_table_id = table_id.pack();
921 0 : o.table_column_id = column;
922 0 : o.referenced_catalog_version = table.catalog_version;
923 0 : }
924 : }
925 2 : CandidateObjectRef::Function(loc) => {
926 2 : is_function = true;
927 2 : if let Some(f) = completion
928 2 : .entry_data(loc.entry)
929 2 : .and_then(|d| d.function_declarations.get(loc.index as usize))
930 2 : {
931 2 : let (db, schema) = f.catalog_schema_id.unpack_schema_id();
932 2 : o.catalog_database_id = db;
933 2 : o.catalog_schema_id = schema;
934 2 : }
935 : }
936 : }
937 7 : o
938 7 : })
939 143 : .collect();
940 143 : if is_function {
941 2 : completion_text.push_str("()");
942 141 : }
943 143 : let mut c = CompletionCandidateT::default();
944 143 : c.completion_cursor_offset = (completion_text.len() - if is_function { 1 } else { 0 }) as u32;
945 143 : c.display_text = Some(display_text);
946 143 : c.completion_text = Some(completion_text);
947 143 : c.candidate_tags = candidate.candidate_tags;
948 143 : c.name_tags = candidate.coarse_name_tags;
949 143 : c.catalog_objects = Some(catalog_objects);
950 143 : c.score = candidate.score;
951 143 : c.target_location = Some(symbol_span(candidate.target_location));
952 143 : c.target_location_qualified = Some(symbol_span(candidate.target_location_qualified));
953 143 : c.keyword_continuation = candidate.keyword_continuation.filter(|k| !k.is_empty()).map(|k| {
954 28 : if upper_case_keywords {
955 4 : k.to_ascii_uppercase()
956 : } else {
957 24 : k.to_string()
958 : }
959 28 : });
960 143 : c
961 143 : })
962 12 : .collect();
963 12 : let mut out = CompletionT::default();
964 12 : out.cursor_offset = completion.cursor.text_offset as u32;
965 12 : out.dot_completion = completion.dot_completion;
966 12 : out.strategy = completion.strategy;
967 12 : out.candidates = Some(candidates);
968 12 : out
969 12 : }
970 :
971 : /// Pack the declarations of a catalog entry as descriptor
972 8 : pub fn pack_catalog_descriptor(entry: &CatalogEntryData) -> crate::error::Result<Vec<u8>> {
973 8 : let mut descriptor = dashql_buffers::catalog::CatalogDescriptorT::default();
974 : // Both vectors are always written, even when empty
975 8 : descriptor.tables = Some(entry.table_declarations.iter().map(pack_table).collect());
976 8 : descriptor.function_declarations =
977 8 : Some(entry.function_declarations.iter().map(pack_function_declaration).collect());
978 8 : let mut builder = flatbuffers::FlatBufferBuilder::new();
979 8 : let packed = descriptor.pack(&mut builder);
980 8 : builder.finish(packed, None);
981 8 : let bytes = builder.finished_data().to_vec();
982 8 : crate::catalog::descriptor::validate_descriptor(&bytes)?;
983 8 : Ok(bytes)
984 8 : }
|
