Line data Source code
1 : //! The script compiler: the terminal statement of a script, its SQL and the compiled visualization of a
2 : //! terminal query with `@vis` tags. Runs execute the plan of the planner, the compilation describes the
3 : //! terminal statement for the editor.
4 :
5 : use dashql_core::analyzer::{AnalyzedScript, VisualizationSpec};
6 : use dashql_core::catalog::{Catalog, PROTO_NULL_U32};
7 : use dashql_core::formatter::Formatter;
8 : use dashql_core::script::Script;
9 : use crate::visualize;
10 : use dashql_buffers::execution::{
11 : CompiledVisualizationT, ScriptCompilationErrorCode as ErrorCode, ScriptCompilationErrorT, ScriptCompilationResultT,
12 : ScriptCompilationStatementKind as StatementKind,
13 : };
14 : use dashql_buffers::formatting::FormattingConfigT;
15 : use dashql_buffers::parser::{AttributeKey, NodeType, StatementType, TextSpanT};
16 : use dashql_buffers::visualization::{DashboardSpecT, MarkdownSpec, TableSpecT};
17 : use dashql_parser::attrs;
18 : use dashql_parser::parser::context::resolve_text_span;
19 : use dashql_parser::parser::{ParsedScript, Statement};
20 : use dashql_parser::scanner::ScannedScript;
21 : use dashql_parser::span::TextSpan;
22 :
23 : /// Options of the compilation
24 : #[derive(Clone, Copy, Debug)]
25 : pub struct CompilationOptions {
26 : /// Allow the DashQL extensions (the visualizations of `@vis` tags)
27 : pub allow_extensions: bool,
28 : /// Parse the script if the text changed
29 : pub parse_if_outdated: bool,
30 : }
31 :
32 : impl Default for CompilationOptions {
33 17 : fn default() -> Self {
34 17 : Self {
35 17 : allow_extensions: true,
36 17 : parse_if_outdated: true,
37 17 : }
38 17 : }
39 : }
40 :
41 5 : fn error(code: ErrorCode, message: &str, statement_id: u32, ast_node_id: u32, text_span: Option<TextSpan>) -> ScriptCompilationErrorT {
42 5 : let mut e = ScriptCompilationErrorT::default();
43 5 : e.code = code;
44 5 : e.statement_id = statement_id;
45 5 : e.ast_node_id = ast_node_id;
46 5 : e.text_span = text_span.map(|s| {
47 2 : let mut t = TextSpanT::default();
48 2 : t.offset = s.offset;
49 2 : t.length = s.length;
50 2 : t
51 2 : });
52 5 : e.message = Some(message.to_string());
53 5 : e
54 5 : }
55 :
56 : /// The text of a span with the obfuscated literals replaced, so that it can be executed
57 849 : pub fn read_executable_text(scanned: &ScannedScript, span: TextSpan) -> String {
58 849 : read_executable_text_with(scanned, span, &[])
59 849 : }
60 :
61 : /// The text of a span with the obfuscated literals replaced and with edits: ascending, disjoint text spans and
62 : /// their replacements, an empty span inserts its text. Edits within obfuscated literals are dropped.
63 1233 : pub fn read_executable_text_with(scanned: &ScannedScript, span: TextSpan, edits: &[(TextSpan, String)]) -> String {
64 1233 : let text = scanned.text.as_bytes();
65 3488 : let clamp = |offset: usize| offset.min(text.len());
66 1233 : let begin = clamp(span.offset as usize);
67 1233 : let end = clamp((span.offset + span.length) as usize);
68 : // The replacements of the obfuscated literals and the edits, ordered by offset with insertions first
69 1233 : let first = scanned.obfuscated_literals.partition_point(|l| (l.location.offset as usize) < begin);
70 1233 : let mut replacements: Vec<(usize, usize, &str)> = scanned.obfuscated_literals[first..]
71 1233 : .iter()
72 1233 : .take_while(|l| (l.location.offset + l.location.length) as usize <= end)
73 1233 : .map(|l| (l.location.offset as usize, (l.location.offset + l.location.length) as usize, l.replacement.as_str()))
74 1233 : .collect();
75 1233 : replacements.extend(
76 1233 : edits
77 1233 : .iter()
78 1233 : .filter(|(s, _)| s.offset as usize >= begin && s.end() as usize <= end)
79 1233 : .map(|(s, t)| (s.offset as usize, s.end() as usize, t.as_str())),
80 : );
81 1233 : replacements.sort_by_key(|&(b, e, _)| (b, e));
82 1233 : let mut out = Vec::with_capacity(end - begin);
83 1233 : let mut at = begin;
84 1233 : for (b, e, replacement) in replacements {
85 512 : if b < at {
86 1 : continue;
87 511 : }
88 511 : out.extend_from_slice(&text[at..clamp(b)]);
89 511 : out.extend_from_slice(replacement.as_bytes());
90 511 : at = clamp(e);
91 : }
92 1233 : out.extend_from_slice(&text[at.min(end)..end]);
93 1233 : String::from_utf8_lossy(&out).into_owned()
94 1233 : }
95 :
96 109 : fn has_returning(parsed: &ParsedScript, statement: &Statement) -> bool {
97 109 : let root = &parsed.nodes[statement.root as usize];
98 109 : let begin = root.children_begin_or_value() as usize;
99 109 : let children = &parsed.nodes[begin..begin + root.children_count() as usize];
100 109 : let [insert_returning, delete_returning] = attrs!(children, [AttributeKey::SQL_INSERT_RETURNING, AttributeKey::SQL_DELETE_RETURNING]);
101 109 : let returning = if statement.statement_type == StatementType::DELETE { delete_returning } else { insert_returning };
102 109 : returning.is_some_and(|r| r.node_type() != NodeType::NONE)
103 109 : }
104 :
105 : /// Does a statement produce output?
106 1064 : pub fn produces_output(parsed: &ParsedScript, statement: &Statement) -> bool {
107 1064 : match statement.statement_type {
108 : // SELECT INTO creates a table and returns no rows
109 752 : StatementType::SELECT | StatementType::EXPLAIN => true,
110 109 : StatementType::INSERT | StatementType::DELETE => has_returning(parsed, statement),
111 203 : _ => false,
112 : }
113 1064 : }
114 :
115 : /// Compile the visualization metadata
116 39 : pub fn compile_visualization(spec: &VisualizationSpec, script: &AnalyzedScript) -> Result<CompiledVisualizationT, String> {
117 39 : let renderer = spec.renderer.as_ref().ok_or("could not resolve visualization renderer")?.to_ascii_lowercase();
118 38 : let mut out = CompiledVisualizationT::default();
119 : // The compiled visualization always writes its strings
120 38 : out.vegalite_spec = Some(String::new());
121 38 : let mut dashboard = DashboardSpecT::default();
122 38 : dashboard.row = spec.dashboard.row;
123 38 : dashboard.column = spec.dashboard.column;
124 38 : dashboard.width = spec.dashboard.width;
125 38 : dashboard.height = spec.dashboard.height;
126 38 : out.dashboard = Some(Box::new(dashboard));
127 38 : match renderer.as_str() {
128 38 : "vegalite" => out.vegalite_spec = Some(visualize::generate_vegalite_spec(spec, script)),
129 34 : "markdown" => {
130 6 : let doc = visualize::compile_markdown_spec(spec).ok_or("could not compile Markdown visualization spec")?;
131 5 : let mut builder = flatbuffers::FlatBufferBuilder::new();
132 5 : let packed = visualize::pack_markdown_spec(&mut builder, &doc);
133 5 : builder.finish(packed, None);
134 5 : out.markdown_spec = Some(Box::new(flatbuffers::root::<MarkdownSpec>(builder.finished_data()).unwrap().unpack()));
135 : }
136 28 : "table" => out.table_spec = Some(Box::new(TableSpecT::default())),
137 1 : _ => return Err(format!("unsupported visualization renderer: {renderer}")),
138 : }
139 36 : out.renderer = Some(renderer);
140 36 : Ok(out)
141 39 : }
142 :
143 : /// Is a byte C `isspace`?
144 2 : fn is_c_space(c: u8) -> bool {
145 2 : matches!(c, b' ' | b'\t' | b'\n' | b'\x0b' | b'\x0c' | b'\r')
146 2 : }
147 :
148 : /// Compile a script
149 19 : pub fn compile(script: &mut Script, catalog: &Catalog, config: &FormattingConfigT, options: CompilationOptions) -> ScriptCompilationResultT {
150 19 : let mut result = ScriptCompilationResultT::default();
151 19 : result.kind = StatementKind::QUERY;
152 19 : result.terminal_statement_id = PROTO_NULL_U32;
153 : // The result always writes its strings
154 19 : result.sql = Some(String::new());
155 19 : let mut errors = Vec::new();
156 19 : let finish = |mut result: ScriptCompilationResultT, errors| {
157 19 : result.errors = Some(errors);
158 19 : result
159 19 : };
160 19 : if options.parse_if_outdated && script.parsed.as_ref().is_none_or(|p| p.scanned.text_version != script.text_version()) {
161 17 : script.parse();
162 18 : }
163 19 : let Some(parsed) = script.parsed.clone() else {
164 1 : errors.push(error(ErrorCode::EMPTY_SCRIPT, "script is not parsed", PROTO_NULL_U32, PROTO_NULL_U32, None));
165 1 : return finish(result, errors);
166 : };
167 18 : let scanned = &parsed.scanned;
168 18 : for e in &scanned.errors {
169 1 : errors.push(error(ErrorCode::SCANNER_ERROR, &e.message, PROTO_NULL_U32, PROTO_NULL_U32, Some(e.location)));
170 1 : }
171 18 : for e in &parsed.errors {
172 1 : let span = resolve_text_span(&scanned.text, &scanned.symbols, e.location);
173 1 : errors.push(error(ErrorCode::PARSER_ERROR, &e.message, PROTO_NULL_U32, PROTO_NULL_U32, Some(span)));
174 1 : }
175 18 : if !errors.is_empty() {
176 : // Without a reliable AST, the original text is the one statement
177 2 : let sql = read_executable_text(scanned, TextSpan::new(0, scanned.text.len() as u32));
178 2 : if sql.bytes().any(|c| !is_c_space(c)) {
179 2 : result.sql = Some(sql);
180 2 : result.terminal_statement_id = 0;
181 2 : }
182 2 : return finish(result, errors);
183 16 : }
184 16 : if parsed.statements.is_empty() {
185 2 : errors.push(error(ErrorCode::EMPTY_SCRIPT, "script has no executable statement", PROTO_NULL_U32, PROTO_NULL_U32, None));
186 2 : return finish(result, errors);
187 14 : }
188 14 : let terminal_statement_id = parsed.statements.len() - 1;
189 14 : result.terminal_statement_id = terminal_statement_id as u32;
190 : // A query with `@vis` tags is a visualization, the shell runs it as a plain query
191 14 : let vis_tags = options.allow_extensions
192 13 : && parsed.statements[terminal_statement_id].statement_type == StatementType::SELECT
193 12 : && dashql_core::analyzer::has_vis_tags(&parsed, terminal_statement_id);
194 14 : let descriptions = parsed.associate_descriptions();
195 14 : let terminal_sql = read_executable_text(scanned, descriptions[terminal_statement_id].statement_span);
196 14 : if !vis_tags {
197 8 : result.sql = Some(terminal_sql);
198 8 : return finish(result, errors);
199 6 : }
200 :
201 6 : result.kind = StatementKind::VISUALIZE;
202 6 : let terminal_root = parsed.statements[terminal_statement_id].root;
203 6 : let current = script.analyzed.as_ref().is_some_and(|a| std::sync::Arc::ptr_eq(&a.parsed, &parsed));
204 6 : if !current {
205 : // A failed analysis leaves the visualization without metadata
206 6 : if let Err(e) = script.analyze(catalog, false) {
207 0 : errors.push(error(ErrorCode::VISUALIZATION_METADATA_UNAVAILABLE, &e.message, terminal_statement_id as u32, terminal_root, None));
208 0 : return finish(result, errors);
209 6 : }
210 0 : }
211 :
212 6 : let mut formatter = Formatter::new(&parsed);
213 6 : formatter.replace_obfuscated_literals();
214 6 : let formatted = formatter.format_node_at(terminal_root as usize, config);
215 : // A placeholder of a node that the formatter does not know would change the query, the source text is used instead
216 6 : let sql = if formatter.is_fully_formatted() { formatted } else { terminal_sql };
217 6 : if sql.is_empty() {
218 0 : errors.push(error(
219 : ErrorCode::FORMAT_ERROR,
220 0 : "could not format the executable query",
221 0 : terminal_statement_id as u32,
222 0 : terminal_root,
223 0 : None,
224 : ));
225 0 : return finish(result, errors);
226 6 : }
227 6 : result.sql = Some(sql);
228 :
229 6 : let analyzed = script.analyzed.clone().unwrap();
230 6 : let visualization = analyzed
231 6 : .visualization_specs
232 6 : .iter()
233 6 : .find(|s| s.ast_statement_id == Some(terminal_statement_id as u32))
234 6 : .filter(|s| s.renderer.is_some());
235 : // Invalid `@vis` tags are analyzer warnings, the query still runs
236 6 : let Some(visualization) = visualization else {
237 2 : result.kind = StatementKind::QUERY;
238 2 : return finish(result, errors);
239 : };
240 4 : match compile_visualization(visualization, &analyzed) {
241 4 : Ok(compiled) => result.visualization = Some(Box::new(compiled)),
242 0 : Err(message) => {
243 0 : result.sql = Some(String::new());
244 0 : errors.push(error(
245 0 : ErrorCode::VISUALIZATION_METADATA_UNAVAILABLE,
246 0 : &message,
247 0 : terminal_statement_id as u32,
248 0 : terminal_root,
249 0 : None,
250 0 : ));
251 0 : }
252 : }
253 4 : finish(result, errors)
254 19 : }
|
