Line data Source code
1 : import * as arrow from 'apache-arrow';
2 :
3 : import { OrderByConstraint, ScalarFilter } from './sql/sqlframe_builder.js';
4 : import { VariantKind } from '../utils/variant.js';
5 : import { DataFrame } from './data_frame.js';
6 : import { ArrowTableFormatter } from './arrow_formatter.js';
7 : import { LoggableException } from '../platform/logger/logger.js';
8 : import { AsyncValue } from '../utils/async_value.js';
9 :
10 : export const TASK_FAILED = Symbol("TASK_FAILED");
11 : export const TASK_RUNNING = Symbol("TASK_RUNNING");
12 : export const TASK_SUCCEDED = Symbol("TASK_SUCCEDED");
13 1 : export const ORDINAL_COLUMN = Symbol("ORDINAL_COLUMN");
14 1 : export const STRING_COLUMN = Symbol("STRING_COLUMN");
15 1 : export const LIST_COLUMN = Symbol("LIST_COLUMN");
16 1 : export const SKIPPED_COLUMN = Symbol("SKIPPED_COLUMN");
17 1 : export const ROWNUMBER_COLUMN = Symbol("ROWNUMBER_COLUMN");
18 :
19 : // ------------------------------------------------------------
20 :
21 1 : export class ComputationStateVersion {
22 : /// The data version - incremented when underlying data changes
23 1 : public data: number;
24 : /// The filter version - incremented when filters change
25 1 : public filter: number;
26 :
27 1 : constructor(data: number = 0, filter: number = 0) {
28 1 : this.data = data;
29 1 : this.filter = filter;
30 1 : }
31 :
32 : /// Clone a computation state version
33 1 : clone(): ComputationStateVersion {
34 1 : return new ComputationStateVersion(this.data, this.filter);
35 1 : }
36 :
37 : /// Create a new version with data incremented
38 1 : withDataIncrement(): ComputationStateVersion {
39 1 : return new ComputationStateVersion(this.data + 1, this.filter);
40 1 : }
41 :
42 : /// Create a new version with filter incremented
43 1 : withFilterIncrement(): ComputationStateVersion {
44 1 : return new ComputationStateVersion(this.data, this.filter + 1);
45 1 : }
46 :
47 : /// Check if this version matches another in the data dimension
48 1 : dataMatches(other: ComputationStateVersion): boolean {
49 1 : return this.data === other.data;
50 1 : }
51 :
52 : /// Check if this version matches another in data + filter dimensions
53 1 : filterMatches(other: ComputationStateVersion): boolean {
54 1 : return this.data === other.data && this.filter === other.filter;
55 1 : }
56 :
57 : /// Serialize the version
58 1 : public toString(): string {
59 1 : return `${this.data}.${this.filter}`;
60 1 : }
61 1 : };
62 :
63 : export interface TableFilteringTask {
64 : /// The table id
65 : tableId: number;
66 : /// The table version
67 : tableVersion: ComputationStateVersion;
68 : /// The data frame
69 : inputDataTable: arrow.Table;
70 : /// The data frame
71 : inputDataTableFieldIndex: Map<string, number>;
72 : /// The data frame
73 : inputDataFrame: DataFrame;
74 : /// The row number columns
75 : rowNumberColumnName: string;
76 : /// The scalar filters
77 : filters: ScalarFilter[];
78 : }
79 :
80 : export interface TableOrderingTask {
81 : /// The table id
82 : tableId: number;
83 : /// The table version
84 : tableVersion: ComputationStateVersion;
85 : /// The data frame
86 : inputDataTable: arrow.Table;
87 : /// The data frame
88 : inputDataTableFieldIndex: Map<string, number>;
89 : /// The data frame
90 : inputDataFrame: DataFrame;
91 : /// The active filter table, if ordering should happen on a filtered subset
92 : filterTable: FilterTable | null;
93 : /// The active Data-search table, if ordering should happen on searched rows
94 : dataSearchTable: DataSearchTable | null;
95 : /// The row number column that provides stable ids into the immutable base table
96 : rowNumberColumnName: string;
97 : /// The ordering constraints
98 : orderingConstraints: OrderByConstraint[];
99 : }
100 :
101 : export type ResultSearchKind = 'columns' | 'data';
102 :
103 : export interface ResultSearchState {
104 : /// The plain-text search requested by the user
105 : requestedPattern: string;
106 : /// The last successfully applied search text
107 : appliedPattern: string;
108 : /// Incremented for every requested search
109 : requestId: number;
110 : /// The last applied request
111 : appliedRequestId: number;
112 : /// Matching column-group indexes for a Columns search
113 : matchingColumnGroups: number[] | null;
114 : /// Matching original-column indexes keyed by 1-based row number
115 : matchingRows: Map<number, number[]> | null;
116 : /// Whether a search is currently running
117 : pending: boolean;
118 : /// Latest search error, if any
119 : error: string | null;
120 : }
121 :
122 : export interface DataSearchTask {
123 : /// The table id
124 : tableId: number;
125 : /// The data/filter version when the search was requested
126 : tableVersion: ComputationStateVersion;
127 : /// The source data frame
128 : inputDataFrame: DataFrame;
129 : /// The stable row-number column in the source data frame
130 : rowNumberColumnName: string;
131 : /// Original result columns searched by the query
132 : columns: Array<{ columnIdx: number; columnGroupIdx: number; fieldName: string }>;
133 : /// The plain-text search pattern
134 : pattern: string;
135 : /// The request id used to reject stale results
136 : requestId: number;
137 : /// Cancels materialization when superseded or when the source table is destroyed
138 : abortController: AbortController;
139 : }
140 :
141 1 : export function createResultSearchState(): ResultSearchState {
142 1 : return {
143 1 : requestedPattern: '',
144 1 : appliedPattern: '',
145 1 : requestId: 0,
146 1 : appliedRequestId: 0,
147 1 : matchingColumnGroups: null,
148 1 : matchingRows: null,
149 1 : pending: false,
150 1 : error: null,
151 1 : };
152 1 : }
153 :
154 : export interface TableAggregationTask {
155 : /// The table id
156 : tableId: number;
157 : /// The table version
158 : tableVersion: ComputationStateVersion;
159 : /// The column entries
160 : columnEntries: ColumnGroup[];
161 : /// The data frame
162 : inputDataFrame: DataFrame;
163 : }
164 :
165 : export interface SystemColumnComputationTask {
166 : /// The table id
167 : tableId: number;
168 : /// The table version
169 : tableVersion: ComputationStateVersion;
170 : /// The column entries
171 : columnEntries: ColumnGroup[];
172 : /// The input table
173 : inputTable: arrow.Table;
174 : /// The input data frame
175 : inputDataFrame: DataFrame;
176 : /// The column of the input data frame with the row numbers of the input table, from 1
177 : rowNumberFieldName: string;
178 : /// The stats table
179 : tableAggregate: TableAggregation;
180 : }
181 :
182 : export interface ColumnAggregationTask {
183 : /// The table id
184 : tableId: number;
185 : /// The table version
186 : tableVersion: ComputationStateVersion;
187 : /// The task id
188 : columnId: number;
189 : /// The column entry
190 : columnEntry: ColumnGroup;
191 : /// The input data frame
192 : inputDataFrame: DataFrame;
193 : /// The table summary
194 : tableAggregate: TableAggregation;
195 : }
196 :
197 : // ------------------------------------------------------------
198 :
199 1 : export enum TaskStatus {
200 1 : TASK_RUNNING = 1,
201 1 : TASK_SUCCEEDED = 2,
202 1 : TASK_FAILED = 3,
203 1 : };
204 :
205 : export interface TaskProgress {
206 : /// Task status
207 : status: TaskStatus;
208 : /// Task started at timestamp
209 : startedAt: Date;
210 : /// Task completed at timestamp
211 : completedAt: Date | null;
212 : /// Task failed at timestamp
213 : failedAt: Date | null;
214 : /// Task failed with error
215 : failedWithError: LoggableException | null;
216 : }
217 :
218 : export type WithProgress<T> = T & { progress: TaskProgress };
219 :
220 : // ------------------------------------------------------------
221 :
222 : export type ColumnGroup =
223 : | VariantKind<typeof ROWNUMBER_COLUMN, RowNumberGridColumnGroup>
224 : | VariantKind<typeof SKIPPED_COLUMN, SkippedGridColumnGroup>
225 : | VariantKind<typeof ORDINAL_COLUMN, OrdinalGridColumnGroup>
226 : | VariantKind<typeof STRING_COLUMN, StringGridColumnGroup>
227 : | VariantKind<typeof LIST_COLUMN, ListGridColumnGroup>
228 : ;
229 :
230 1 : export function getGridColumnTypeName(variant: ColumnGroup) {
231 1 : switch (variant.type) {
232 0 : case ROWNUMBER_COLUMN: return "ROWNUMBER";
233 0 : case SKIPPED_COLUMN: return "SKIPPED";
234 1 : case ORDINAL_COLUMN: return "ORDINAL";
235 1 : case STRING_COLUMN: return "STRING";
236 1 : case LIST_COLUMN: return "LIST";
237 1 : }
238 1 : }
239 :
240 : export interface ColumnStatsFields {
241 : /// Entry count (!= null)
242 : countFieldName: string;
243 : /// Distinct entry count (only for strings and lists)
244 : distinctCountFieldName: string | null;
245 : /// Maximum value
246 : minAggregateFieldName: string | null;
247 : /// Minimum value
248 : maxAggregateFieldName: string | null;
249 : }
250 :
251 : export interface ColumnBinningFields {
252 : /// The bin field
253 : binFieldName: string;
254 : }
255 :
256 : export interface RowNumberGridColumnGroup {
257 : /// The input field
258 : rowNumberFieldName: string;
259 : }
260 :
261 : export interface OrdinalGridColumnGroup {
262 : /// The input field name
263 : inputFieldName: string;
264 : /// The input field type
265 : inputFieldType: arrow.DataType;
266 : /// Is the input nullable?
267 : inputFieldNullable: boolean;
268 : /// The column stats
269 : statsFields: ColumnStatsFields | null;
270 : /// The bin field name
271 : binFieldName: string | null;
272 : /// The bin count
273 : binCount: number;
274 : }
275 :
276 : export interface StringGridColumnGroup {
277 : /// The input field name
278 : inputFieldName: string;
279 : /// The input field type
280 : inputFieldType: arrow.DataType;
281 : /// Is the input nullable?
282 : inputFieldNullable: boolean;
283 : /// The column stats
284 : statsFields: ColumnStatsFields | null;
285 : /// The identifier field
286 : valueIdFieldName: string | null;
287 : }
288 :
289 : export interface ListGridColumnGroup {
290 : /// The input field name
291 : inputFieldName: string;
292 : /// The input field type
293 : inputFieldType: arrow.DataType;
294 : /// Is the input nullable?
295 : inputFieldNullable: boolean;
296 : /// The column stats
297 : statsFields: ColumnStatsFields | null;
298 : /// The identifier field
299 : valueIdFieldName: string | null;
300 : }
301 :
302 : export interface SkippedGridColumnGroup {
303 : /// The input field name
304 : inputFieldName: string;
305 : /// The input field type
306 : inputFieldType: arrow.DataType;
307 : /// Is the input nullable?
308 : inputFieldNullable: boolean;
309 : }
310 :
311 : // ------------------------------------------------------------
312 :
313 : export interface OrderingTable {
314 : /// The row number column that is used for the ordering ids
315 : inputRowNumberColumnName: string;
316 : /// The ordering constraints
317 : orderingConstraints: OrderByConstraint[];
318 : /// The arrow table, only containing the row ids in display order
319 : dataTable: arrow.Table;
320 : /// The data frame
321 : dataFrame: DataFrame;
322 : /// The version when this ordering was computed
323 : version: ComputationStateVersion;
324 : /// The Data-search request included in the ordering, if any
325 : dataSearchRequestId: number | null;
326 : }
327 :
328 : export interface FilterTable {
329 : /// The row number column that is used for the filter
330 : inputRowNumberColumnName: string;
331 : /// The arrow table, only containing the row ids of the filtered rows
332 : dataTable: arrow.Table;
333 : /// The data frame
334 : dataFrame: DataFrame;
335 : /// The version when this filter was computed
336 : version: ComputationStateVersion;
337 : }
338 :
339 : export interface DataSearchTable {
340 : /// The source row-number column matched by this search
341 : inputRowNumberColumnName: string;
342 : /// The Arrow table containing matched row ids and matching original-column ids
343 : dataTable: arrow.Table;
344 : /// The materialized search result
345 : dataFrame: DataFrame;
346 : /// The source data version searched
347 : version: ComputationStateVersion;
348 : /// The request represented by this table
349 : requestId: number;
350 : }
351 :
352 : // ------------------------------------------------------------
353 :
354 : export type ColumnAggregationVariant =
355 : VariantKind<typeof ORDINAL_COLUMN, OrdinalColumnAggregation>
356 : | VariantKind<typeof STRING_COLUMN, StringColumnAggregation>
357 : | VariantKind<typeof LIST_COLUMN, ListColumnAggregation>
358 : | VariantKind<typeof SKIPPED_COLUMN, null>
359 : ;
360 :
361 : export type WithFilter<T> = T & {
362 : /// The cross-filter table, if active
363 : filterTable: FilterTable | null,
364 : /// The Data-search table, if active
365 : dataSearchTable: DataSearchTable | null,
366 : /// Number of rows in the effective intersection
367 : selectionRowCount: number,
368 : /// The unfiltered aggregate
369 : unfilteredAggregate: ColumnAggregationVariant;
370 : };
371 :
372 : export type WithFilterEpoch<T> = T & {
373 : /// The filter version when this aggregate was computed
374 : filterVersion: ComputationStateVersion,
375 : /// The Data-search request when this aggregate was computed
376 : dataSearchRequestId: number | null,
377 : };
378 :
379 : export interface TableAggregation {
380 : /// The statistics
381 : dataFrame: DataFrame;
382 : /// The statistics
383 : table: arrow.Table;
384 : /// The statistics field index
385 : tableFieldsByName: Map<string, number>;
386 : /// The formatter for the stats table
387 : tableFormatter: ArrowTableFormatter;
388 : /// Maximum value
389 : countStarFieldName: string;
390 : }
391 :
392 : export interface OrdinalColumnAggregation<WidthType extends arrow.DataType = arrow.DataType, BoundType extends arrow.DataType = arrow.DataType> {
393 : /// The column entry
394 : columnEntry: OrdinalGridColumnGroup;
395 : /// The binned data frame
396 : binnedDataFrame: DataFrame;
397 : /// The binned values
398 : binnedValues: BinnedValuesTable;
399 : /// The formatter for the binned values
400 : binnedValuesFormatter: ArrowTableFormatter;
401 : /// The analyzed information for an ordinal column
402 : columnAnalysis: OrdinalColumnAnalysis;
403 : }
404 :
405 : export interface OrdinalColumnAnalysis {
406 : /// The total row count
407 : totalCount: number;
408 : /// The value count
409 : countNotNull: number;
410 : /// The null count
411 : countNull: number;
412 : /// The minimum value
413 : minValue: string;
414 : /// The maximum value
415 : maxValue: string;
416 : /// The bin count
417 : binCount: number;
418 : /// The bin counts
419 : binValueCounts: BigInt64Array;
420 : /// The bin percentages
421 : binPercentages: Float64Array;
422 : /// The bin lower bounds
423 : binLowerBounds: string[];
424 : }
425 :
426 : export interface OrdinalColumnFilterAnalysis {
427 : /// The bin counts
428 : binValueCounts: BigInt64Array;
429 : /// The bin percentages
430 : binPercentages: Float64Array;
431 : }
432 :
433 : export interface StringColumnAggregation {
434 : /// The string column entry
435 : columnEntry: StringGridColumnGroup;
436 : /// The frequent values
437 : frequentValuesDataFrame: DataFrame;
438 : /// The frequent values
439 : frequentValuesTable: FrequentValuesTable;
440 : /// The formatter for the frequent values
441 : frequentValuesFormatter: ArrowTableFormatter;
442 : /// The analyzed column information
443 : analysis: StringColumnAnalysis;
444 : }
445 :
446 : export interface StringColumnAnalysis {
447 : /// The total row count
448 : totalCount: number;
449 : /// The value count
450 : countNotNull: number;
451 : /// The null count
452 : countNull: number;
453 : /// The distinct count
454 : countDistinct: number;
455 : /// Is unique?
456 : isUnique: boolean;
457 : /// The frequent values
458 : frequentValueStrings: (string | null)[];
459 : /// The frequent value ids (for plotting)
460 : frequentValueIds: BigInt64Array;
461 : /// The frequent value counts
462 : frequentValueCounts: BigInt64Array;
463 : /// The frequent value percentages
464 : frequentValuePercentages: Float64Array;
465 : }
466 :
467 : export interface ListColumnAggregation {
468 : /// The list column entry
469 : columnEntry: ListGridColumnGroup;
470 : /// The frequent values
471 : frequentValuesDataFrame: DataFrame;
472 : /// The frequent values
473 : frequentValuesTable: FrequentValuesTable;
474 : /// The formatter for the frequent values
475 : frequentValuesFormatter: ArrowTableFormatter;
476 : /// The analyzed information for a list column
477 : analysis: ListColumnAnalysis;
478 : }
479 :
480 : export interface ListColumnAnalysis {
481 : /// The value count
482 : countNotNull: number;
483 : /// The null count
484 : countNull: number;
485 : /// The distinct count
486 : countDistinct: number;
487 : /// Is unique?
488 : isUnique: boolean;
489 : /// The frequent value is null
490 : frequentValueIsNull: Uint8Array;
491 : /// The frequent value counts
492 : frequentValueCounts: BigInt64Array;
493 : /// The frequent value percentages
494 : frequentValuePercentages: Float64Array;
495 : }
496 :
497 : // ------------------------------------------------------------
498 :
499 : export type BinnedValuesTable<WidthType extends arrow.DataType = arrow.DataType, BoundType extends arrow.DataType = arrow.DataType> = arrow.Table<{
500 : bin: arrow.Int32,
501 : binWidth: WidthType,
502 : binLowerBound: BoundType,
503 : binUpperBound: BoundType,
504 : count: arrow.Int64,
505 : }>;
506 :
507 : export type FrequentValuesTable<KeyType extends arrow.DataType = arrow.DataType> = arrow.Table<{
508 : key: KeyType,
509 : keyId: arrow.Int64,
510 : count: arrow.Int64,
511 : }>
512 :
513 : // ------------------------------------------------------------
514 :
515 : export type ComputationTask<Type, Task, Result> = VariantKind<Type, Task> & {
516 : result: AsyncValue<Result, LoggableException>,
517 : taskId?: number,
518 : };
519 :
520 1 : export const FILTERED_COLUMN_AGGREGATION_TASK = Symbol("FILTERED_COLUMN_AGGREGATION_TASK");
521 1 : export const COLUMN_AGGREGATION_TASK = Symbol("COLUMN_AGGREGATION_TASK");
522 1 : export const TABLE_FILTERING_TASK = Symbol("TABLE_FILTERING_TASK");
523 1 : export const TABLE_ORDERING_TASK = Symbol("TABLE_ORDERING_TASK");
524 1 : export const TABLE_AGGREGATION_TASK = Symbol("TABLE_AGGREGATION_TASK");
525 1 : export const SYSTEM_COLUMN_COMPUTATION_TASK = Symbol("SYSTEM_COLUMN_COMPUTATION_TASK");
526 1 : export const DATA_SEARCH_TASK = Symbol("DATA_SEARCH_TASK");
527 :
528 : export type TaskVariant =
529 : | ComputationTask<typeof TABLE_FILTERING_TASK, TableFilteringTask, FilterTable | null>
530 : | ComputationTask<typeof TABLE_ORDERING_TASK, TableOrderingTask, OrderingTable>
531 : | ComputationTask<typeof TABLE_AGGREGATION_TASK, TableAggregationTask, [TableAggregation, ColumnGroup[]]>
532 : | ComputationTask<typeof SYSTEM_COLUMN_COMPUTATION_TASK, SystemColumnComputationTask, [arrow.Table, DataFrame, ColumnGroup[]]>
533 : | ComputationTask<typeof COLUMN_AGGREGATION_TASK, ColumnAggregationTask, ColumnAggregationVariant>
534 : | ComputationTask<typeof FILTERED_COLUMN_AGGREGATION_TASK, WithFilter<ColumnAggregationTask>, WithFilterEpoch<ColumnAggregationVariant> | null>
535 : | ComputationTask<typeof DATA_SEARCH_TASK, DataSearchTask, DataSearchTable>
536 : ;
537 :
538 : /// The title of the queries of a task in the query history
539 1 : export function computationTaskTitle(task: TaskVariant): string {
540 1 : switch (task.type) {
541 1 : case COLUMN_AGGREGATION_TASK: return 'Column aggregation';
542 0 : case TABLE_FILTERING_TASK: return 'Table filtering';
543 0 : case TABLE_ORDERING_TASK: return 'Table ordering';
544 1 : case TABLE_AGGREGATION_TASK: return 'Table aggregation';
545 1 : case SYSTEM_COLUMN_COMPUTATION_TASK: return 'System column computation';
546 0 : case FILTERED_COLUMN_AGGREGATION_TASK: return 'Filtered column aggregation';
547 0 : case DATA_SEARCH_TASK: return 'Data search';
548 1 : }
549 1 : }
|
