Line data Source code
1 : import * as arrow from 'apache-arrow';
2 : import * as styles from './arrow_formatter.module.css';
3 : import { Int128, Decimal128 } from '../utils/int128.js';
4 : import { LoggerLike } from '../platform/logger/logger.js';
5 :
6 1 : const LOG_CTX = 'arrow_formatter';
7 :
8 : /// Format a duration value to a human-readable string
9 1 : function formatDuration(value: bigint, unit: arrow.TimeUnit): string {
10 : // Convert to milliseconds first
11 1 : let ms: bigint;
12 1 : switch (unit) {
13 1 : case arrow.TimeUnit.SECOND:
14 1 : ms = value * 1000n;
15 1 : break;
16 1 : case arrow.TimeUnit.MILLISECOND:
17 1 : ms = value;
18 1 : break;
19 1 : case arrow.TimeUnit.MICROSECOND:
20 1 : ms = value / 1000n;
21 1 : break;
22 1 : case arrow.TimeUnit.NANOSECOND:
23 1 : ms = value / 1000000n;
24 1 : break;
25 0 : default:
26 0 : ms = value;
27 1 : }
28 :
29 1 : const isNegative = ms < 0n;
30 1 : if (isNegative) ms = -ms;
31 :
32 1 : const totalSeconds = ms / 1000n;
33 1 : const milliseconds = ms % 1000n;
34 1 : const seconds = totalSeconds % 60n;
35 1 : const totalMinutes = totalSeconds / 60n;
36 1 : const minutes = totalMinutes % 60n;
37 1 : const totalHours = totalMinutes / 60n;
38 1 : const hours = totalHours % 24n;
39 1 : const days = totalHours / 24n;
40 :
41 1 : const parts: string[] = [];
42 1 : if (days > 0n) parts.push(`${days}d`);
43 1 : if (hours > 0n) parts.push(`${hours}h`);
44 1 : if (minutes > 0n) parts.push(`${minutes}m`);
45 1 : if (seconds > 0n || milliseconds > 0n) {
46 1 : if (milliseconds > 0n) {
47 1 : parts.push(`${seconds}.${milliseconds.toString().padStart(3, '0')}s`);
48 : } else {
49 1 : parts.push(`${seconds}s`);
50 : }
51 1 : }
52 :
53 1 : if (parts.length === 0) return '0s';
54 1 : return (isNegative ? '-' : '') + parts.join(' ');
55 1 : }
56 :
57 : /// Format a list/array value to a string representation
58 1 : function formatList(vector: arrow.Vector): string {
59 1 : const values: string[] = [];
60 1 : const maxItems = 5; // Limit display to avoid very long strings
61 :
62 1 : for (let i = 0; i < Math.min(vector.length, maxItems); i++) {
63 1 : const val = vector.get(i);
64 1 : if (val === null) {
65 1 : values.push('null');
66 1 : } else if (typeof val === 'string') {
67 0 : values.push(`"${val}"`);
68 1 : } else if (typeof val === 'bigint') {
69 0 : values.push(val.toString());
70 1 : } else if (typeof val === 'number') {
71 1 : values.push(Number.isInteger(val) ? val.toString() : val.toFixed(4));
72 0 : } else if (val instanceof arrow.Vector) {
73 0 : values.push(formatList(val)); // Nested list
74 : } else {
75 0 : values.push(String(val));
76 : }
77 1 : }
78 :
79 1 : if (vector.length > maxItems) {
80 1 : values.push(`... +${vector.length - maxItems} more`);
81 : }
82 :
83 1 : return `[${values.join(', ')}]`;
84 1 : }
85 :
86 : /// Convert Arrow's nested runtime values (Vectors and row proxies) into values understood by the
87 : /// JSON tree viewer. BigInts and Dates are intentionally preserved; the viewer renders both.
88 1 : export function arrowValueToJson(value: unknown): unknown {
89 1 : if (value instanceof arrow.Vector) {
90 1 : return Array.from(value, item => arrowValueToJson(item));
91 : }
92 1 : if (value instanceof Map) {
93 0 : return Object.fromEntries(Array.from(value, ([key, item]) => [String(key), arrowValueToJson(item)]));
94 : }
95 1 : if (Array.isArray(value)) {
96 0 : return value.map(item => arrowValueToJson(item));
97 : }
98 1 : if (ArrayBuffer.isView(value) && !(value instanceof DataView) && 'length' in value) {
99 0 : return Array.from(value as unknown as ArrayLike<unknown>, item => arrowValueToJson(item));
100 : }
101 1 : if (value instanceof Date || value == null || typeof value !== 'object') {
102 1 : return value;
103 : }
104 :
105 1 : const jsonValue = 'toJSON' in value && typeof value.toJSON === 'function'
106 0 : ? value.toJSON()
107 1 : : value;
108 1 : if (jsonValue !== value) {
109 0 : return arrowValueToJson(jsonValue);
110 : }
111 1 : return Object.fromEntries(
112 1 : Object.entries(jsonValue).map(([key, item]) => [key, arrowValueToJson(item)]),
113 1 : );
114 1 : }
115 :
116 1 : export function isArrowListType(type: arrow.DataType): boolean {
117 1 : return type.typeId === arrow.Type.List || type.typeId === arrow.Type.FixedSizeList;
118 1 : }
119 :
120 1 : export function isArrowStructuredType(type: arrow.DataType): boolean {
121 1 : return isArrowListType(type) || type.typeId === arrow.Type.Struct;
122 1 : }
123 :
124 : /// Format binary data as hex string
125 1 : function formatBinary(data: Uint8Array): string {
126 1 : if (data == null) return '';
127 1 : const maxBytes = 32; // Limit display length
128 1 : const bytes = data.slice(0, maxBytes);
129 1 : const hex = Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join('');
130 1 : if (data.length > maxBytes) {
131 1 : return `0x${hex}... (+${data.length - maxBytes} bytes)`;
132 : }
133 1 : return `0x${hex}`;
134 1 : }
135 :
136 : /// Format time value (time of day) to a string
137 1 : function formatTime(value: number | bigint, unit: arrow.TimeUnit): string {
138 1 : let totalMs: number;
139 1 : if (typeof value === 'bigint') {
140 1 : switch (unit) {
141 0 : case arrow.TimeUnit.SECOND:
142 0 : totalMs = Number(value) * 1000;
143 0 : break;
144 0 : case arrow.TimeUnit.MILLISECOND:
145 0 : totalMs = Number(value);
146 0 : break;
147 1 : case arrow.TimeUnit.MICROSECOND:
148 1 : totalMs = Number(value / 1000n);
149 1 : break;
150 1 : case arrow.TimeUnit.NANOSECOND:
151 1 : totalMs = Number(value / 1000000n);
152 1 : break;
153 0 : default:
154 0 : totalMs = Number(value);
155 1 : }
156 : } else {
157 1 : switch (unit) {
158 1 : case arrow.TimeUnit.SECOND:
159 1 : totalMs = value * 1000;
160 1 : break;
161 1 : case arrow.TimeUnit.MILLISECOND:
162 1 : totalMs = value;
163 1 : break;
164 0 : case arrow.TimeUnit.MICROSECOND:
165 0 : totalMs = value / 1000;
166 0 : break;
167 0 : case arrow.TimeUnit.NANOSECOND:
168 0 : totalMs = value / 1000000;
169 0 : break;
170 0 : default:
171 0 : totalMs = value;
172 1 : }
173 : }
174 :
175 1 : const hours = Math.floor(totalMs / 3600000);
176 1 : const minutes = Math.floor((totalMs % 3600000) / 60000);
177 1 : const seconds = Math.floor((totalMs % 60000) / 1000);
178 1 : const ms = Math.floor(totalMs % 1000);
179 :
180 1 : if (ms > 0) {
181 1 : return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.${ms.toString().padStart(3, '0')}`;
182 : }
183 1 : return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`;
184 1 : }
185 :
186 : /// Format a struct value to a string representation
187 1 : function formatStruct(value: Map<string, any> | Record<string, any>): string {
188 1 : if (value == null) return '';
189 1 : const entries: string[] = [];
190 1 : const maxEntries = 5;
191 1 : let count = 0;
192 :
193 1 : const iterate = (key: string, val: any) => {
194 1 : if (count >= maxEntries) return;
195 1 : let formatted: string;
196 1 : if (val === null) {
197 1 : formatted = 'null';
198 1 : } else if (typeof val === 'string') {
199 1 : formatted = `"${val}"`;
200 1 : } else if (typeof val === 'number' || typeof val === 'bigint') {
201 1 : formatted = String(val);
202 : } else {
203 0 : formatted = String(val);
204 : }
205 1 : entries.push(`${key}: ${formatted}`);
206 1 : count++;
207 1 : };
208 :
209 1 : if (value instanceof Map) {
210 1 : for (const [key, val] of value) {
211 1 : iterate(key, val);
212 : }
213 1 : if (value.size > maxEntries) {
214 0 : entries.push(`... +${value.size - maxEntries} more`);
215 : }
216 1 : } else {
217 1 : const keys = Object.keys(value);
218 1 : for (const key of keys) {
219 1 : iterate(key, value[key]);
220 : }
221 1 : if (keys.length > maxEntries) {
222 1 : entries.push(`... +${keys.length - maxEntries} more`);
223 : }
224 1 : }
225 :
226 1 : return `{${entries.join(', ')}}`;
227 1 : }
228 :
229 : /// Format interval value
230 1 : function formatInterval(value: any, typeId: arrow.Type): string {
231 1 : if (value == null) return '';
232 :
233 : // IntervalDayTime: { days, milliseconds }
234 : // IntervalYearMonth: months as Int32
235 : // IntervalMonthDayNano: { months, days, nanoseconds }
236 :
237 1 : if (typeof value === 'number') {
238 : // IntervalYearMonth - value is months
239 1 : const years = Math.floor(value / 12);
240 1 : const months = value % 12;
241 1 : const parts: string[] = [];
242 1 : if (years !== 0) parts.push(`${years}y`);
243 1 : if (months !== 0 || parts.length === 0) parts.push(`${months}mo`);
244 1 : return parts.join(' ');
245 1 : }
246 :
247 1 : if (typeof value === 'object') {
248 1 : const parts: string[] = [];
249 1 : if ('days' in value && value.days !== 0) {
250 1 : parts.push(`${value.days}d`);
251 : }
252 1 : if ('months' in value && value.months !== 0) {
253 1 : const years = Math.floor(value.months / 12);
254 1 : const months = value.months % 12;
255 1 : if (years !== 0) parts.push(`${years}y`);
256 1 : if (months !== 0) parts.push(`${months}mo`);
257 1 : }
258 1 : if ('milliseconds' in value && value.milliseconds !== 0) {
259 1 : parts.push(`${value.milliseconds}ms`);
260 : }
261 1 : if ('nanoseconds' in value) {
262 1 : const ns = BigInt(value.nanoseconds);
263 1 : if (ns !== 0n) {
264 1 : const ms = ns / 1000000n;
265 1 : if (ms !== 0n) {
266 1 : parts.push(`${ms}ms`);
267 : } else {
268 1 : parts.push(`${ns}ns`);
269 : }
270 1 : }
271 1 : }
272 1 : return parts.length > 0 ? parts.join(' ') : '0';
273 1 : }
274 :
275 0 : return String(value);
276 1 : }
277 :
278 : export interface ColumnLayoutInfo {
279 : /// The header width
280 : headerWidth: number;
281 : /// The average width of values
282 : valueAvgWidth: number;
283 : /// The max width of values
284 : valueMaxWidth: number;
285 : }
286 :
287 : export interface ArrowColumnFormatter {
288 : /// The the column name
289 : getColumnName(): string;
290 : /// The statistics about the column layout
291 : getLayoutInfo(): ColumnLayoutInfo;
292 : /// We maintain a row to batch mapping per table.
293 : /// Individual column formatter therefore don't have to worry about resolving the "correct" batch.
294 : /// It's provided as input.
295 : getValue(batch: number, row: number): (string | null);
296 : }
297 :
298 : /// A per-value formatter for a single Arrow field: maps a (non-null) cell value to its display
299 : /// string, plus the CSS class the data-table uses for that value's alignment/appearance.
300 : export interface ArrowValueFormatter {
301 : /// Format a single cell value. Callers pass the raw Arrow value; `null`/`undefined` map to null.
302 : format: (value: unknown) => string | null;
303 : /// The value CSS class (number vs. text alignment) matching the field's type.
304 : valueClassName: string;
305 : }
306 :
307 : /// Format an epoch-millisecond count as a date, or as the plain count beyond the range of dates
308 1 : function formatEpochMilliseconds(format: Intl.DateTimeFormat, value: number | bigint): string {
309 1 : const date = new Date(Number(value));
310 1 : return Number.isNaN(date.getTime()) ? value.toString() : format.format(date);
311 1 : }
312 :
313 : /// Did Arrow JS refuse to read an Int64 value beyond the safe integers as a number?
314 1 : function isUnsafeIntegerConversion(error: unknown): boolean {
315 1 : return error instanceof TypeError && String(error).includes('not safe to convert');
316 1 : }
317 :
318 : /// Read the raw Int64 value of a row, for values that Arrow JS refuses to convert
319 1 : function readRawInt64(column: arrow.Vector, row: number): bigint | null {
320 1 : for (const chunk of column.data) {
321 1 : if (row >= chunk.length) {
322 0 : row -= chunk.length;
323 0 : continue;
324 0 : }
325 1 : const i = chunk.offset + row;
326 1 : const isValid = chunk.nullCount === 0 ||
327 1 : (chunk.nullBitmap != null && (chunk.nullBitmap[i >> 3] & (1 << (i & 7))) !== 0);
328 1 : if (!isValid) return null;
329 1 : return new DataView(chunk.values.buffer, chunk.values.byteOffset).getBigInt64(i * 8, true);
330 1 : }
331 0 : return null;
332 1 : }
333 :
334 : /// Build a value formatter for an Arrow field. This is the single source of truth for how each
335 : /// Arrow type renders as text — `ArrowTextColumnFormatter` uses it per column, and callers that
336 : /// need to format an individual cell reuse it directly instead of hand-rolling per-type
337 : /// stringification.
338 1 : export function makeArrowValueFormatter(field: arrow.Field, logger?: LoggerLike): ArrowValueFormatter {
339 : // The default: an unknown/unsupported type renders as empty text.
340 1 : let valueClassName = styles.data_value_text;
341 1 : let formatter: (o: any) => (null | string) = _ => "";
342 :
343 1 : switch (field.type.typeId) {
344 1 : case arrow.Type.Int:
345 1 : case arrow.Type.Int16:
346 1 : case arrow.Type.Int32:
347 1 : case arrow.Type.Int64:
348 1 : case arrow.Type.Float:
349 1 : case arrow.Type.Float16:
350 1 : case arrow.Type.Float32:
351 1 : case arrow.Type.Float64: {
352 1 : valueClassName = styles.data_value_number;
353 1 : const fmt = Intl.NumberFormat('en-US');
354 1 : formatter = (v: number) => (v == null ? null : fmt.format(v));
355 1 : break;
356 1 : }
357 1 : case arrow.Type.Decimal: {
358 1 : valueClassName = styles.data_value_number;
359 1 : const decimalType = field.type as arrow.Decimal;
360 1 : formatter = (v: any) => {
361 1 : const i = Int128.decodeLE(v);
362 1 : return Decimal128.format(i, decimalType.scale);
363 1 : }
364 1 : break;
365 1 : }
366 1 : case arrow.Type.Utf8:
367 1 : case arrow.Type.LargeUtf8:
368 1 : valueClassName = styles.data_value_text;
369 1 : formatter = (v: string) => v || null;
370 1 : break;
371 1 : case arrow.Type.Bool: {
372 1 : valueClassName = styles.data_value_text;
373 1 : formatter = (v: boolean) => (v == null ? null : v ? 'true' : 'false');
374 1 : break;
375 : }
376 1 : case arrow.Type.Null: {
377 1 : formatter = () => null;
378 1 : break;
379 : }
380 1 : case arrow.Type.Binary:
381 1 : case arrow.Type.LargeBinary:
382 1 : case arrow.Type.FixedSizeBinary: {
383 1 : valueClassName = styles.data_value_text;
384 1 : formatter = (v: Uint8Array) => (v == null ? null : formatBinary(v));
385 1 : break;
386 : }
387 1 : case arrow.Type.Time:
388 1 : case arrow.Type.TimeSecond:
389 1 : case arrow.Type.TimeMillisecond:
390 1 : case arrow.Type.TimeMicrosecond:
391 1 : case arrow.Type.TimeNanosecond: {
392 1 : valueClassName = styles.data_value_text;
393 1 : const timeType = field.type as arrow.Time;
394 1 : formatter = (v: number | bigint) => (v == null ? null : formatTime(v, timeType.unit));
395 1 : break;
396 1 : }
397 1 : case arrow.Type.Timestamp:
398 1 : case arrow.Type.TimestampSecond:
399 1 : case arrow.Type.TimestampMillisecond:
400 1 : case arrow.Type.TimestampMicrosecond:
401 1 : case arrow.Type.TimestampNanosecond: {
402 1 : valueClassName = styles.data_value_text;
403 : // Arrow's GetVisitor already normalizes every timestamp unit to milliseconds when a
404 : // value is read (seconds ×1000, micro ÷1000, nano ÷1e6). The value handed to us is
405 : // therefore always an epoch-millisecond count regardless of `type.unit`; applying a
406 : // second per-unit conversion here would double-scale it (e.g. microsecond `now()`
407 : // rendered as 1970). Just interpret the value as milliseconds.
408 1 : const fmt = Intl.DateTimeFormat('en-US', { dateStyle: 'short', timeStyle: 'medium' });
409 1 : formatter = (v: number | bigint) => (v == null ? null : formatEpochMilliseconds(fmt, v));
410 1 : break;
411 1 : }
412 0 : case arrow.Type.DateMillisecond:
413 0 : case arrow.Type.DateDay:
414 1 : case arrow.Type.Date: {
415 1 : valueClassName = styles.data_value_text;
416 1 : const fmt = Intl.DateTimeFormat('en-US', {
417 1 : weekday: 'short',
418 1 : day: 'numeric',
419 1 : month: 'short',
420 1 : year: 'numeric',
421 1 : });
422 1 : formatter = (v: number | bigint) => (v == null ? null : formatEpochMilliseconds(fmt, v));
423 1 : break;
424 1 : }
425 1 : case arrow.Type.Duration:
426 1 : case arrow.Type.DurationSecond:
427 1 : case arrow.Type.DurationMillisecond:
428 1 : case arrow.Type.DurationMicrosecond:
429 1 : case arrow.Type.DurationNanosecond: {
430 1 : valueClassName = styles.data_value_text;
431 1 : const durationType = field.type as arrow.Duration;
432 1 : formatter = (v: bigint) => {
433 1 : if (v == null) return null;
434 1 : return formatDuration(v, durationType.unit);
435 1 : };
436 1 : break;
437 1 : }
438 1 : case arrow.Type.List:
439 1 : case arrow.Type.FixedSizeList: {
440 1 : valueClassName = styles.data_value_text;
441 1 : formatter = (v: arrow.Vector) => {
442 1 : if (v == null) return null;
443 1 : return formatList(v);
444 1 : };
445 1 : break;
446 : }
447 1 : case arrow.Type.Struct: {
448 1 : valueClassName = styles.data_value_text;
449 1 : formatter = (v: any) => {
450 1 : if (v == null) return null;
451 1 : return formatStruct(v);
452 1 : };
453 1 : break;
454 : }
455 1 : case arrow.Type.Map: {
456 1 : valueClassName = styles.data_value_text;
457 1 : formatter = (v: any) => {
458 1 : if (v == null) return null;
459 1 : return formatStruct(v);
460 1 : };
461 1 : break;
462 : }
463 1 : case arrow.Type.Interval:
464 1 : case arrow.Type.IntervalDayTime:
465 1 : case arrow.Type.IntervalYearMonth:
466 1 : case arrow.Type.IntervalMonthDayNano: {
467 1 : valueClassName = styles.data_value_text;
468 1 : const intervalType = field.type;
469 1 : formatter = (v: any) => {
470 1 : if (v == null) return null;
471 1 : return formatInterval(v, intervalType.typeId);
472 1 : };
473 1 : break;
474 1 : }
475 1 : case arrow.Type.Union:
476 1 : case arrow.Type.DenseUnion:
477 1 : case arrow.Type.SparseUnion: {
478 1 : valueClassName = styles.data_value_text;
479 1 : formatter = (v: any) => {
480 1 : if (v == null) return null;
481 1 : if (typeof v === 'string') return v;
482 1 : if (typeof v === 'number' || typeof v === 'bigint') return String(v);
483 1 : return JSON.stringify(v);
484 1 : };
485 1 : break;
486 : }
487 1 : case arrow.Type.Dictionary: {
488 : // Dictionary values are decoded to their actual values by Arrow
489 1 : valueClassName = styles.data_value_text;
490 1 : formatter = (v: any) => {
491 1 : if (v == null) return null;
492 1 : if (typeof v === 'string') return v;
493 0 : if (typeof v === 'number' || typeof v === 'bigint') return String(v);
494 0 : return String(v);
495 1 : };
496 1 : break;
497 : }
498 1 : default:
499 1 : logger?.warn("unsupport column type in Arrow text formatter", {
500 1 : typeId: field.type.toString(),
501 1 : field: field.name.toString(),
502 1 : }, LOG_CTX);
503 : break;
504 1 : }
505 :
506 1 : return { format: (v: unknown) => (v == null ? null : formatter(v)), valueClassName };
507 1 : }
508 :
509 1 : export class ArrowTextColumnFormatter implements ArrowColumnFormatter {
510 1 : readonly logger: LoggerLike;
511 1 : readonly columnId: number;
512 1 : readonly columnName: string;
513 1 : readonly batches: arrow.RecordBatch[];
514 1 : readonly batchValues: ((string | null)[] | null)[];
515 1 : readonly loadedBatchValueCount: Uint32Array;
516 1 : readonly batchFormattedLengthSum: Float64Array;
517 1 : readonly batchFormattedLengthMax: Uint32Array;
518 1 : readonly valueClassName: string;
519 1 : formattedRowCount: number;
520 1 : formattedLengthMax: number;
521 1 : formattedLengthSum: number;
522 1 : formatter: ((o: any) => (null | string));
523 :
524 1 : public constructor(
525 1 : logger: LoggerLike,
526 1 : columnId: number,
527 1 : schema: arrow.Schema,
528 1 : batches: arrow.RecordBatch[]
529 1 : ) {
530 1 : this.logger = logger;
531 1 : this.columnId = columnId;
532 1 : this.columnName = schema.fields[columnId].name;
533 1 : this.batches = batches;
534 1 : this.batchValues = Array.from({ length: batches.length }, () => null);
535 1 : this.loadedBatchValueCount = new Uint32Array(batches.length);
536 1 : this.batchFormattedLengthSum = new Float64Array(batches.length);
537 1 : this.batchFormattedLengthMax = new Uint32Array(batches.length);
538 1 : this.formattedRowCount = 0;
539 1 : this.formattedLengthMax = 0;
540 1 : this.formattedLengthSum = 0;
541 :
542 : // Setup the formatter (shared with single-value callers via makeArrowValueFormatter).
543 1 : const valueFormatter = makeArrowValueFormatter(schema.fields[columnId], logger);
544 1 : this.valueClassName = valueFormatter.valueClassName;
545 1 : this.formatter = valueFormatter.format;
546 1 : }
547 :
548 : /// We do not eagerly format all columns.
549 : /// Instead, we just check lazily if a batch is loaded before resolving a value.
550 1 : public ensureBatchIsLoaded(index: number): (string | null)[] {
551 1 : if (this.batchValues[index] != null && this.loadedBatchValueCount[index] === this.batches[index].numRows) {
552 1 : return this.batchValues[index]!;
553 : }
554 1 : const data = this.batches[index];
555 1 : const column = data.getChildAt(this.columnId)!;
556 :
557 1 : this.formattedRowCount -= this.loadedBatchValueCount[index];
558 1 : this.formattedLengthSum -= this.batchFormattedLengthSum[index];
559 :
560 1 : const values: (string | null)[] = [];
561 1 : let valueLengthSum = 0;
562 1 : let valueLengthMax = 0;
563 :
564 1 : const pushFormatted = (value: any) => {
565 1 : if (value == null) {
566 1 : values.push(null);
567 1 : } else {
568 1 : const text = this.formatter(value) || '';
569 1 : values.push(text);
570 1 : valueLengthSum += text.length;
571 1 : valueLengthMax = Math.max(valueLengthMax, text.length);
572 1 : }
573 1 : };
574 :
575 1 : try {
576 1 : for (const value of column) {
577 1 : pushFormatted(value);
578 : }
579 1 : } catch (e) {
580 1 : if (!isUnsafeIntegerConversion(e)) {
581 0 : throw e;
582 : }
583 : // Arrow JS throws for Int64 values exceeding MAX_SAFE_INTEGER.
584 : // Fall back to reading raw bigint values via DataView.
585 1 : values.length = 0;
586 1 : valueLengthSum = 0;
587 1 : valueLengthMax = 0;
588 1 : for (let row = 0; row < column.length; ++row) {
589 1 : pushFormatted(readRawInt64(column, row));
590 : }
591 1 : }
592 :
593 1 : this.formattedLengthSum += valueLengthSum;
594 1 : this.formattedRowCount += values.length;
595 1 : this.batchValues[index] = values;
596 1 : this.loadedBatchValueCount[index] = values.length;
597 1 : this.batchFormattedLengthSum[index] = valueLengthSum;
598 1 : this.batchFormattedLengthMax[index] = valueLengthMax;
599 1 : this.formattedLengthMax = Math.max(...this.batchFormattedLengthMax);
600 1 : return values;
601 1 : }
602 :
603 1 : public getBatchValues(batch: number) {
604 1 : return this.ensureBatchIsLoaded(batch);
605 1 : }
606 1 : public getValue(batch: number, row: number): string | null {
607 1 : const loaded = this.batchValues[batch];
608 1 : if (loaded != null && this.loadedBatchValueCount[batch] === this.batches[batch].numRows) {
609 1 : return loaded[row];
610 : }
611 :
612 1 : const values = loaded ?? new Array<string | null>(this.batches[batch].numRows);
613 1 : if (loaded == null) {
614 1 : this.batchValues[batch] = values;
615 : }
616 1 : if (row in values) {
617 1 : return values[row];
618 : }
619 :
620 1 : const column = this.batches[batch].getChildAt(this.columnId)!;
621 1 : let raw: unknown;
622 1 : try {
623 1 : raw = column.get(row);
624 1 : } catch (e) {
625 1 : if (!isUnsafeIntegerConversion(e)) throw e;
626 1 : raw = readRawInt64(column, row);
627 1 : }
628 1 : const text = raw == null ? null : (this.formatter(raw) || '');
629 1 : values[row] = text;
630 1 : this.loadedBatchValueCount[batch]++;
631 1 : this.formattedRowCount++;
632 1 : if (text != null) {
633 1 : this.formattedLengthSum += text.length;
634 1 : this.formattedLengthMax = Math.max(this.formattedLengthMax, text.length);
635 1 : this.batchFormattedLengthSum[batch] += text.length;
636 1 : this.batchFormattedLengthMax[batch] = Math.max(this.batchFormattedLengthMax[batch], text.length);
637 1 : }
638 1 : return text;
639 1 : }
640 1 : public getColumnName(): string {
641 1 : return this.columnName;
642 1 : }
643 1 : public getLayoutInfo(): ColumnLayoutInfo {
644 1 : return {
645 1 : headerWidth: this.columnName.length,
646 1 : valueMaxWidth: this.formattedLengthMax,
647 1 : valueAvgWidth: this.formattedLengthSum / Math.max(this.formattedRowCount, 1),
648 1 : };
649 1 : }
650 1 : }
651 :
652 1 : export class ArrowTableFormatter {
653 : /// The formatters for the individual columns
654 1 : columns: ArrowColumnFormatter[];
655 : /// Stores the offsets of a batch
656 1 : batchOffsets: Uint32Array;
657 : /// Matches rows to batches
658 1 : rowIndex: Uint32Array;
659 :
660 1 : public constructor(schema: arrow.Schema, batches: arrow.RecordBatch[], logger: LoggerLike) {
661 : // Resolve the starting offset for all batches
662 1 : const batchOffsets = new Uint32Array(batches.length);
663 1 : let numRows = 0;
664 1 : for (let i = 0; i < batches.length; ++i) {
665 1 : batchOffsets[i] = numRows;
666 1 : numRows += batches[i].numRows;
667 1 : }
668 : // Write for each row in which batch they are stored.
669 : // This is not cheap but gives us very fast access later.
670 1 : const rowIndex = new Uint32Array(numRows);
671 1 : let rowIndexWriter = 0;
672 1 : for (let i = 0; i < batches.length; ++i) {
673 1 : for (let j = 0; j < batches[i].numRows; ++j) {
674 1 : rowIndex[rowIndexWriter++] = i;
675 : }
676 : }
677 1 : const columns: ArrowColumnFormatter[] = [];
678 1 : for (let i = 0; i < schema.fields.length; ++i) {
679 1 : const renderer = new ArrowTextColumnFormatter(logger, i, schema, batches);
680 1 : columns.push(renderer);
681 1 : }
682 1 : this.columns = columns;
683 1 : this.batchOffsets = batchOffsets;
684 1 : this.rowIndex = rowIndex;
685 1 : }
686 :
687 1 : public getValue(row: number, column: number): (string | null) {
688 1 : const batch = this.rowIndex[row];
689 1 : const indexInBatch = row - this.batchOffsets[batch];
690 1 : return this.columns[column].getValue(batch, indexInBatch);
691 1 : }
692 1 : }
693 :
694 1 : export function dataTypeToString(t: arrow.DataType): string {
695 1 : switch (t.typeId) {
696 1 : case arrow.Type.Decimal: {
697 1 : const d = t as arrow.Decimal;
698 1 : return `Decimal(${d.precision},${d.scale})`;
699 1 : }
700 1 : case arrow.Type.Timestamp: {
701 1 : const ts = t as arrow.Timestamp;
702 1 : let unit = "";
703 1 : switch (ts.unit) {
704 1 : case arrow.TimeUnit.SECOND:
705 1 : unit = "s";
706 1 : break;
707 1 : case arrow.TimeUnit.MILLISECOND:
708 1 : unit = "ms";
709 1 : break;
710 1 : case arrow.TimeUnit.MICROSECOND:
711 1 : unit = "us";
712 1 : break;
713 1 : case arrow.TimeUnit.NANOSECOND:
714 1 : unit = "ns";
715 : break;
716 1 : }
717 1 : return `Timestamp(${unit})`;
718 1 : }
719 1 : case arrow.Type.Utf8:
720 1 : return `Text`;
721 1 : default:
722 1 : return t.toString();
723 1 : }
724 1 : }
|
