Line data    Source code

       1              : import * as arrow from 'apache-arrow';
       2              : 
       3              : import { ArrowTableFormatter } from './arrow_formatter.js';
       4              : import { TableAggregation, OrdinalColumnAnalysis, StringColumnAnalysis, StringGridColumnGroup, OrdinalGridColumnGroup, BinnedValuesTable, FrequentValuesTable } from './computation_types.js';
       5              : import { assert } from '../utils/assert.js';
       6              : import { isTemporalType } from './arrow_columns.js';
       7              : 
       8            1 : export function analyzeOrdinalColumn(tableSummary: TableAggregation, columnEntry: OrdinalGridColumnGroup, binnedValues: BinnedValuesTable, binnedValuesFormatter: ArrowTableFormatter, filteredTotalCount: number | null = null): OrdinalColumnAnalysis {
       9            1 :     const totalCountVector = tableSummary.table.getChild(tableSummary.countStarFieldName!) as arrow.Vector<arrow.Int64>;
      10              : 
      11            1 :     const unfilteredTotalCount = Number(totalCountVector.get(0) ?? BigInt(0));
      12            1 :     const totalCount = filteredTotalCount ?? unfilteredTotalCount;
      13            1 :     const minFieldId = tableSummary.tableFieldsByName.get(columnEntry.statsFields!.minAggregateFieldName!)!;
      14            1 :     const maxFieldId = tableSummary.tableFieldsByName.get(columnEntry.statsFields!.maxAggregateFieldName!)!;
      15            1 :     const minValue = tableSummary.tableFormatter.getValue(0, minFieldId) ?? "";
      16            1 :     const maxValue = tableSummary.tableFormatter.getValue(0, maxFieldId) ?? "";
      17              : 
      18            1 :     assert(binnedValues.schema.fields[0].name == "bin");
      19            1 :     assert(binnedValues.schema.fields[1].name == "count");
      20            1 :     assert(binnedValues.schema.fields[3].name == "binLowerBound");
      21            1 :     const binIdVector = binnedValues.getChildAt(0) as arrow.Vector<arrow.Uint32>;
      22            1 :     const binCountVector = binnedValues.getChildAt(1) as arrow.Vector<arrow.Int64>;
      23              : 
      24            1 :     const nullBinRowIdx = binnedValues.numRows - 1;
      25            1 :     const nullBinId = Number(binIdVector.get(nullBinRowIdx) ?? -1);
      26            1 :     assert(binnedValues.numRows === (columnEntry.binCount + 1));
      27            1 :     assert(nullBinId === columnEntry.binCount);
      28              : 
      29            1 :     const countNull = Number(binCountVector.get(nullBinRowIdx) ?? BigInt(0));
      30            1 :     const countNotNull = filteredTotalCount == null
      31            1 :         ? totalCount - countNull
      32            1 :         : unfilteredTotalCount - countNull;
      33              : 
      34            1 :     const regularBinCount = columnEntry.binCount;
      35            1 :     const binLowerBounds: string[] = [];
      36            1 :     const binPercentages = new Float64Array(regularBinCount);
      37            1 :     const regularBinValueCounts = new BigInt64Array(regularBinCount);
      38              : 
      39            1 :     const isTemporal = isTemporalType(columnEntry.inputFieldType.typeId);
      40            1 :     const temporalFmt = isTemporal ? Intl.DateTimeFormat('en-US', { dateStyle: 'short', timeStyle: 'medium' }) : null;
      41            1 :     const lbCol = isTemporal ? binnedValues.getChildAt(3)! : null;
      42              : 
      43            1 :     for (let i = 0; i < regularBinCount; ++i) {
      44            1 :         const binCount = binCountVector.get(i) ?? BigInt(0);
      45            1 :         let binLB: string;
      46            1 :         if (isTemporal && lbCol) {
      47            1 :             const epochMs = lbCol.get(i);
      48            1 :             binLB = epochMs == null ? "" : temporalFmt!.format(new Date(epochMs));
      49            1 :         } else {
      50            1 :             binLB = binnedValuesFormatter.getValue(i, 3) ?? "";
      51              :         }
      52            1 :         const binPercentage = (totalCount == 0) ? 0 : (Number(binCount) / totalCount);
      53            1 :         binLowerBounds.push(binLB);
      54            1 :         binPercentages[i] = binPercentage;
      55            1 :         regularBinValueCounts[i] = binCount;
      56            1 :     }
      57              : 
      58            1 :     return {
      59            1 :         totalCount,
      60            1 :         countNotNull: countNotNull,
      61            1 :         countNull: countNull,
      62            1 :         minValue: minValue,
      63            1 :         maxValue: maxValue,
      64            1 :         binCount: regularBinCount,
      65            1 :         binValueCounts: regularBinValueCounts,
      66            1 :         binPercentages: binPercentages,
      67            1 :         binLowerBounds: binLowerBounds,
      68            1 :     };
      69            1 : }
      70              : 
      71            1 : export function analyzeStringColumn(tableSummary: TableAggregation, columnEntry: StringGridColumnGroup, frequentValueTable: FrequentValuesTable, frequentValuesFormatter: ArrowTableFormatter, filteredTotalCount: number | null = null): StringColumnAnalysis {
      72            1 :     const totalCountVector = tableSummary.table.getChild(tableSummary.countStarFieldName!) as arrow.Vector<arrow.Int64>;
      73            1 :     const notNullCountVector = tableSummary.table.getChild(columnEntry.statsFields!.countFieldName) as arrow.Vector<arrow.Int64>;
      74            1 :     const distinctCountVector = tableSummary.table.getChild(columnEntry.statsFields!.distinctCountFieldName!) as arrow.Vector<arrow.Int64>;
      75              : 
      76            1 :     const unfilteredTotalCount = Number(totalCountVector.get(0) ?? BigInt(0));
      77            1 :     const totalCount = filteredTotalCount ?? unfilteredTotalCount;
      78            1 :     const notNullCount = Number(notNullCountVector.get(0) ?? BigInt(0));
      79            1 :     const distinctCount = Number(distinctCountVector.get(0) ?? BigInt(0));
      80              : 
      81            1 :     assert(frequentValueTable.schema.fields[0].name == "key");
      82            1 :     assert(frequentValueTable.getChild("count") != null);
      83              : 
      84            1 :     const frequentValueCounts = frequentValueTable.getChild("count")!.toArray();
      85            1 :     const frequentValuePercentages = new Float64Array(frequentValueTable.numRows);
      86            1 :     const frequentValueStrings = [];
      87            1 :     for (let i = 0; i < frequentValueTable.numRows; ++i) {
      88            1 :         frequentValuePercentages[i] = totalCount == 0 ? 0 : (Number(frequentValueCounts[i]) / totalCount);
      89            1 :         frequentValueStrings.push(frequentValuesFormatter.getValue(i, 0));
      90            1 :     }
      91              : 
      92            1 :     const keyIdColumn = (frequentValueTable as arrow.Table).getChild("keyId");
      93            1 :     if (keyIdColumn == null) {
      94            0 :         throw new Error("Missing keyId column in frequent values table");
      95              :     }
      96            1 :     const frequentValueIds = keyIdColumn.toArray() as BigInt64Array;
      97              : 
      98            1 :     return {
      99            1 :         totalCount,
     100            1 :         countNotNull: notNullCount,
     101            1 :         countNull: unfilteredTotalCount - notNullCount,
     102            1 :         countDistinct: distinctCount,
     103            1 :         isUnique: notNullCount == distinctCount,
     104            1 :         frequentValueStrings: frequentValueStrings,
     105            1 :         frequentValueIds: frequentValueIds,
     106            1 :         frequentValueCounts: frequentValueCounts,
     107            1 :         frequentValuePercentages: frequentValuePercentages,
     108            1 :     };
     109            1 : }