Line data    Source code

       1              : import * as dashql from '@ankoh/dashql-core';
       2              : import * as connection from '@ankoh/dashql-jsonschema/connection.js';
       3              : 
       4              : import { SalesforceApiClientInterface, type SalesforceMetadataProgress } from './salesforce_api_client.js';
       5              : import { getSalesforceDataSpace } from './salesforce_api_client.js';
       6              : import { SalesforceConnectionStateDetails } from './salesforce_connection_state.js';
       7              : import { generateSchemaSQL, generateCatalogScriptHeader, CatalogSource, type ColumnMetadata } from '../catalog/catalog_sql_generator.js';
       8              : import { LoggerLike } from '../../platform/logger/logger.js';
       9              : import { fetchPrefetchedHyperFunctions } from '../catalog/prefetched_hyper_functions.js';
      10              : import { beginCatalogBatch, publishCatalogBatch, type CatalogBatchOptions } from '../catalog/catalog_batch.js';
      11              : import type { AttachedDatabase } from '../hyper/hyperdb_grpc_client.js';
      12              : 
      13            1 : export const SALESFORCE_CATALOG_DATABASE = 'lakehouse';
      14            1 : export const SALESFORCE_CATALOG_SCHEMA = 'public';
      15              : /// The Hyper Cloud database of SIDB connections that don't name one
      16            1 : export const SIDB_DEFAULT_DATABASE_NAME = 'default';
      17              : 
      18              : /// The Hyper Cloud database that SIDB connections attach as the catalog database
      19            1 : export function getSidbAttachedDatabase(params: connection.SalesforceConnectionParams | undefined): AttachedDatabase {
      20            1 :     const databaseName = params?.databaseName || SIDB_DEFAULT_DATABASE_NAME;
      21            1 :     return { path: `hyper.cloud:${databaseName}`, alias: SALESFORCE_CATALOG_DATABASE };
      22            1 : }
      23              : 
      24              : export interface ResolvedSalesforceCatalog {
      25              :     tables: Map<string, ColumnMetadata[]>;
      26              :     functionsSQL: string;
      27              :     tableCount: number;
      28              :     columnCount: number;
      29              : }
      30              : 
      31            1 : export async function resolveSalesforceCatalog(
      32            1 :     logger: LoggerLike,
      33            1 :     coreAccessToken: connection.SalesforceCoreAccessToken,
      34            1 :     dataCloudAccessToken: connection.SalesforceDataCloudAccessToken,
      35            1 :     api: SalesforceApiClientInterface,
      36            1 :     signal: AbortSignal,
      37            1 :     onProgress?: (progress: SalesforceMetadataProgress) => void,
      38            1 : ): Promise<ResolvedSalesforceCatalog> {
      39            1 :     if (!coreAccessToken.accessToken || !coreAccessToken.instanceUrl) {
      40            0 :         throw new Error('Salesforce core access token is missing');
      41              :     }
      42            1 :     const dataSpace = getSalesforceDataSpace(dataCloudAccessToken);
      43            1 :     logger.info('Resolving Salesforce catalog metadata', { dataSpace }, 'salesforce_catalog');
      44            1 :     const metadataStartedAt = performance.now();
      45            1 :     const [metadata, functionsSQL] = await Promise.all([
      46            1 :         api.getDataCloudMetadata(coreAccessToken, dataSpace, signal, onProgress),
      47            1 :         fetchPrefetchedHyperFunctions(signal),
      48            1 :     ]);
      49            1 :     logger.info('Received Salesforce catalog metadata', {
      50            1 :         dataSpace,
      51            1 :         entities: (metadata.metadata?.length ?? 0).toString(),
      52            1 :         durationMs: (performance.now() - metadataStartedAt).toFixed(2),
      53            1 :     }, 'salesforce_catalog');
      54              : 
      55            1 :     const tables = new Map<string, ColumnMetadata[]>();
      56            1 :     for (const entry of metadata.metadata ?? []) {
      57            1 :         tables.set(entry.name, (entry.fields ?? []).map((field, ordinalPosition) => ({
      58            1 :             name: field.name,
      59            1 :             ordinalPosition,
      60            1 :             dataType: field.type ?? null,
      61            1 :         })));
      62              :     }
      63            1 :     const columnCount = Array.from(tables.values()).reduce((total, columns) => total + columns.length, 0);
      64            1 :     if (tables.size === 0 || columnCount === 0) {
      65            1 :         throw new Error('Salesforce metadata returned no usable catalog relations');
      66              :     }
      67            1 :     return { tables, functionsSQL, tableCount: tables.size, columnCount };
      68            1 : }
      69              : 
      70            1 : export async function updateSalesforceCatalog(
      71            1 :     logger: LoggerLike,
      72            1 :     conn: SalesforceConnectionStateDetails,
      73            1 :     catalog: dashql.DashQLCatalog,
      74            1 :     dql: dashql.DashQL,
      75            1 :     catalogRelationScript: dashql.DashQLScript,
      76            1 :     catalogFunctionScript: dashql.DashQLScript,
      77            1 :     api: SalesforceApiClientInterface,
      78            1 :     abortController: AbortController,
      79            1 :     options: CatalogBatchOptions = {},
      80            1 : ): Promise<dashql.DashQLScript> {
      81            1 :     options = { ...options, abortSignal: abortController.signal };
      82            1 :     const validate = beginCatalogBatch(catalog, options);
      83            1 :     const coreAccessToken = conn.proto.oauthState?.coreAccessToken;
      84            1 :     if (!coreAccessToken?.accessToken || !coreAccessToken.instanceUrl) {
      85            0 :         throw new Error(`Salesforce core access token is missing`);
      86              :     }
      87            1 :     if (!conn.proto.oauthState?.dataCloudAccessToken) {
      88            0 :         throw new Error('Salesforce data cloud access token is missing');
      89              :     }
      90            1 :     const dataSpace = getSalesforceDataSpace(conn.proto.oauthState.dataCloudAccessToken);
      91            1 :     const { tables, functionsSQL, columnCount } = await resolveSalesforceCatalog(
      92            1 :         logger,
      93            1 :         coreAccessToken,
      94            1 :         conn.proto.oauthState.dataCloudAccessToken,
      95            1 :         api,
      96            1 :         abortController.signal,
      97            1 :     );
      98            1 :     validate();
      99              : 
     100              :     // Generate SQL from metadata
     101            1 :     const header = generateCatalogScriptHeader(CatalogSource.SalesforceMetadataApi);
     102            1 :     const catalogSQL = generateSchemaSQL(SALESFORCE_CATALOG_DATABASE, SALESFORCE_CATALOG_SCHEMA, tables);
     103            1 :     logger.info("Generated Salesforce catalog script", {
     104            1 :         dataSpace,
     105            1 :         tables: tables.size.toString(),
     106            1 :         columns: columnCount.toString(),
     107            1 :         scriptBytes: new TextEncoder().encode(catalogSQL).byteLength.toString(),
     108            1 :     }, "salesforce_catalog");
     109              : 
     110            1 :     const [relations, functions] = await publishCatalogBatch(dql, catalog, [
     111            1 :         { script: catalogRelationScript, text: `${header}${catalogSQL}` },
     112            1 :         { script: catalogFunctionScript, text: functionsSQL, requireFunctions: true },
     113            1 :     ], options, validate);
     114            1 :     logger.info("Loaded Salesforce catalog script", {
     115            1 :         dataSpace,
     116            1 :         tables: relations.tables.toString(),
     117            1 :         functions: functions.functions.toString(),
     118            1 :     }, "salesforce_catalog");
     119              : 
     120            1 :     return catalogRelationScript;
     121            1 : }