Line data Source code
1 : import * as connection from '@ankoh/dashql-jsonschema/connection.js';
2 : import * as pb from "../../proto.js";
3 :
4 : import { Logger } from '../../platform/logger/logger.js';
5 : import { HttpClient } from '../../platform/http/http_client.js';
6 : import { HyperDatabaseChannel, HyperQueryExecutionOptions, HyperQueryResultStream } from '../hyper/hyperdb_grpc_client.js';
7 : import { BASE64URL_CODEC } from "../../utils/base64.js";
8 :
9 1 : const LOG_CTX = "salesforce_api";
10 1 : const SALESFORCE_API_VERSION = "v64.0";
11 1 : export const DEFAULT_SALESFORCE_DATA_SPACE = "default";
12 1 : export const SALESFORCE_METADATA_ENTITY_TYPES = ['DataModelObject', 'DataLakeObject', 'CalculatedInsight'] as const;
13 :
14 : /// The Data Cloud auth infos
15 : export interface SalesforceAuthInfo {
16 : /// The core tenant id
17 : coreTenantId: string | null;
18 : /// The core access token
19 : coreAccessToken: string | null;
20 : /// The offcore jwt
21 : offcoreRawJwt: string;
22 : /// The offcore instance url
23 : offcoreInstanceUrl: string | null;
24 : /// The offcore tenant id
25 : offcoreTenantId: string | null;
26 : /// The offcore access token
27 : offcoreAccessToken: string | null;
28 : /// The dataspace
29 : dataspace: string;
30 : }
31 :
32 : export interface SalesforceMetadataProgress {
33 : collection: string;
34 : page: number;
35 : loaded: number;
36 : total: number | null;
37 : state: 'requesting' | 'complete' | 'failed';
38 : error?: string;
39 : }
40 :
41 1 : export function formatSalesforceMetadataProgress(
42 1 : progressByCollection: ReadonlyMap<string, SalesforceMetadataProgress>,
43 1 : ): string {
44 1 : const labels: Record<string, string> = {
45 1 : DataModelObject: 'DMO',
46 1 : DataLakeObject: 'DLO',
47 1 : CalculatedInsight: 'CI',
48 1 : };
49 1 : let completed = 0;
50 1 : let running = 0;
51 1 : let failed = 0;
52 1 : let pending = 0;
53 1 : const stages = SALESFORCE_METADATA_ENTITY_TYPES.map(collection => {
54 1 : const progress = progressByCollection.get(collection);
55 1 : if (progress?.state === 'complete') {
56 1 : completed += 1;
57 1 : return `${labels[collection]} ${progress.loaded}`;
58 1 : }
59 1 : if (progress?.state === 'failed') {
60 1 : failed += 1;
61 1 : return `${labels[collection]} failed`;
62 1 : }
63 1 : if (progress?.state === 'requesting') {
64 1 : running += 1;
65 1 : return `${labels[collection]} -`;
66 1 : }
67 0 : pending += 1;
68 0 : return `${labels[collection]} -`;
69 1 : });
70 1 : const counts = [`${completed} done`, `${running} active`, `${failed} failed`];
71 1 : if (pending > 0) counts.push(`${pending} pending`);
72 1 : return `Metadata: ${counts.join(', ')} | ${stages.join(' | ')}`;
73 1 : }
74 :
75 1 : export function getSalesforceDataSpace(access: connection.SalesforceDataCloudAccessToken): string {
76 1 : const customAttributes = access.jwt?.payload?.customAttributes as unknown;
77 1 : if (customAttributes && typeof customAttributes === "object") {
78 1 : const attributes = customAttributes as Record<string, unknown>;
79 1 : const nested = attributes.data;
80 1 : const dataSpace = nested && typeof nested === "object"
81 1 : ? (nested as Record<string, unknown>).dataspace
82 1 : : attributes.dataspace;
83 1 : if (typeof dataSpace === "string" && dataSpace.length > 0) {
84 1 : return dataSpace;
85 : }
86 1 : }
87 1 : return DEFAULT_SALESFORCE_DATA_SPACE;
88 1 : }
89 :
90 : /// Read the Salesforce auth tokens
91 1 : export function getSalesforceDataCloudUrl(endpoint: string | undefined): string {
92 1 : if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
93 0 : throw new Error('Data Cloud Instance URL cannot be empty');
94 : }
95 1 : const value = endpoint.trim();
96 1 : const normalized = /^[a-z][a-z0-9+.-]*:\/\//i.test(value) ? value : `https://${value}`;
97 1 : const url = new URL(normalized);
98 1 : if (!/^https:\/\/[^/\\\s@?#]+\/?$/i.test(normalized)
99 1 : || url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || url.pathname !== '/') {
100 1 : throw new Error('Data Cloud Instance URL must be an HTTPS origin without credentials, path, query, or fragment');
101 : }
102 1 : return url.toString();
103 1 : }
104 :
105 1 : export function collectSalesforceAuthInfo(coreToken: connection.SalesforceCoreAccessToken, offcoreToken: connection.SalesforceDataCloudAccessToken): SalesforceAuthInfo | null {
106 1 : const jwt = offcoreToken?.jwt;
107 1 : if (jwt) {
108 1 : return {
109 1 : offcoreRawJwt: jwt.raw,
110 1 : offcoreInstanceUrl: offcoreToken?.instanceUrl ?? null,
111 1 : offcoreTenantId: jwt.payload?.audienceTenantId ?? null,
112 1 : offcoreAccessToken: null,
113 1 : coreTenantId: jwt.payload?.audienceTenantId ?? null,
114 1 : coreAccessToken: coreToken?.accessToken ?? null,
115 1 : dataspace: getSalesforceDataSpace(offcoreToken),
116 1 : };
117 : } else {
118 0 : return null;
119 : }
120 1 : }
121 :
122 1 : function redactTokenDiagnostics(value: unknown): string {
123 1 : return JSON.stringify(value, (key, item) => {
124 : // Redact credentials and identities at every nesting level, not the routing context.
125 1 : const normalized = key.replace(/[-_]/g, '').toLowerCase();
126 1 : if ((/(token|secret|password|signature|assertion|authorization|email|username)/.test(normalized)
127 1 : && normalized !== 'tokentype' && normalized !== 'tokenformat')
128 1 : || ['id', 'sub', 'jti', 'clientid', 'userid', 'orgid', 'sfuid', 'sfoid', 'sfappid',
129 1 : 'tnk', 'sfi', 'sfapop', 'code', 'codeverifier', 'name', 'givenname', 'familyname', 'preferredusername'].includes(normalized)) {
130 1 : return '<redacted>';
131 : }
132 1 : return item;
133 1 : }) ?? 'null';
134 1 : }
135 :
136 1 : export function parseCoreAccessToken(obj: any, logger?: Logger): connection.SalesforceCoreAccessToken {
137 1 : if (typeof obj?.access_token !== 'string' || !obj.access_token) {
138 1 : logger?.warn('Rejected Salesforce OAuth token response', {
139 1 : oauthResponse: redactTokenDiagnostics(obj),
140 1 : }, LOG_CTX);
141 1 : throw new Error('Salesforce OAuth response is missing access_token');
142 1 : }
143 : // Core OAuth credentials may be opaque; only the exchanged Data Cloud token is decoded.
144 1 : return {
145 1 : createdAt: new Date().toISOString(),
146 1 : accessToken: obj.access_token,
147 1 : apiInstanceUrl: obj.api_instance_url,
148 1 : id: obj.id,
149 1 : idToken: obj.id_token,
150 1 : instanceUrl: obj.instance_url,
151 1 : issuedAt: obj.issued_at,
152 1 : refreshToken: obj.refresh_token,
153 1 : scope: obj.scope,
154 1 : signature: obj.signature,
155 1 : tokenType: obj.token_type,
156 1 : };
157 1 : }
158 :
159 1 : export function parseCoreUserInfo(obj: any): connection.SalesforceCoreUserInfo {
160 1 : return ({
161 1 : active: obj.active,
162 1 : email: obj.email,
163 1 : emailVerified: obj.email_verified,
164 1 : familyName: obj.family_name,
165 1 : givenName: obj.given_name,
166 1 : isAppInstalled: obj.is_app_installed,
167 1 : isSalesforceIntegrationUser: obj.is_salesforce_integration_user,
168 1 : language: obj.language,
169 1 : locale: obj.locale,
170 1 : name: obj.name,
171 1 : nickname: obj.nickname,
172 1 : organizationId: obj.organization_id,
173 1 : photos: obj.photos,
174 1 : picture: obj.picture,
175 1 : preferredUsername: obj.preferred_username,
176 1 : profile: obj.profile,
177 1 : sub: obj.sub,
178 1 : updatedAt: obj.updated_at,
179 1 : userId: obj.user_id,
180 1 : userType: obj.user_type,
181 1 : utcOffset: obj.utcOffset,
182 1 : zoneinfo: obj.zoneinfo,
183 1 : });
184 1 : }
185 :
186 : export interface SalesforceApiClientInterface {
187 : getCoreAccessToken(
188 : authConfig: connection.SalesforceOAuthConfig,
189 : authParams: connection.SalesforceConnectionParams,
190 : authCode: string,
191 : pkceVerifier: string,
192 : cancel: AbortSignal,
193 : ): Promise<connection.SalesforceCoreAccessToken>;
194 : getCoreUserInfo(
195 : access: connection.SalesforceCoreAccessToken,
196 : cancel: AbortSignal,
197 : ): Promise<connection.SalesforceCoreUserInfo>;
198 : getDataCloudMetadata(
199 : access: connection.SalesforceCoreAccessToken,
200 : dataSpace: string,
201 : cancel: AbortSignal,
202 : onProgress?: (progress: SalesforceMetadataProgress) => void,
203 : ): Promise<connection.SalesforceDataCloudMetadata>;
204 : getDataCloudAccessToken(
205 : access: connection.SalesforceCoreAccessToken,
206 : cancel: AbortSignal,
207 : ): Promise<connection.SalesforceDataCloudAccessToken>;
208 : }
209 :
210 1 : export class SalesforceApiClient implements SalesforceApiClientInterface {
211 1 : logger: Logger;
212 1 : httpClient: HttpClient;
213 :
214 1 : constructor(logger: Logger, httpClient: HttpClient) {
215 1 : this.logger = logger;
216 1 : this.httpClient = httpClient;
217 1 : }
218 :
219 1 : public async getCoreAccessToken(
220 1 : authConfig: connection.SalesforceOAuthConfig,
221 1 : authParams: connection.SalesforceConnectionParams,
222 1 : authCode: string,
223 1 : pkceVerifier: string,
224 1 : cancel: AbortSignal,
225 1 : ): Promise<connection.SalesforceCoreAccessToken> {
226 1 : const params: Record<string, string> = {
227 1 : grant_type: 'authorization_code',
228 1 : code: authCode!,
229 1 : redirect_uri: authConfig.oauthRedirect.toString(),
230 1 : client_id: authParams.appConsumerKey,
231 1 : code_verifier: pkceVerifier,
232 1 : format: 'json',
233 1 : };
234 1 : if (authParams.appConsumerSecret) {
235 0 : params.client_secret = authParams.appConsumerSecret;
236 : }
237 1 : const body = new URLSearchParams(params);
238 : // Get the access token
239 1 : const headers = new Headers({
240 1 : Accept: 'application/json',
241 1 : 'Content-Type': 'application/x-www-form-urlencoded',
242 1 : });
243 1 : const url = new URL(`${authParams.instanceUrl}/services/oauth2/token`);
244 1 : const response = await this.httpClient.fetch(url, {
245 1 : method: 'POST',
246 1 : headers,
247 1 : body: body,
248 1 : signal: cancel,
249 1 : });
250 1 : const responseBody = await response.json();
251 1 : if (response.status < 200 || response.status >= 300 || responseBody.error) {
252 1 : const errorDesc = responseBody.error_description ?? responseBody.error ?? `OAuth token request failed: ${response.status}`;
253 1 : this.logger.warn(errorDesc, {
254 1 : endpoint: url.toString(),
255 1 : status: response.status.toString(),
256 1 : oauthResponse: redactTokenDiagnostics(responseBody),
257 1 : }, LOG_CTX);
258 1 : throw new Error(errorDesc);
259 1 : } else {
260 1 : const parsed = parseCoreAccessToken(responseBody, this.logger);
261 1 : return parsed;
262 : }
263 1 : }
264 :
265 1 : protected readDataCloudAccessToken(obj: any): connection.SalesforceDataCloudAccessToken {
266 1 : if (typeof obj?.access_token !== 'string' || !obj.access_token) throw new Error('missing access_token');
267 1 : if (!obj.instance_url) throw new Error('missing instance_url');
268 1 : const jwtParts = obj.access_token.split('.');
269 1 : if (jwtParts.length !== 3) throw new Error(`invalid jwt, expected 3 parts, received ${jwtParts.length}`);
270 1 : const decoder = new TextDecoder();
271 1 : const header = JSON.parse(decoder.decode(BASE64URL_CODEC.decode(jwtParts[0])));
272 1 : const payload = JSON.parse(decoder.decode(BASE64URL_CODEC.decode(jwtParts[1])));
273 1 : if (!header || typeof header !== 'object' || Array.isArray(header)
274 1 : || !payload || typeof payload !== 'object' || Array.isArray(payload)) {
275 0 : throw new Error('invalid Data Cloud JWT');
276 : }
277 1 : const expiresAt = new Date(Number(payload.exp) * 1000);
278 1 : if (payload.exp == null || !Number.isFinite(expiresAt.getTime())) throw new Error('invalid Data Cloud JWT expiration');
279 1 : return {
280 1 : createdAt: new Date().toISOString(),
281 1 : tokenType: obj.token_type,
282 1 : issuedTokenType: obj.issued_token_type,
283 1 : expiresAt: expiresAt.toISOString(),
284 1 : jwt: {
285 1 : raw: obj.access_token,
286 1 : header,
287 1 : payload: { ...payload, customAttributes: payload.customAttributes ?? payload.custom_attributes },
288 1 : },
289 1 : instanceUrl: getSalesforceDataCloudUrl(obj.instance_url),
290 1 : };
291 1 : }
292 :
293 1 : public async getDataCloudAccessToken(
294 1 : access: connection.SalesforceCoreAccessToken,
295 1 : cancel: AbortSignal,
296 1 : ): Promise<connection.SalesforceDataCloudAccessToken> {
297 1 : const body = new URLSearchParams({
298 1 : grant_type: 'urn:salesforce:grant-type:external:cdp',
299 1 : subject_token: access.accessToken,
300 1 : subject_token_type: 'urn:ietf:params:oauth:token-type:access_token',
301 1 : });
302 1 : const url = new URL(`${access.instanceUrl ?? ''}/services/a360/token`);
303 1 : const response = await this.httpClient.fetch(url, {
304 1 : method: 'POST',
305 1 : headers: new Headers({ Accept: 'application/json', 'Content-Type': 'application/x-www-form-urlencoded' }),
306 1 : body,
307 1 : signal: cancel,
308 1 : });
309 1 : const responseBody = await response.json();
310 1 : if (response.status < 200 || response.status >= 300 || responseBody.error) {
311 1 : const message = responseBody.error_description ?? responseBody.error ?? `Data Cloud token request failed: ${response.status}`;
312 1 : this.logger.warn('Data Cloud token request failed', {
313 1 : endpoint: url.toString(), status: response.status.toString(), oauthResponse: redactTokenDiagnostics(responseBody),
314 1 : }, LOG_CTX);
315 1 : throw new Error(message);
316 1 : }
317 1 : try {
318 1 : return this.readDataCloudAccessToken(responseBody);
319 1 : } catch (error) {
320 1 : this.logger.warn('Rejected Data Cloud token response', { oauthResponse: redactTokenDiagnostics(responseBody) }, LOG_CTX);
321 1 : throw error;
322 1 : }
323 1 : }
324 :
325 1 : public async getCoreUserInfo(
326 1 : access: connection.SalesforceCoreAccessToken,
327 1 : cancel: AbortSignal,
328 1 : ): Promise<connection.SalesforceCoreUserInfo> {
329 1 : const params = new URLSearchParams();
330 1 : params.set('format', 'json');
331 1 : params.set('access_token', access.accessToken ?? '');
332 1 : const headers = new Headers({
333 1 : authorization: `Bearer ${access.accessToken}`,
334 1 : accept: 'application/json',
335 1 : });
336 1 : const url = new URL(`${access.instanceUrl ?? ""}/services/oauth2/userinfo?${params.toString()}`);
337 1 : const response = await this.httpClient.fetch(url, {
338 1 : headers,
339 1 : signal: cancel,
340 1 : });
341 1 : const responseJson = await response.json();
342 1 : return parseCoreUserInfo(responseJson);
343 1 : }
344 :
345 1 : public async getDataCloudMetadata(
346 1 : access: connection.SalesforceCoreAccessToken,
347 1 : dataSpace: string,
348 1 : cancel: AbortSignal,
349 1 : onProgress: (progress: SalesforceMetadataProgress) => void = () => { },
350 1 : ): Promise<connection.SalesforceDataCloudMetadata> {
351 1 : const base = (access.instanceUrl ?? "").replace(/\/+$/, '');
352 1 : const headers = new Headers({
353 1 : authorization: `Bearer ${access.accessToken}`,
354 1 : accept: 'application/json',
355 1 : });
356 1 : const endpoint = `${base}/services/data/${SALESFORCE_API_VERSION}/ssot/metadata`;
357 1 : this.logger.info("Requesting Salesforce metadata", {
358 1 : method: "GET",
359 1 : endpoint,
360 1 : apiVersion: SALESFORCE_API_VERSION,
361 1 : dataSpace,
362 1 : entityTypes: SALESFORCE_METADATA_ENTITY_TYPES.join(','),
363 1 : }, LOG_CTX);
364 1 : const requestStartedAt = performance.now();
365 1 : const requestResults = await Promise.allSettled(SALESFORCE_METADATA_ENTITY_TYPES.map(async entityType => {
366 1 : const url = new URL(endpoint);
367 1 : url.searchParams.set('dataspace', dataSpace);
368 1 : url.searchParams.set('entityType', entityType);
369 1 : onProgress({ collection: entityType, page: 1, loaded: 0, total: null, state: 'requesting' });
370 1 : try {
371 1 : const response = await this.httpClient.fetch(url, {
372 1 : headers,
373 1 : signal: cancel,
374 1 : });
375 1 : if (response.status < 200 || response.status >= 300) {
376 1 : const bodyText = await response.text().catch(() => '');
377 1 : throw new Error(`${response.status} ${response.statusText}${bodyText ? ` — ${bodyText.slice(0, 200)}` : ''}`);
378 1 : }
379 1 : const body = await response.json();
380 1 : const entities = Array.isArray(body.metadata) ? body.metadata : [];
381 1 : onProgress({
382 1 : collection: entityType,
383 1 : page: 1,
384 1 : loaded: entities.length,
385 1 : total: entities.length,
386 1 : state: 'complete',
387 1 : });
388 1 : return entities;
389 1 : } catch (error) {
390 1 : const message = error instanceof Error ? error.message : String(error);
391 1 : this.logger.warn('Salesforce metadata request failed', {
392 1 : dataSpace,
393 1 : entityType,
394 1 : error: message,
395 1 : }, LOG_CTX);
396 1 : onProgress({
397 1 : collection: entityType,
398 1 : page: 1,
399 1 : loaded: 0,
400 1 : total: null,
401 1 : state: 'failed',
402 1 : error: message,
403 1 : });
404 1 : throw new Error(`Salesforce ${entityType} metadata request failed: ${message}`);
405 1 : }
406 1 : }));
407 1 : const progressByCollection = new Map<string, SalesforceMetadataProgress>();
408 1 : const responseEntities: any[][] = [];
409 1 : let failedRequests = 0;
410 1 : requestResults.forEach((result, index) => {
411 1 : const collection = SALESFORCE_METADATA_ENTITY_TYPES[index];
412 1 : if (result.status === 'fulfilled') {
413 1 : responseEntities.push(result.value);
414 1 : progressByCollection.set(collection, {
415 1 : collection,
416 1 : page: 1,
417 1 : loaded: result.value.length,
418 1 : total: result.value.length,
419 1 : state: 'complete',
420 1 : });
421 1 : } else {
422 1 : const message = result.reason instanceof Error ? result.reason.message : String(result.reason);
423 1 : failedRequests += 1;
424 1 : progressByCollection.set(collection, {
425 1 : collection,
426 1 : page: 1,
427 1 : loaded: 0,
428 1 : total: null,
429 1 : state: 'failed',
430 1 : error: message,
431 1 : });
432 1 : }
433 1 : });
434 1 : if (failedRequests === SALESFORCE_METADATA_ENTITY_TYPES.length) {
435 1 : throw new Error(formatSalesforceMetadataProgress(progressByCollection));
436 : }
437 1 : this.logger.info("Received Salesforce metadata response", {
438 1 : durationMs: (performance.now() - requestStartedAt).toFixed(2),
439 1 : dataSpace,
440 1 : failedRequests: failedRequests.toString(),
441 1 : }, LOG_CTX);
442 :
443 : // Parse the Data Cloud metadata
444 1 : const entities: connection.SalesforceDataCloudMetadataEntity[] = [];
445 1 : let fieldCount = 0;
446 1 : for (const collectionEntities of responseEntities) {
447 1 : for (const entityJson of collectionEntities) {
448 1 : const rawFields = entityJson.fields ?? [];
449 1 : const fields: connection.SalesforceDataCloudMetadataEntityField[] = [];
450 1 : if (Array.isArray(rawFields)) {
451 1 : for (const fieldJson of rawFields) {
452 1 : fields.push(({
453 1 : name: fieldJson.name ?? '',
454 1 : displayName: fieldJson.displayName ?? '',
455 1 : type: fieldJson.type ?? '',
456 1 : businessType: fieldJson.businessType ?? '',
457 1 : }));
458 1 : fieldCount += 1;
459 1 : }
460 : }
461 1 : const primaryKeys: connection.SalesforceDataCloudMetadataPrimaryKey[] = [];
462 1 : if (entityJson.primaryKeys && Array.isArray(entityJson.primaryKeys)) {
463 1 : for (const pkJson of entityJson.primaryKeys) {
464 1 : primaryKeys.push(({
465 1 : indexOrder: pkJson.indexOrder ?? '',
466 1 : name: pkJson.name ?? '',
467 1 : displayName: pkJson.displayName ?? '',
468 1 : }));
469 : }
470 : }
471 1 : entities.push(({
472 1 : name: entityJson.name ?? '',
473 1 : displayName: entityJson.displayName ?? '',
474 1 : category: entityJson.category ?? '',
475 1 : fields: fields,
476 1 : primaryKeys: primaryKeys,
477 1 : }));
478 1 : }
479 : }
480 1 : this.logger.info("Parsed Salesforce metadata", {
481 1 : dataSpace,
482 1 : entities: entities.length.toString(),
483 1 : fields: fieldCount.toString(),
484 1 : }, LOG_CTX);
485 1 : return ({
486 1 : metadata: entities
487 1 : });
488 1 : }
489 1 : }
490 :
491 1 : export class SalesforceDatabaseChannel implements HyperDatabaseChannel {
492 : /// The api client
493 1 : protected apiClient: SalesforceApiClientInterface;
494 : /// The core access token
495 1 : public readonly coreToken: connection.SalesforceCoreAccessToken;
496 1 : public readonly dataCloudToken: connection.SalesforceDataCloudAccessToken;
497 : /// The Hyper database channel
498 1 : hyperChannel: HyperDatabaseChannel;
499 :
500 : /// The constructor
501 1 : constructor(apiClient: SalesforceApiClientInterface, coreToken: connection.SalesforceCoreAccessToken, dataCloudToken: connection.SalesforceDataCloudAccessToken, channel: HyperDatabaseChannel) {
502 1 : this.apiClient = apiClient;
503 1 : this.coreToken = coreToken;
504 1 : this.dataCloudToken = dataCloudToken;
505 1 : this.hyperChannel = channel;
506 1 : }
507 :
508 : /// Execute Query
509 0 : async executeQuery(param: pb.salesforce_hyperdb_grpc_v1.pb.QueryParam, abort?: AbortSignal, options?: HyperQueryExecutionOptions): Promise<HyperQueryResultStream> {
510 0 : return this.hyperChannel.executeQuery(param, abort, options);
511 0 : }
512 : /// Destroy the connection
513 0 : async close(): Promise<void> {
514 0 : return this.hyperChannel.close();
515 0 : }
516 1 : }
|
