Skip to content

Commit fde44d2

Browse files
mbani01mbani01
authored
feat: committees implementation [CM-1066] (#3995)
Signed-off-by: Mouad BANI <mouad-mb@outlook.com>
1 parent 02aa8f9 commit fde44d2

11 files changed

Lines changed: 307 additions & 0 deletions

File tree

‎backend/src/database/migrations/U1775064222__addCommitteesActivityTypes.sql‎

Whitespace-only changes.

‎backend/src/database/migrations/V1775064222__addCommitteesActivityTypes.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration", description, "label") VALUES
2+
('added-to-committee', 'committees', false, false, 'Member is added to a committee', 'Added to committee'),
3+
('removed-from-committee', 'committees', false, false, 'Member is removed from a committee', 'Removed from committee');

‎services/apps/snowflake_connectors/src/integrations/committees/committees/buildSourceQuery.ts‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import { IS_PROD_ENV } from '@crowd/common'
2+
3+
// Main: FIVETRAN_INGEST.SFDC_CONNECTOR_PROD_PLATFORM.COMMUNITY__C
4+
// Joins:
5+
// - ANALYTICS.SILVER_DIM.COMMITTEE (committee metadata + project slug)
6+
// - ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS (segment resolution)
7+
// - ANALYTICS.SILVER_DIM.USERS (member identity: email, lf_username, name)
8+
// - ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.ACCOUNTS (org data)
9+
10+
const CDP_MATCHED_SEGMENTS = `
11+
cdp_matched_segments AS (
12+
SELECT DISTINCT
13+
s.SOURCE_ID AS sourceId,
14+
s.slug
15+
FROM ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS s
16+
WHERE s.PARENT_SLUG IS NOT NULL
17+
AND s.GRANDPARENTS_SLUG IS NOT NULL
18+
AND s.SOURCE_ID IS NOT NULL
19+
)`
20+
21+
const ORG_ACCOUNTS = `
22+
org_accounts AS (
23+
SELECT account_id, account_name, website, domain_aliases
24+
FROM ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.ACCOUNTS
25+
WHERE website IS NOT NULL
26+
)`
27+
28+
export const buildSourceQuery = (sinceTimestamp?: string): string => {
29+
let select = `
30+
SELECT
31+
c.SFID,
32+
c._FIVETRAN_DELETED AS FIVETRAN_DELETED,
33+
c.CONTACTEMAIL__C,
34+
c.CREATEDBYID,
35+
c.COLLABORATION_NAME__C,
36+
c.ACCOUNT__C,
37+
c.ROLE__C,
38+
c.CREATEDDATE::TIMESTAMP_NTZ AS CREATEDDATE,
39+
c.LASTMODIFIEDDATE::TIMESTAMP_NTZ AS LASTMODIFIEDDATE,
40+
c._FIVETRAN_SYNCED::TIMESTAMP_NTZ AS FIVETRAN_SYNCED,
41+
cm.COMMITTEE_ID,
42+
cm.COMMITTEE_NAME,
43+
cm.PROJECT_ID,
44+
cm.PROJECT_NAME,
45+
cm.PROJECT_SLUG,
46+
su.EMAIL AS SU_EMAIL,
47+
su.LF_USERNAME,
48+
su.PRIMARY_SOURCE_USER_ID,
49+
su.FIRST_NAME AS SU_FIRST_NAME,
50+
su.LAST_NAME AS SU_LAST_NAME,
51+
su.FULL_NAME AS SU_FULL_NAME,
52+
org.account_name AS ACCOUNT_NAME,
53+
org.website AS ORG_WEBSITE,
54+
org.domain_aliases AS ORG_DOMAIN_ALIASES
55+
FROM FIVETRAN_INGEST.SFDC_CONNECTOR_PROD_PLATFORM.COMMUNITY__C c
56+
JOIN ANALYTICS.SILVER_DIM.COMMITTEE cm
57+
ON c.COLLABORATION_NAME__C = cm.COMMITTEE_ID
58+
INNER JOIN cdp_matched_segments cms
59+
ON cms.slug = cm.PROJECT_SLUG
60+
AND cms.sourceId = cm.PROJECT_ID
61+
LEFT JOIN ANALYTICS.SILVER_DIM.USERS su
62+
ON LOWER(c.CONTACTEMAIL__C) = LOWER(su.EMAIL)
63+
LEFT JOIN org_accounts org
64+
ON c.ACCOUNT__C = org.account_id
65+
WHERE c.LASTMODIFIEDDATE IS NOT NULL`
66+
67+
// Limit to a single project in non-prod to avoid exporting all project data
68+
if (!IS_PROD_ENV) {
69+
select += ` AND cm.PROJECT_SLUG = 'cncf'`
70+
}
71+
72+
const dedup = `
73+
QUALIFY ROW_NUMBER() OVER (PARTITION BY c.SFID ORDER BY org.website DESC) = 1`
74+
75+
if (!sinceTimestamp) {
76+
return `
77+
WITH ${ORG_ACCOUNTS},
78+
${CDP_MATCHED_SEGMENTS}
79+
${select}
80+
${dedup}`.trim()
81+
}
82+
83+
return `
84+
WITH ${ORG_ACCOUNTS},
85+
${CDP_MATCHED_SEGMENTS},
86+
new_cdp_segments AS (
87+
SELECT DISTINCT
88+
s.SOURCE_ID AS sourceId,
89+
s.slug
90+
FROM ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS s
91+
WHERE s.CREATED_TS >= '${sinceTimestamp}'
92+
AND s.PARENT_SLUG IS NOT NULL
93+
AND s.GRANDPARENTS_SLUG IS NOT NULL
94+
AND s.SOURCE_ID IS NOT NULL
95+
)
96+
97+
-- Updated committee memberships since last export
98+
${select}
99+
AND (
100+
c.LASTMODIFIEDDATE > '${sinceTimestamp}'
101+
OR (c._FIVETRAN_DELETED = TRUE AND c._FIVETRAN_SYNCED > '${sinceTimestamp}')
102+
)
103+
${dedup}
104+
105+
UNION
106+
107+
-- All committee memberships in newly created segments
108+
${select}
109+
AND EXISTS (
110+
SELECT 1 FROM new_cdp_segments ncs
111+
WHERE ncs.slug = cms.slug AND ncs.sourceId = cms.sourceId
112+
)
113+
${dedup}`.trim()
114+
}

‎services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import { COMMITTEES_GRID, CommitteesActivityType } from '@crowd/integrations'
2+
import { getServiceChildLogger } from '@crowd/logging'
3+
import {
4+
IActivityData,
5+
IOrganizationIdentity,
6+
OrganizationIdentityType,
7+
OrganizationSource,
8+
PlatformType,
9+
} from '@crowd/types'
10+
11+
import { TransformedActivity, TransformerBase } from '../../../core/transformerBase'
12+
13+
const log = getServiceChildLogger('committeesCommitteesTransformer')
14+
15+
export class CommitteesCommitteesTransformer extends TransformerBase {
16+
readonly platform = PlatformType.COMMITTEES
17+
18+
transformRow(row: Record<string, unknown>): TransformedActivity | null {
19+
const email = (row.CONTACTEMAIL__C as string | null)?.trim() || null
20+
if (!email) {
21+
log.warn(
22+
{ sfid: row.SFID, committeeId: row.COMMITTEE_ID, rawEmail: row.CONTACTEMAIL__C },
23+
'Skipping row: missing email',
24+
)
25+
return null
26+
}
27+
28+
const committeeId = (row.COMMITTEE_ID as string).trim()
29+
const fivetranDeleted = row.FIVETRAN_DELETED as boolean
30+
const lfUsername = (row.LF_USERNAME as string | null)?.trim() || null
31+
const suFullName = (row.SU_FULL_NAME as string | null)?.trim() || null
32+
const suFirstName = (row.SU_FIRST_NAME as string | null)?.trim() || null
33+
const suLastName = (row.SU_LAST_NAME as string | null)?.trim() || null
34+
35+
const displayName =
36+
suFullName ||
37+
(suFirstName && suLastName ? `${suFirstName} ${suLastName}` : suFirstName || suLastName) ||
38+
email.split('@')[0]
39+
40+
const type = fivetranDeleted
41+
? CommitteesActivityType.REMOVED_FROM_COMMITTEE
42+
: CommitteesActivityType.ADDED_TO_COMMITTEE
43+
44+
const sourceId = (row.PRIMARY_SOURCE_USER_ID as string | null)?.trim() || undefined
45+
const identities = this.buildMemberIdentities({
46+
email,
47+
platformUsername: null,
48+
sourceId,
49+
lfUsername,
50+
})
51+
52+
const activityTimestamp =
53+
type === CommitteesActivityType.ADDED_TO_COMMITTEE
54+
? (row.CREATEDDATE as string | null) || null
55+
: (row.FIVETRAN_SYNCED as string | null) || null
56+
57+
const committeeName = (row.COMMITTEE_NAME as string | null) || null
58+
59+
const activity: IActivityData = {
60+
type,
61+
platform: PlatformType.COMMITTEES,
62+
timestamp: activityTimestamp,
63+
score: COMMITTEES_GRID[type].score,
64+
sourceId: `${committeeId}-${row.SFID}`,
65+
sourceParentId: null,
66+
channel: committeeName,
67+
member: {
68+
displayName,
69+
identities,
70+
organizations: this.buildOrganizations(row),
71+
},
72+
attributes: {
73+
committeeId: (row.COLLABORATION_NAME__C as string | null) || null,
74+
committeeName,
75+
role: (row.ROLE__C as string | null) || null,
76+
projectId: (row.PROJECT_ID as string | null) || null,
77+
projectName: (row.PROJECT_NAME as string | null) || null,
78+
organizationId: (row.ACCOUNT__C as string | null) || null,
79+
organizationName: (row.ACCOUNT_NAME as string | null) || null,
80+
},
81+
}
82+
83+
const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null
84+
const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null
85+
86+
if (!segmentSlug || !segmentSourceId) {
87+
log.warn(
88+
{ sfid: row.SFID, committeeId, segmentSlug, segmentSourceId },
89+
'Skipping row: missing segment slug or sourceId',
90+
)
91+
return null
92+
}
93+
94+
return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } }
95+
}
96+
97+
private buildOrganizations(
98+
row: Record<string, unknown>,
99+
): IActivityData['member']['organizations'] {
100+
const website = (row.ORG_WEBSITE as string | null)?.trim() || null
101+
const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null
102+
103+
if (!website && !domainAliases) {
104+
return undefined
105+
}
106+
107+
const displayName = (row.ACCOUNT_NAME as string | null)?.trim() || website
108+
109+
if (this.isIndividualNoAccount(displayName)) {
110+
return [
111+
{
112+
displayName,
113+
source: OrganizationSource.COMMITTEES,
114+
identities: website
115+
? [
116+
{
117+
platform: PlatformType.COMMITTEES,
118+
value: website,
119+
type: OrganizationIdentityType.PRIMARY_DOMAIN,
120+
verified: true,
121+
},
122+
]
123+
: [],
124+
},
125+
]
126+
}
127+
128+
const identities: IOrganizationIdentity[] = []
129+
130+
if (website) {
131+
identities.push({
132+
platform: PlatformType.COMMITTEES,
133+
value: website,
134+
type: OrganizationIdentityType.PRIMARY_DOMAIN,
135+
verified: true,
136+
})
137+
}
138+
139+
if (domainAliases) {
140+
for (const alias of domainAliases.split(',')) {
141+
const trimmed = alias.trim()
142+
if (trimmed) {
143+
identities.push({
144+
platform: PlatformType.COMMITTEES,
145+
value: trimmed,
146+
type: OrganizationIdentityType.ALTERNATIVE_DOMAIN,
147+
verified: true,
148+
})
149+
}
150+
}
151+
}
152+
153+
return [
154+
{
155+
displayName,
156+
source: OrganizationSource.COMMITTEES,
157+
identities,
158+
},
159+
]
160+
}
161+
}

‎services/apps/snowflake_connectors/src/integrations/index.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
*/
77
import { PlatformType } from '@crowd/types'
88

9+
import { buildSourceQuery as committeesCommitteesBuildQuery } from './committees/committees/buildSourceQuery'
10+
import { CommitteesCommitteesTransformer } from './committees/committees/transformer'
911
import { buildSourceQuery as cventBuildSourceQuery } from './cvent/event-registrations/buildSourceQuery'
1012
import { CventTransformer } from './cvent/event-registrations/transformer'
1113
import { buildSourceQuery as tncCertificatesBuildQuery } from './tnc/certificates/buildSourceQuery'
@@ -20,6 +22,15 @@ export type { BuildSourceQuery, DataSource, PlatformDefinition } from './types'
2022
export { DataSourceName } from './types'
2123

2224
const supported: Partial<Record<PlatformType, PlatformDefinition>> = {
25+
[PlatformType.COMMITTEES]: {
26+
sources: [
27+
{
28+
name: DataSourceName.COMMITTEES_COMMITTEES,
29+
buildSourceQuery: committeesCommitteesBuildQuery,
30+
transformer: new CommitteesCommitteesTransformer(),
31+
},
32+
],
33+
},
2334
[PlatformType.CVENT]: {
2435
sources: [
2536
{

‎services/apps/snowflake_connectors/src/integrations/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ export enum DataSourceName {
88
TNC_ENROLLMENTS = 'enrollments',
99
TNC_CERTIFICATES = 'certificates',
1010
TNC_COURSES = 'courses',
11+
COMMITTEES_COMMITTEES = 'committees',
1112
}
1213

1314
export interface DataSource {

‎services/libs/data-access-layer/src/organizations/attributesConfig.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ export const ORG_DB_ATTRIBUTE_SOURCE_PRIORITY = [
233233
OrganizationAttributeSource.ENRICHMENT_PEOPLEDATALABS,
234234
OrganizationAttributeSource.CVENT,
235235
OrganizationAttributeSource.TNC,
236+
OrganizationAttributeSource.COMMITTEES,
236237
// legacy - keeping this for backward compatibility
237238
OrganizationAttributeSource.ENRICHMENT,
238239
OrganizationAttributeSource.GITHUB,

‎services/libs/integrations/src/integrations/committees/types.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { IActivityScoringGrid } from '@crowd/types'
2+
3+
export enum CommitteesActivityType {
4+
ADDED_TO_COMMITTEE = 'added-to-committee',
5+
REMOVED_FROM_COMMITTEE = 'removed-from-committee',
6+
}
7+
8+
export const COMMITTEES_GRID: Record<CommitteesActivityType, IActivityScoringGrid> = {
9+
[CommitteesActivityType.ADDED_TO_COMMITTEE]: { score: 1 },
10+
[CommitteesActivityType.REMOVED_FROM_COMMITTEE]: { score: 1 },
11+
}

‎services/libs/integrations/src/integrations/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,6 @@ export * from './cvent/types'
5252

5353
export * from './tnc/types'
5454

55+
export * from './committees/types'
56+
5557
export * from './activityDisplayService'

‎services/libs/types/src/enums/organizations.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export enum OrganizationSource {
1414
UI = 'ui',
1515
CVENT = 'cvent',
1616
TNC = 'tnc',
17+
COMMITTEES = 'committees',
1718
}
1819

1920
export enum OrganizationMergeSuggestionType {
@@ -42,6 +43,7 @@ export enum OrganizationAttributeSource {
4243
ENRICHMENT_PEOPLEDATALABS = 'enrichment-peopledatalabs',
4344
CVENT = 'cvent',
4445
TNC = 'tnc',
46+
COMMITTEES = 'committees',
4547
// legacy - keeping this for backward compatibility
4648
ENRICHMENT = 'enrichment',
4749
GITHUB = 'github',

0 commit comments

Comments
 (0)