chore: add script for pruning and refreshing member organizations

Author: skwowet

services/apps/script_executor_worker/src/activities.ts

@@ -41,8 +41,11 @@ import {
 } from './activities/merge-members-with-similar-identities'
 import { getUnprocessedLLMApprovedSuggestions } from './activities/process-llm-verified-merges'
 import {
+  getMemberOrganizationsToPrune,
   getOrganizationsToPrune,
+  pruneMemberOrganization,
   pruneOrganization,
+  refreshMemberAffiliations,
 } from './activities/prune-duplicate-organizations'
 import { deleteIndexedEntities } from './activities/sync/entity-index'
 import { getMembersForSync, syncMembersBatch } from './activities/sync/member'
@@ -86,4 +89,7 @@ export {
   calculateMemberAffiliations,
   getOrganizationsToPrune,
   pruneOrganization,
+  getMemberOrganizationsToPrune,
+  pruneMemberOrganization,
+  refreshMemberAffiliations,
 }

services/apps/script_executor_worker/src/activities/prune-duplicate-organizations.ts

@@ -1,4 +1,7 @@
+import { refreshMemberOrganizationAffiliations } from '@crowd/data-access-layer/src/member-organization-affiliation'
+import { deleteMemberOrganizations } from '@crowd/data-access-layer/src/members'
 import OrganizationRepository from '@crowd/data-access-layer/src/old/apps/script_executor_worker/organization.repo'
+import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
 
 import { svc } from '../main'
 
@@ -23,3 +26,38 @@ export async function getOrganizationsToPrune(
     throw error
   }
 }
+
+export async function getMemberOrganizationsToPrune(
+  batchSize: number,
+): Promise<{ id: string; memberId: string }[]> {
+  try {
+    const orgRepo = new OrganizationRepository(svc.postgres.reader.connection(), svc.log)
+    return orgRepo.getMemberOrganizationsToPrune(batchSize)
+  } catch (error) {
+    svc.log.error(error, 'Error getting member organizations to prune!')
+    throw error
+  }
+}
+
+export async function pruneMemberOrganization(
+  memberOrganizationId: string,
+  memberId: string,
+): Promise<void> {
+  try {
+    const qx = pgpQx(svc.postgres.writer.connection())
+    await deleteMemberOrganizations(qx, memberId, [memberOrganizationId], false)
+  } catch (error) {
+    svc.log.error(error, 'Error pruning member organization!')
+    throw error
+  }
+}
+
+export async function refreshMemberAffiliations(memberId: string): Promise<void> {
+  try {
+    const qx = pgpQx(svc.postgres.writer.connection())
+    await refreshMemberOrganizationAffiliations(qx, memberId)
+  } catch (error) {
+    svc.log.error(error, 'Error refreshing member affiliations!')
+    throw error
+  }
+}

services/apps/script_executor_worker/src/workflows.ts

@@ -8,6 +8,7 @@ import { fixBotMembersAffiliation } from './workflows/fix-bot-members-affiliatio
 import { fixOrgIdentitiesWithWrongUrls } from './workflows/fixOrgIdentitiesWithWrongUrls'
 import { processLLMVerifiedMerges } from './workflows/processLLMVerifiedMerges'
 import { pruneDuplicateOrganizations } from './workflows/prune-duplicate-organizations'
+import { pruneIncorrectMemberOrganizations } from './workflows/prune-incorrect-member-organizations'
 import { syncMembers } from './workflows/sync/members'
 import { syncOrganizations } from './workflows/sync/organizations'
 
@@ -24,4 +25,5 @@ export {
   pruneDuplicateOrganizations,
   fixBotMembersAffiliation,
   blockOrganizationAffiliation,
+  pruneIncorrectMemberOrganizations,
 }

services/apps/script_executor_worker/src/workflows/prune-incorrect-member-organizations.ts

@@ -0,0 +1,42 @@
+import { proxyActivities } from '@temporalio/workflow'
+
+import * as activities from '../activities'
+import { IScriptBatchTestArgs } from '../types'
+import { chunkArray } from '../utils/common'
+
+const { getMemberOrganizationsToPrune, pruneMemberOrganization, refreshMemberAffiliations } =
+  proxyActivities<typeof activities>({
+    startToCloseTimeout: '30 minutes',
+    retry: { maximumAttempts: 3, backoffCoefficient: 3 },
+  })
+
+export async function pruneIncorrectMemberOrganizations(args: IScriptBatchTestArgs): Promise<void> {
+  const BATCH_SIZE = args.batchSize ?? 100
+
+  const memberOrganizationsToPrune = await getMemberOrganizationsToPrune(BATCH_SIZE)
+
+  if (memberOrganizationsToPrune.length === 0) {
+    console.log('No more member organizations to prune!')
+    return
+  }
+
+  const memberIdsToRefresh = new Set<string>()
+  const CHUNK_SIZE = 25
+
+  for (const chunk of chunkArray(memberOrganizationsToPrune, CHUNK_SIZE)) {
+    const cleanupTasks = chunk.map(async (mo) => {
+      console.log('Pruning member organization', mo.id)
+      await pruneMemberOrganization(mo.id, mo.memberId)
+      memberIdsToRefresh.add(mo.memberId)
+    })
+
+    await Promise.all(cleanupTasks).catch((err) => {
+      console.error('Error pruning member organizations!', err)
+    })
+  }
+
+  for (const memberId of memberIdsToRefresh) {
+    console.log('Refreshing member affiliations', memberId)
+    await refreshMemberAffiliations(memberId)
+  }
+}

services/libs/data-access-layer/src/old/apps/script_executor_worker/organization.repo.ts

@@ -254,14 +254,67 @@ class OrganizationRepository {
           FROM "memberOrganizations" mo
           WHERE mo."organizationId" = o.id
             AND mo."deletedAt" IS NULL
-            AND (mo.title IS NOT NULL AND mo.title != '')
+            AND (mo.title IS NOT NULL OR mo.title != '')
             AND (mo.source IS NOT NULL AND mo.source NOT IN ('email-domain'))
       )
       LIMIT $(batchSize);
       `,
       { batchSize },
     )
   }
+
+  public async getMemberOrganizationsToPrune(
+    batchSize: number,
+  ): Promise<{ id: string; memberId: string }[]> {
+    return this.connection.query(
+      `
+      WITH email_providers AS (
+          SELECT unnest(ARRAY[
+              'gmail.com', 'gmail.co.uk', 'gmail.com.au', 'gmail.com.tr',
+              'yahoo.com', 'yahoo.co.uk', 'yahoo.com.br', 'yahoo.co.in', 'yahoo.fr', 'yahoo.es', 'yahoo.it', 'yahoo.de', 'yahoo.ca', 'yahoo.com.au', 'yahoo.in', 'yahoo.co.jp', 'yahoo.com.ar', 'yahoo.com.mx', 'yahoo.co.id', 'yahoo.com.sg', 'yahoo.co.za', 'yahoo.com.ph', 'yahoo.com.tw', 'yahoo.com.hk', 'yahoo.com.vn',
+              'hotmail.com', 'hotmail.co.uk', 'hotmail.fr', 'hotmail.ca', 'hotmail.it', 'hotmail.es', 'hotmail.de', 'hotmail.com.au', 'hotmail.com.mx',
+              'icloud.com', 'icloud.com.cn',
+              'fastmail.com', 'tutanota.com', 'tuta.io',
+              'gmx.com', 'gmx.de', 'gmx.net', 'gmx.at', 'gmx.ch', 'gmx.fr', 'gmx.co.uk',
+              'aol.com', 'aol.co.uk', 'aol.fr', 'aol.de',
+              'msn.com', 'wanadoo.fr', 'orange.fr', 'comcast.net',
+              'live.com', 'live.co.uk', 'live.fr', 'live.nl', 'live.it', 'live.com.au', 'live.ca', 'live.cn',
+              'rediffmail.com', 'sify.com', 'indiatimes.com', 'free.fr', 'web.de',
+              'yandex.ru', 'yandex.com', 'yandex.com.tr', 'ya.ru',
+              'ymail.com', 'libero.it',
+              'outlook.com', 'outlook.fr', 'outlook.co.uk', 'outlook.de', 'outlook.es', 'outlook.it', 'outlook.com.au', 'outlook.com.br', 'outlook.com.mx', 'outlook.co.jp', 'outlook.in', 'outlook.com.sg', 'outlook.co.za', 'outlook.co.in',
+              'uol.com.br', 'bol.com.br',
+              'mail.ru', 'inbox.ru', 'list.ru', 'bk.ru',
+              'mail.com', 'mail.de', 'mail.co.uk',
+              'cox.net', 'sbcglobal.net', 'sfr.fr', 'verizon.net', 'googlemail.com', 'ig.com.br', 'bigpond.com', 'bigpond.net.au', 'terra.com.br', 'neuf.fr', 'alice.it', 'rocketmail.com', 'att.net', 'laposte.net', 'bellsouth.net', 'charter.net', 'rambler.ru', 'tiscali.it', 'tiscali.co.uk', 'shaw.ca', 'sky.com', 'earthlink.net', 'optonline.net', 'freenet.de', 't-online.de', 'aliceadsl.fr', 'virgilio.it', 'home.nl', 'qq.com', 'vip.qq.com', 'telenet.be', 'pandora.be', 'me.com', 'voila.fr', 'planet.nl', 'tin.it', 'ntlworld.com', 'arcor.de', 'frontiernet.net', 'hetnet.nl', 'zonnet.nl', 'club-internet.fr', 'juno.com', 'optusnet.com.au', 'blueyonder.co.uk', 'bluewin.ch', 'skynet.be', 'sympatico.ca', 'windstream.net', 'mac.com', 'centurytel.net', 'chello.nl', 'aim.com',
+              'protonmail.com', 'protonmail.ch', 'proton.me', 'pm.me', 'duck.com',
+              'zoho.com', 'zohomail.com',
+              'users.noreply.github.com',
+              '126.com', '139.com', '163.com', '188.com', 'foxmail.com', 'tom.com', '21cn.com', 'yeah.net',
+              'naver.com', 'daum.net', 'hanmail.net',
+              'hey.com', 'inbox.com', 'lycos.com', 'excite.com', 'hushmail.com', 'mailfence.com', 'mailbox.org', 'posteo.de', 'startmail.com', 'runbox.com', 'countermail.com', 'mynet.com',
+              'wp.pl', 'onet.pl', 'interia.pl', 'o2.pl',
+              'seznam.cz', 'centrum.cz',
+              'mailinator.com', 'guerrillamail.com', '10minutemail.com', 'tempmail.com'
+          ]) AS provider
+      ),
+      orgs_with_email_provider AS (
+          SELECT DISTINCT o.id
+          FROM organizations o
+          INNER JOIN "organizationIdentities" oi ON o.id = oi."organizationId"
+          INNER JOIN email_providers ep ON LOWER(oi.value) = ep.provider
+          WHERE o."deletedAt" IS NULL
+            AND oi.type = 'primary-domain'
+      )
+      SELECT DISTINCT mo.id, mo."memberId"
+      FROM "memberOrganizations" mo
+      INNER JOIN orgs_with_email_provider oep ON mo."organizationId" = oep.id
+      WHERE mo.source = 'email-domain' AND (mo.title IS NULL OR mo.title = '')
+      LIMIT $(batchSize);
+      `,
+      { batchSize },
+    )
+  }
 }
 
 export default OrganizationRepository