chore: backport #2594 and #2596 to v1

packages/audio-filters-web/src/FastEnhancerNoiseCancellation.ts

@@ -163,7 +163,10 @@ export class FastEnhancerNoiseCancellation extends BaseNoiseCancellation {
   /** Aborts the pending `init()` with the given cause; used by `dispose()`. */
   private abortInit?: (error: Error) => void;
 
-  private readonly model: FastEnhancerModel;
+  /**
+   * The model size this instance runs.
+   */
+  readonly model: FastEnhancerModel;
   private readonly basePath: string;
   private readonly hpf: boolean;
   private readonly agc: boolean;

packages/audio-filters-web/src/INoiseCancellation.ts

@@ -30,6 +30,11 @@ export interface INoiseCancellation {
    * The noise cancellation engine this implementation runs, e.g. `krisp`.
    */
   engine?: string;
+  /**
+   * The model variant the engine runs, e.g. `krisp-nc-o-med-v7` or `tiny`.
+   * Undefined when unknown.
+   */
+  readonly model?: string;
   /**
    * Applies options from the call settings.
    * Call it before {@link isSupported} and {@link init}.

packages/audio-filters-web/src/KrispNoiseCancellation.ts

@@ -80,6 +80,14 @@ export type KrispNoiseCancellationOptions = {
   krispSDKParams?: ISDKPartialOptions['params'];
 };
 
+// https://sdk-docs.krisp.ai/docs/krisp-audio-sdk-model-selection-guide
+const DEFAULT_MODEL_FILE = 'krisp-nc-o-med-v7.kef';
+
+const toModelName = (url: string | undefined) => {
+  const file = url?.split(/[?#]/)[0].split('/').pop();
+  return file?.replace(/\.[^.]*$/, '') || undefined;
+};
+
 /**
  * A wrapper around the Krisp.AI SDK.
  */
@@ -123,6 +131,21 @@ export class KrispNoiseCancellation extends BaseNoiseCancellation {
     this.krispSDKParams = krispSDKParams;
   }
 
+  /**
+   * The model the engine runs: its file name without the extension,
+   * e.g. `krisp-nc-o-med-v7`.
+   */
+  get model(): string | undefined {
+    const models = this.krispSDKParams?.models;
+    if (models) {
+      const modelNC = models.modelNC;
+      return toModelName(typeof modelNC === 'string' ? modelNC : modelNC?.url);
+    }
+    return toModelName(
+      this.modelPath ?? `${this.basePath}/${DEFAULT_MODEL_FILE}`,
+    );
+  }
+
   /**
    * Applies options from the call settings. Options passed to the
    * constructor take precedence. Call it before {@link isSupported} and
@@ -186,8 +209,7 @@ export class KrispNoiseCancellation extends BaseNoiseCancellation {
         logProcessStats: false,
         useSharedArrayBuffer: false,
         models: {
-          // https://sdk-docs.krisp.ai/docs/krisp-audio-sdk-model-selection-guide
-          modelNC: this.modelPath ?? `${this.basePath}/krisp-nc-o-med-v7.kef`,
+          modelNC: this.modelPath ?? `${this.basePath}/${DEFAULT_MODEL_FILE}`,
         },
         ...this.krispSDKParams,
       },

packages/audio-filters-web/src/NoiseCancellation.ts

@@ -85,6 +85,14 @@ export class NoiseCancellation implements INoiseCancellation {
     return this.delegate?.engine ?? this.resolveEngine();
   }
 
+  /**
+   * The running engine's model variant, e.g. `krisp-nc-o-med-v7` or `tiny`.
+   * Undefined when no engine was created yet.
+   */
+  get model(): string | undefined {
+    return this.delegate?.model;
+  }
+
   /**
    * Applies options from the call settings: the engine to run, and the
    * engine options. Options passed to the constructor take precedence.

packages/audio-filters-web/src/__tests__/NoiseCancellation.test.ts

@@ -10,6 +10,7 @@ class EngineStub {
   static instances: EngineStub[] = [];
   readonly engine: string;
   readonly options: unknown;
+  model?: string;
   readonly listeners: Record<string, Listener[]> = {};
   configuration?: NoiseCancellationConfiguration;
 
@@ -86,6 +87,18 @@ describe('NoiseCancellation', () => {
     expect(lastEngine().init).toHaveBeenCalled();
   });
 
+  it('reports the running engine model', async () => {
+    const nc = new NoiseCancellation();
+    expect(nc.model).toBeUndefined();
+
+    await nc.init();
+    lastEngine().model = 'krisp-nc-o-med-v7';
+    expect(nc.model).toBe('krisp-nc-o-med-v7');
+
+    await nc.dispose();
+    expect(nc.model).toBeUndefined();
+  });
+
   it('runs the configured engine', async () => {
     const nc = new NoiseCancellation();
     nc.configure({ engine: 'stream' });

packages/audio-filters-web/src/__tests__/engineModel.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest';
+import { KrispNoiseCancellation } from '../KrispNoiseCancellation';
+import { FastEnhancerNoiseCancellation } from '../FastEnhancerNoiseCancellation';
+
+describe('KrispNoiseCancellation model', () => {
+  it('reports the default model', () => {
+    expect(new KrispNoiseCancellation().model).toBe('krisp-nc-o-med-v7');
+  });
+
+  it('reports the model from the call settings model path', () => {
+    const nc = new KrispNoiseCancellation();
+    nc.configure({
+      modelPath: 'https://cdn.example.com/models/krisp-nc-o-hi-v8.kef?sig=1',
+    });
+    expect(nc.model).toBe('krisp-nc-o-hi-v8');
+  });
+
+  it('reports the model from the Krisp SDK params', () => {
+    const nc = new KrispNoiseCancellation({
+      krispSDKParams: {
+        models: { modelNC: { url: '/models/krisp-nc-s-v1.kef' } },
+      },
+    });
+    expect(nc.model).toBe('krisp-nc-s-v1');
+  });
+});
+
+describe('FastEnhancerNoiseCancellation model', () => {
+  it('reports the model size', () => {
+    expect(new FastEnhancerNoiseCancellation().model).toBe('tiny');
+    expect(new FastEnhancerNoiseCancellation({ model: 'base' }).model).toBe(
+      'base',
+    );
+  });
+});

packages/client/src/Call.ts

@@ -134,6 +134,10 @@ import {
 } from './types';
 import { BehaviorSubject, Subject, takeWhile } from 'rxjs';
 import { ReconnectDetails } from './gen/video/sfu/event/events';
+import type {
+  StartNoiseCancellationRequest,
+  StopNoiseCancellationRequest,
+} from './gen/video/sfu/signal_rpc/signal';
 import {
   ClientCapability,
   ClientDetails,
@@ -2920,22 +2924,32 @@ export class Call {
    * Notifies the SFU that a noise cancellation process has started.
    *
    * @internal
+   * @param noiseCancellation the engine and model that run.
    */
-  notifyNoiseCancellationStarting = async () => {
-    return this.sfuClient?.startNoiseCancellation().catch((err) => {
-      this.logger.warn('Failed to notify start of noise cancellation', err);
-    });
+  notifyNoiseCancellationStarting = async (
+    noiseCancellation: Omit<StartNoiseCancellationRequest, 'sessionId'>,
+  ) => {
+    return this.sfuClient
+      ?.startNoiseCancellation(noiseCancellation)
+      .catch((err) => {
+        this.logger.warn('Failed to notify start of noise cancellation', err);
+      });
   };
 
   /**
    * Notifies the SFU that a noise cancellation process has stopped.
    *
    * @internal
+   * @param noiseCancellation the engine and model that ran.
    */
-  notifyNoiseCancellationStopped = async () => {
-    return this.sfuClient?.stopNoiseCancellation().catch((err) => {
-      this.logger.warn('Failed to notify stop of noise cancellation', err);
-    });
+  notifyNoiseCancellationStopped = async (
+    noiseCancellation: Omit<StopNoiseCancellationRequest, 'sessionId'>,
+  ) => {
+    return this.sfuClient
+      ?.stopNoiseCancellation(noiseCancellation)
+      .catch((err) => {
+        this.logger.warn('Failed to notify stop of noise cancellation', err);
+      });
   };
 
   /**

packages/client/src/StreamSfuClient.ts

@@ -23,6 +23,8 @@ import {
   SendAnswerRequest,
   SendStatsRequest,
   SetPublisherRequest,
+  StartNoiseCancellationRequest,
+  StopNoiseCancellationRequest,
   TrackMuteState,
   TrackSubscriptionDetails,
 } from './gen/video/sfu/signal_rpc/signal';
@@ -546,24 +548,38 @@ export class StreamSfuClient {
     return this.rpc.sendStats({ ...stats, sessionId: this.sessionId });
   };
 
-  startNoiseCancellation = async () => {
+  /**
+   * Notifies the SFU that noise cancellation started.
+   *
+   * @internal use `call.microphone.enableNoiseCancellation()` instead.
+   */
+  startNoiseCancellation = async (
+    noiseCancellation: Omit<StartNoiseCancellationRequest, 'sessionId'>,
+  ) => {
     await this.joinTask;
     return retryable(
       (invocationMeta) =>
         this.rpc.startNoiseCancellation(
-          { sessionId: this.sessionId },
+          { ...noiseCancellation, sessionId: this.sessionId },
           { invocationMeta },
         ),
       this.abortController.signal,
     );
   };
 
-  stopNoiseCancellation = async () => {
+  /**
+   * Notifies the SFU that noise cancellation stopped.
+   *
+   * @internal use `call.microphone.disableNoiseCancellation()` instead.
+   */
+  stopNoiseCancellation = async (
+    noiseCancellation: Omit<StopNoiseCancellationRequest, 'sessionId'>,
+  ) => {
     await this.joinTask;
     return retryable(
       (invocationMeta) =>
         this.rpc.stopNoiseCancellation(
-          { sessionId: this.sessionId },
+          { ...noiseCancellation, sessionId: this.sessionId },
           { invocationMeta },
         ),
       this.abortController.signal,

packages/client/src/__tests__/StreamSfuClient.test.ts

@@ -6,6 +6,7 @@ import { getTimers } from '../timers';
 import { getSdkInfo, setSdkInfo } from '../helpers/client-details';
 import {
   ErrorCode,
+  NoiseCancellationEngine,
   SdkType,
   WebsocketReconnectStrategy,
 } from '../gen/video/sfu/models/models';
@@ -477,7 +478,10 @@ describe('StreamSfuClient RPC headers', () => {
         }
       ).joinResponseTask.resolve({});
 
-      await sfuClient.startNoiseCancellation();
+      await sfuClient.startNoiseCancellation({
+        engine: NoiseCancellationEngine.STREAM,
+        model: 'tiny',
+      });
 
       const [, init] = fetchMock.mock.calls[0];
       expect(new Headers(init.headers).get('x-stream-client')).toBe(

packages/client/src/devices/MicrophoneManager.ts

@@ -13,7 +13,12 @@ import { type BrowserPermissionState } from './BrowserPermission';
 import { MicrophoneManagerState } from './MicrophoneManagerState';
 import { TrackDisableMode } from './DeviceManagerState';
 import { getAudioDevices, getAudioStream, resolveDeviceId } from './devices';
-import { AudioBitrateProfile, TrackType } from '../gen/video/sfu/models/models';
+import {
+  AudioBitrateProfile,
+  NoiseCancellationEngine,
+  TrackType,
+} from '../gen/video/sfu/models/models';
+import type { StartNoiseCancellationRequest } from '../gen/video/sfu/signal_rpc/signal';
 import { createSoundDetector } from '../helpers/sound-detector';
 import { createNoAudioDetector } from '../helpers/no-audio-detector';
 import { isReactNative } from '../helpers/platforms';
@@ -41,6 +46,10 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
   private soundDetectorDeviceId?: string;
   private noAudioDetectorCleanup?: () => Promise<void>;
   private noiseCancellation: INoiseCancellation | undefined;
+  private startedNoiseCancellation?: Omit<
+    StartNoiseCancellationRequest,
+    'sessionId'
+  >;
   private noiseCancellationUnsubscribers: Array<() => void> = [];
   private noiseCancellationRegistration?: Promise<void>;
   private unregisterNoiseCancellation?: () => Promise<void>;
@@ -141,7 +150,7 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
             })
             .catch((err) => {
               this.logger.warn(`Failed to enable noise cancellation`, err);
-              return this.call.notifyNoiseCancellationStopped();
+              return this.notifyNoiseCancellationStopped();
             });
         } else if (callingState === CallingState.LEFT) {
           this.noiseCancellationRegistration
@@ -270,6 +279,7 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
     }
     try {
       this.noiseCancellation = noiseCancellation;
+      this.startedNoiseCancellation = undefined;
 
       // listen for change events and notify the SFU
       this.noiseCancellationUnsubscribers.push(
@@ -279,11 +289,11 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
             enabled ? 'enabled' : 'disabled',
           );
           if (enabled) {
-            this.call.notifyNoiseCancellationStarting().catch((err) => {
+            this.notifyNoiseCancellationStarting().catch((err) => {
               this.logger.warn(`notifyNoiseCancellationStart failed`, err);
             });
           } else {
-            this.call.notifyNoiseCancellationStopped().catch((err) => {
+            this.notifyNoiseCancellationStopped().catch((err) => {
               this.logger.warn(`notifyNoiseCancellationStop failed`, err);
             });
           }
@@ -358,9 +368,33 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
     this.state.setNoiseCancellationStatus('disabled');
 
     this.call.tracer.trace('noiseCancellation.disabled', true);
-    await this.call.notifyNoiseCancellationStopped();
+    await this.notifyNoiseCancellationStopped();
   }
 
+  private describeNoiseCancellation = () => ({
+    engine:
+      this.noiseCancellation?.engine === 'stream'
+        ? NoiseCancellationEngine.STREAM
+        : NoiseCancellationEngine.KRISP_UNSPECIFIED,
+    model: this.noiseCancellation?.model ?? '',
+  });
+
+  private notifyNoiseCancellationStarting = () => {
+    const noiseCancellation = this.describeNoiseCancellation();
+    this.startedNoiseCancellation = noiseCancellation;
+    return this.call.notifyNoiseCancellationStarting(noiseCancellation);
+  };
+
+  /**
+   * Reports the engine and model that were last started, so every stop
+   * matches the start even after the engine was disposed or reconfigured.
+   */
+  private notifyNoiseCancellationStopped = () => {
+    const noiseCancellation =
+      this.startedNoiseCancellation ?? this.describeNoiseCancellation();
+    return this.call.notifyNoiseCancellationStopped(noiseCancellation);
+  };
+
   /**
    * Enables speaking while muted notification.
    */