fix(voice): bundle deepfilter assets and enforce DSP toggle compatibility
This commit is contained in:
parent
1512c2a895
commit
7a85935eee
|
|
@ -45,11 +45,20 @@ const sanitizeDeviceSettings = (
|
||||||
if (merged.noiseSuppressionRnnoise && merged.noiseSuppressionEnhanced) {
|
if (merged.noiseSuppressionRnnoise && merged.noiseSuppressionEnhanced) {
|
||||||
merged.noiseSuppressionEnhanced = false;
|
merged.noiseSuppressionEnhanced = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (merged.noiseSuppressionDeepFilterNet) {
|
if (merged.noiseSuppressionDeepFilterNet) {
|
||||||
merged.noiseSuppressionEnhanced = false;
|
merged.noiseSuppressionEnhanced = false;
|
||||||
merged.noiseSuppressionRnnoise = false;
|
merged.noiseSuppressionRnnoise = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const hasWasmSuppression =
|
||||||
|
merged.noiseSuppressionDeepFilterNet || merged.noiseSuppressionRnnoise;
|
||||||
|
if (hasWasmSuppression) {
|
||||||
|
merged.echoCancellation = false;
|
||||||
|
merged.noiseSuppression = false;
|
||||||
|
merged.autoGainControl = false;
|
||||||
|
}
|
||||||
|
|
||||||
if (!Number.isFinite(merged.voiceSensitivity)) {
|
if (!Number.isFinite(merged.voiceSensitivity)) {
|
||||||
merged.voiceSensitivity = DEFAULT_DEVICE_SETTINGS.voiceSensitivity;
|
merged.voiceSensitivity = DEFAULT_DEVICE_SETTINGS.voiceSensitivity;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,7 @@ const getConfiguredMicChain = (values: {
|
||||||
const chain: string[] = [];
|
const chain: string[] = [];
|
||||||
if (values.noiseSuppressionDeepFilterNet) chain.push('DeepFilterNet');
|
if (values.noiseSuppressionDeepFilterNet) chain.push('DeepFilterNet');
|
||||||
if (values.noiseSuppressionRnnoise) chain.push('RNNoise');
|
if (values.noiseSuppressionRnnoise) chain.push('RNNoise');
|
||||||
if (values.keyboardSuppression !== false) chain.push('Keyboard Gate');
|
if (values.keyboardSuppression) chain.push('Keyboard Gate');
|
||||||
if (values.noiseSuppressionEnhanced) chain.push('Enhanced (browser)');
|
if (values.noiseSuppressionEnhanced) chain.push('Enhanced (browser)');
|
||||||
if (
|
if (
|
||||||
!chain.length &&
|
!chain.length &&
|
||||||
|
|
@ -131,6 +131,8 @@ const Devices = memo(() => {
|
||||||
if (checked) {
|
if (checked) {
|
||||||
onChange('noiseSuppressionEnhanced', false);
|
onChange('noiseSuppressionEnhanced', false);
|
||||||
onChange('noiseSuppressionDeepFilterNet', false);
|
onChange('noiseSuppressionDeepFilterNet', false);
|
||||||
|
onChange('echoCancellation', false);
|
||||||
|
onChange('autoGainControl', false);
|
||||||
onChange('noiseSuppression', false);
|
onChange('noiseSuppression', false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -143,12 +145,50 @@ const Devices = memo(() => {
|
||||||
if (checked) {
|
if (checked) {
|
||||||
onChange('noiseSuppressionEnhanced', false);
|
onChange('noiseSuppressionEnhanced', false);
|
||||||
onChange('noiseSuppressionRnnoise', false);
|
onChange('noiseSuppressionRnnoise', false);
|
||||||
|
onChange('echoCancellation', false);
|
||||||
|
onChange('autoGainControl', false);
|
||||||
onChange('noiseSuppression', false);
|
onChange('noiseSuppression', false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[onChange]
|
[onChange]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const handleEchoCancellationToggle = useCallback(
|
||||||
|
(checked: boolean) => {
|
||||||
|
onChange('echoCancellation', checked);
|
||||||
|
if (checked) {
|
||||||
|
onChange('noiseSuppressionRnnoise', false);
|
||||||
|
onChange('noiseSuppressionDeepFilterNet', false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[onChange]
|
||||||
|
);
|
||||||
|
|
||||||
|
const handleNoiseSuppressionToggle = useCallback(
|
||||||
|
(checked: boolean) => {
|
||||||
|
onChange('noiseSuppression', checked);
|
||||||
|
if (checked) {
|
||||||
|
onChange('noiseSuppressionRnnoise', false);
|
||||||
|
onChange('noiseSuppressionDeepFilterNet', false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[onChange]
|
||||||
|
);
|
||||||
|
|
||||||
|
const handleAutoGainControlToggle = useCallback(
|
||||||
|
(checked: boolean) => {
|
||||||
|
onChange('autoGainControl', checked);
|
||||||
|
if (checked) {
|
||||||
|
onChange('noiseSuppressionRnnoise', false);
|
||||||
|
onChange('noiseSuppressionDeepFilterNet', false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[onChange]
|
||||||
|
);
|
||||||
|
|
||||||
|
const hasWasmSuppression =
|
||||||
|
!!values.noiseSuppressionRnnoise || !!values.noiseSuppressionDeepFilterNet;
|
||||||
|
|
||||||
const saveDeviceSettings = useCallback(() => {
|
const saveDeviceSettings = useCallback(() => {
|
||||||
saveDevices(values);
|
saveDevices(values);
|
||||||
toast.success('Device settings saved');
|
toast.success('Device settings saved');
|
||||||
|
|
@ -186,21 +226,23 @@ const Devices = memo(() => {
|
||||||
<Group label="Echo cancellation">
|
<Group label="Echo cancellation">
|
||||||
<Switch
|
<Switch
|
||||||
checked={!!values.echoCancellation}
|
checked={!!values.echoCancellation}
|
||||||
onCheckedChange={(checked) =>
|
onCheckedChange={handleEchoCancellationToggle}
|
||||||
onChange('echoCancellation', checked)
|
|
||||||
}
|
|
||||||
/>
|
/>
|
||||||
</Group>
|
</Group>
|
||||||
|
|
||||||
<Group label="Noise suppression">
|
<Group label="Noise suppression">
|
||||||
<Switch
|
<Switch
|
||||||
checked={!!values.noiseSuppression}
|
checked={!!values.noiseSuppression}
|
||||||
onCheckedChange={(checked) =>
|
onCheckedChange={handleNoiseSuppressionToggle}
|
||||||
onChange('noiseSuppression', checked)
|
|
||||||
}
|
|
||||||
/>
|
/>
|
||||||
</Group>
|
</Group>
|
||||||
|
|
||||||
|
{hasWasmSuppression ? (
|
||||||
|
<div className="text-xs text-muted-foreground max-w-xs">
|
||||||
|
Browser echo/noise/AGC are disabled while RNNoise or DeepFilterNet is active.
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
<Group label="Enhanced noise suppression">
|
<Group label="Enhanced noise suppression">
|
||||||
<Switch
|
<Switch
|
||||||
checked={!!values.noiseSuppressionEnhanced}
|
checked={!!values.noiseSuppressionEnhanced}
|
||||||
|
|
@ -227,7 +269,7 @@ const Devices = memo(() => {
|
||||||
description="Speech-safe gate to reduce key clicks while keeping voice pickup."
|
description="Speech-safe gate to reduce key clicks while keeping voice pickup."
|
||||||
>
|
>
|
||||||
<Switch
|
<Switch
|
||||||
checked={values.keyboardSuppression !== false}
|
checked={!!values.keyboardSuppression}
|
||||||
onCheckedChange={(checked) =>
|
onCheckedChange={(checked) =>
|
||||||
onChange('keyboardSuppression', checked)
|
onChange('keyboardSuppression', checked)
|
||||||
}
|
}
|
||||||
|
|
@ -279,9 +321,7 @@ const Devices = memo(() => {
|
||||||
<Group label="Automatic gain control">
|
<Group label="Automatic gain control">
|
||||||
<Switch
|
<Switch
|
||||||
checked={!!values.autoGainControl}
|
checked={!!values.autoGainControl}
|
||||||
onCheckedChange={(checked) =>
|
onCheckedChange={handleAutoGainControlToggle}
|
||||||
onChange('autoGainControl', checked)
|
|
||||||
}
|
|
||||||
/>
|
/>
|
||||||
</Group>
|
</Group>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -100,7 +100,8 @@ const getConfiguredMicChain = (devices: TDeviceSettings): string => {
|
||||||
if (devices.keyboardSuppression) chain.push('Keyboard Gate');
|
if (devices.keyboardSuppression) chain.push('Keyboard Gate');
|
||||||
if (devices.noiseSuppressionEnhanced) chain.push('Enhanced (browser)');
|
if (devices.noiseSuppressionEnhanced) chain.push('Enhanced (browser)');
|
||||||
if (
|
if (
|
||||||
!chain.length &&
|
!devices.noiseSuppressionDeepFilterNet &&
|
||||||
|
!devices.noiseSuppressionRnnoise &&
|
||||||
(devices.echoCancellation || devices.noiseSuppression || devices.autoGainControl)
|
(devices.echoCancellation || devices.noiseSuppression || devices.autoGainControl)
|
||||||
) {
|
) {
|
||||||
chain.push('Browser DSP');
|
chain.push('Browser DSP');
|
||||||
|
|
@ -114,14 +115,19 @@ const buildMicConstraints = (devices: TDeviceSettings): MediaTrackConstraints =>
|
||||||
const deepFilterNet = !!devices.noiseSuppressionDeepFilterNet;
|
const deepFilterNet = !!devices.noiseSuppressionDeepFilterNet;
|
||||||
const keyboardSuppression = !!devices.keyboardSuppression;
|
const keyboardSuppression = !!devices.keyboardSuppression;
|
||||||
const processed = enhanced || rnnoise || deepFilterNet || keyboardSuppression;
|
const processed = enhanced || rnnoise || deepFilterNet || keyboardSuppression;
|
||||||
|
const hasWasmSuppression = rnnoise || deepFilterNet;
|
||||||
|
|
||||||
const constraints: MediaTrackConstraints = {
|
const constraints: MediaTrackConstraints = {
|
||||||
deviceId: devices.microphoneId
|
deviceId: devices.microphoneId
|
||||||
? { exact: devices.microphoneId }
|
? { exact: devices.microphoneId }
|
||||||
: undefined,
|
: undefined,
|
||||||
autoGainControl: devices.autoGainControl,
|
autoGainControl: hasWasmSuppression ? false : devices.autoGainControl,
|
||||||
echoCancellation: devices.echoCancellation,
|
echoCancellation: hasWasmSuppression ? false : devices.echoCancellation,
|
||||||
noiseSuppression: (rnnoise || deepFilterNet) ? false : (enhanced ? true : devices.noiseSuppression),
|
noiseSuppression: hasWasmSuppression
|
||||||
|
? false
|
||||||
|
: enhanced
|
||||||
|
? true
|
||||||
|
: devices.noiseSuppression,
|
||||||
sampleRate: 48000,
|
sampleRate: 48000,
|
||||||
channelCount: processed ? 1 : 2
|
channelCount: processed ? 1 : 2
|
||||||
};
|
};
|
||||||
|
|
@ -560,8 +566,8 @@ const VoiceProvider = memo(({ children }: TVoiceProviderProps) => {
|
||||||
track.onended = () => {
|
track.onended = () => {
|
||||||
logVoice('Audio track ended, cleaning up microphone');
|
logVoice('Audio track ended, cleaning up microphone');
|
||||||
|
|
||||||
localAudioStream?.getAudioTracks().forEach((track) => {
|
stream.getAudioTracks().forEach((streamTrack) => {
|
||||||
track.stop();
|
streamTrack.stop();
|
||||||
});
|
});
|
||||||
localAudioProducer.current?.close();
|
localAudioProducer.current?.close();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,12 @@ import fs from 'fs';
|
||||||
import fsp from 'fs/promises';
|
import fsp from 'fs/promises';
|
||||||
import http from 'http';
|
import http from 'http';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
import { DATA_PATH } from '../helpers/paths';
|
||||||
|
|
||||||
const ROUTE_PREFIX = '/deepfilter-assets/';
|
const ROUTE_PREFIX = '/deepfilter-assets/';
|
||||||
const ALLOWED_CDN_PATH_PREFIX =
|
const ALLOWED_CDN_PATH_PREFIX =
|
||||||
'AI/models/datas/noise_suppression/deepfilternet3/';
|
'AI/models/datas/noise_suppression/deepfilternet3/';
|
||||||
const CDN_BASE_URL = 'https://cdn.mezon.ai/';
|
const CACHE_DIR = path.join(DATA_PATH, 'deepfilter-assets');
|
||||||
const CACHE_DIR = '/root/.config/pulse/deepfilter-assets';
|
|
||||||
|
|
||||||
const guessContentType = (filePath: string): string => {
|
const guessContentType = (filePath: string): string => {
|
||||||
if (filePath.endsWith('.wasm')) return 'application/wasm';
|
if (filePath.endsWith('.wasm')) return 'application/wasm';
|
||||||
|
|
@ -37,7 +37,15 @@ const deepfilterAssetsRouteHandler = async (
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const relativePath = decodeURIComponent(routePath.slice(ROUTE_PREFIX.length));
|
let relativePath = '';
|
||||||
|
try {
|
||||||
|
relativePath = decodeURIComponent(routePath.slice(ROUTE_PREFIX.length));
|
||||||
|
} catch {
|
||||||
|
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({ error: 'Invalid path' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!relativePath.startsWith(ALLOWED_CDN_PATH_PREFIX)) {
|
if (!relativePath.startsWith(ALLOWED_CDN_PATH_PREFIX)) {
|
||||||
res.writeHead(403, { 'Content-Type': 'application/json' });
|
res.writeHead(403, { 'Content-Type': 'application/json' });
|
||||||
res.end(JSON.stringify({ error: 'Forbidden' }));
|
res.end(JSON.stringify({ error: 'Forbidden' }));
|
||||||
|
|
@ -46,7 +54,8 @@ const deepfilterAssetsRouteHandler = async (
|
||||||
|
|
||||||
const resolvedPath = path.resolve(CACHE_DIR, relativePath);
|
const resolvedPath = path.resolve(CACHE_DIR, relativePath);
|
||||||
const cacheBasePath = path.resolve(CACHE_DIR);
|
const cacheBasePath = path.resolve(CACHE_DIR);
|
||||||
if (!resolvedPath.startsWith(cacheBasePath)) {
|
const relativeToBase = path.relative(cacheBasePath, resolvedPath);
|
||||||
|
if (relativeToBase.startsWith('..') || path.isAbsolute(relativeToBase)) {
|
||||||
res.writeHead(403, { 'Content-Type': 'application/json' });
|
res.writeHead(403, { 'Content-Type': 'application/json' });
|
||||||
res.end(JSON.stringify({ error: 'Forbidden' }));
|
res.end(JSON.stringify({ error: 'Forbidden' }));
|
||||||
return;
|
return;
|
||||||
|
|
@ -56,31 +65,9 @@ const deepfilterAssetsRouteHandler = async (
|
||||||
await sendFile(res, resolvedPath);
|
await sendFile(res, resolvedPath);
|
||||||
return;
|
return;
|
||||||
} catch {
|
} catch {
|
||||||
// Cache miss; continue to fetch from upstream.
|
res.writeHead(404, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({ error: 'DeepFilter asset not bundled on server' }));
|
||||||
}
|
}
|
||||||
|
|
||||||
await fsp.mkdir(path.dirname(resolvedPath), { recursive: true });
|
|
||||||
|
|
||||||
const upstreamUrl = new URL(relativePath, CDN_BASE_URL).toString();
|
|
||||||
const upstream = await fetch(upstreamUrl);
|
|
||||||
if (!upstream.ok) {
|
|
||||||
res.writeHead(upstream.status, { 'Content-Type': 'application/json' });
|
|
||||||
res.end(JSON.stringify({ error: 'Upstream fetch failed' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const body = Buffer.from(await upstream.arrayBuffer());
|
|
||||||
const tempPath = `${resolvedPath}.tmp-${Date.now()}`;
|
|
||||||
await fsp.writeFile(tempPath, body);
|
|
||||||
await fsp.rename(tempPath, resolvedPath);
|
|
||||||
|
|
||||||
res.writeHead(200, {
|
|
||||||
'Content-Type':
|
|
||||||
upstream.headers.get('content-type') || guessContentType(resolvedPath),
|
|
||||||
'Content-Length': body.length,
|
|
||||||
'Cache-Control': 'public, max-age=31536000, immutable'
|
|
||||||
});
|
|
||||||
res.end(body);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export { deepfilterAssetsRouteHandler };
|
export { deepfilterAssetsRouteHandler };
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue