diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 906972e4..5ea57460 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -691,6 +691,9 @@ importers: '@radix-ui/react-slot': specifier: ^1.2.4 version: 1.2.4(@types/react@19.2.14)(react@19.2.4) + '@ricky0123/vad-react': + specifier: ^0.0.36 + version: 0.0.36(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@tailwindcss/typography': specifier: ^0.5.19 version: 0.5.19(tailwindcss@4.1.18) @@ -3255,6 +3258,15 @@ packages: peerDependencies: react: '>=16.8' + '@ricky0123/vad-react@0.0.36': + resolution: {integrity: sha512-cD55/RrZAY/ju6SJzZnL/6J5NyJ7ERg99XHeCe4WidRPZnN+tonsTWD/lG6bRFqRfEIFHirUtU60r6SXGLiK8Q==} + peerDependencies: + react: '>=16.8.0' + react-dom: '>=16.8.0' + + '@ricky0123/vad-web@0.0.30': + resolution: {integrity: sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==} + '@rolldown/pluginutils@1.0.0-beta.27': resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} @@ -9599,6 +9611,17 @@ snapshots: dependencies: react: 19.2.4 + '@ricky0123/vad-react@0.0.36(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@ricky0123/vad-web': 0.0.30 + onnxruntime-web: 1.24.3 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@ricky0123/vad-web@0.0.30': + dependencies: + onnxruntime-web: 1.24.3 + '@rolldown/pluginutils@1.0.0-beta.27': {} '@rollup/plugin-node-resolve@16.0.3(rollup@4.57.1)': diff --git a/ui/package.json b/ui/package.json index c09d4ffe..2ca47032 100644 --- a/ui/package.json +++ b/ui/package.json @@ -19,6 +19,7 @@ "preview": "vite preview", "typecheck": "tsc -b", "clean": "rm -rf dist tsconfig.tsbuildinfo", + "copy-vad-assets": "cp node_modules/@ricky0123/vad-web/dist/vad.worklet.bundle.min.js public/ && cp node_modules/@ricky0123/vad-web/dist/silero_vad_legacy.onnx public/ && cp node_modules/@ricky0123/vad-web/dist/silero_vad_v5.onnx public/", "prepack": "rm -f package.dev.json && cp package.json package.dev.json && node ../scripts/generate-ui-package-json.mjs", "postpack": "if [ -f package.dev.json ]; then mv package.dev.json package.json; fi" }, @@ -43,6 +44,7 @@ "@paperclipai/branding": "workspace:*", "@paperclipai/shared": "workspace:*", "@radix-ui/react-slot": "^1.2.4", + "@ricky0123/vad-react": "^0.0.36", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-query": "^5.90.21", "@tanstack/react-virtual": "^3.13.23", diff --git a/ui/public/silero_vad_legacy.onnx b/ui/public/silero_vad_legacy.onnx new file mode 100644 index 00000000..e6db48d6 Binary files /dev/null and b/ui/public/silero_vad_legacy.onnx differ diff --git a/ui/public/silero_vad_v5.onnx b/ui/public/silero_vad_v5.onnx new file mode 100644 index 00000000..b3e3a900 Binary files /dev/null and b/ui/public/silero_vad_v5.onnx differ diff --git a/ui/public/vad.worklet.bundle.min.js b/ui/public/vad.worklet.bundle.min.js new file mode 100644 index 00000000..dfa2c2b9 --- /dev/null +++ b/ui/public/vad.worklet.bundle.min.js @@ -0,0 +1 @@ +(()=>{"use strict";var e={710:(e,t)=>{Object.defineProperty(t,"__esModule",{value:!0}),t.log=void 0;const s=e=>t=>{console.log(`VAD | ${e} >`,t)};t.log={error:s("error"),debug:s("debug"),warn:s("warn")}},954:(e,t)=>{var s;Object.defineProperty(t,"__esModule",{value:!0}),t.Message=void 0,function(e){e.AudioFrame="AUDIO_FRAME",e.SpeechStart="SPEECH_START",e.VADMisfire="VAD_MISFIRE",e.SpeechEnd="SPEECH_END",e.SpeechStop="SPEECH_STOP",e.SpeechRealStart="SPEECH_REAL_START",e.FrameProcessed="FRAME_PROCESSED"}(s||(t.Message=s={}))},825:(e,t,s)=>{Object.defineProperty(t,"__esModule",{value:!0}),t.Resampler=void 0;const r=s(710);t.Resampler=class{constructor(e){this.options=e,this.process=e=>{const t=[];for(const s of e)for(this.inputBuffer.push(s);this.hasEnoughDataForFrame();){const e=this.generateOutputFrame();t.push(e)}return t},e.nativeSampleRate<16e3&&r.log.error("nativeSampleRate is too low. Should have 16000 = targetSampleRate <= nativeSampleRate"),this.inputBuffer=[]}async*stream(e){for(const t of e)for(this.inputBuffer.push(t);this.hasEnoughDataForFrame();){const e=this.generateOutputFrame();yield e}}hasEnoughDataForFrame(){return this.inputBuffer.length*this.options.targetSampleRate/this.options.nativeSampleRate>=this.options.targetFrameSize}generateOutputFrame(){const e=new Float32Array(this.options.targetFrameSize);let t=0,s=0;for(;t{const e=s(710),t=s(954),r=s(825);class o extends AudioWorkletProcessor{constructor(s){super(),this._stopProcessing=!1,this.options=s.processorOptions,this.port.onmessage=s=>{s.data===t.Message.SpeechStop&&(e.log.debug("Worklet received speech stop message"),this._stopProcessing=!0)},this.resampler=new r.Resampler({nativeSampleRate:sampleRate,targetSampleRate:16e3,targetFrameSize:this.options.frameSamples})}process(e){if(this._stopProcessing)return!1;const s=e[0];if(void 0===s)return!0;const r=s[0];if(void 0===r)return!0;const o=this.resampler.process(r);for(const e of o)this.port.postMessage({message:t.Message.AudioFrame,data:e.buffer},[e.buffer]);return!0}}registerProcessor("vad-helper-worklet",o)})()})(); \ No newline at end of file diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 09ddb586..90f2534b 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -30,6 +30,10 @@ export default defineConfig({ }, server: { port: 5173, + headers: { + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Embedder-Policy": "require-corp", + }, proxy: { "/api": { target: "http://localhost:3100",