From 847f3163198aae439f395bc8566f43bb36dde023 Mon Sep 17 00:00:00 2001 From: Nexus Dev Date: Fri, 3 Apr 2026 22:34:44 +0000 Subject: [PATCH] feat(34-01): create usePiperTts hook and TtsButton component with piper-tts-web - Install @mintplex-labs/piper-tts-web as UI dependency - Create usePiperTts hook with prewarm/speak/stop/status/progress (VOICE-01, VOICE-02) - tts.stored() checks IndexedDB cache to skip re-download - tts.download() with progress callback for visible download progress - tts.predict() returns WAV blob URL for CPU-safe WASM synthesis - Create TtsButton component showing download progress during prewarm - TtsButton shows Volume2/VolumeX icons for idle/speaking states --- pnpm-lock.yaml | 121 ++++++++++++++++++++++++++++++++ ui/package.json | 1 + ui/src/components/TtsButton.tsx | 62 ++++++++++++++++ ui/src/hooks/usePiperTts.ts | 66 +++++++++++++++++ 4 files changed, 250 insertions(+) create mode 100644 ui/src/components/TtsButton.tsx create mode 100644 ui/src/hooks/usePiperTts.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1b8d6246..1fa9e515 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -615,6 +615,9 @@ importers: '@mdxeditor/editor': specifier: ^3.52.4 version: 3.52.4(@codemirror/language@6.12.1)(@lezer/highlight@1.2.3)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(yjs@13.6.29) + '@mintplex-labs/piper-tts-web': + specifier: ^1.0.4 + version: 1.0.4(onnxruntime-web@1.24.3) '@paperclipai/adapter-claude-local': specifier: workspace:* version: link:../packages/adapters/claude-local @@ -2130,6 +2133,11 @@ packages: '@mermaid-js/parser@1.0.0': resolution: {integrity: sha512-vvK0Hi/VWndxoh03Mmz6wa1KDriSPjS2XMZL/1l19HFwygiObEEoEwSDxOqyLzzAI6J2PU3261JjTMTO7x+BPw==} + '@mintplex-labs/piper-tts-web@1.0.4': + resolution: {integrity: sha512-Y24X+CJaGXoY5HFPSstHvJI6408OAtw3Pmq2OIYwpRpcwLLbgadWg8l1ODHNkgpB0Ps5fS9PAAQB60fHA3Bdag==} + peerDependencies: + onnxruntime-web: ^1.18.0 + '@neon-rs/load@0.0.4': resolution: {integrity: sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==} @@ -2162,6 +2170,36 @@ packages: engines: {node: '>=18'} hasBin: true + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@radix-ui/colors@3.0.0': resolution: {integrity: sha512-FUOsGBkHrYJwCSEtWRCIfQbZG7q1e6DgxCIOe1SUQzDe/7rXXeA47s8yCn6fuTNQAj1Zq4oTFi9Yjp3wzElcxg==} @@ -4572,6 +4610,9 @@ packages: resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} engines: {node: '>= 18.0.0'} + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} @@ -4639,6 +4680,9 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + hachure-fill@0.5.2: resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==} @@ -4951,6 +4995,9 @@ packages: lodash-es@4.17.23: resolution: {integrity: sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -5289,6 +5336,12 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onnxruntime-common@1.24.3: + resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} + + onnxruntime-web@1.24.3: + resolution: {integrity: sha512-41dDq7fxtTm0XzGE7N0d6m8FcOY8EWtUA65GkOixJPB/G7DGzBmiDAnVVXHznRw9bgUZpb+4/1lQK/PNxGpbrQ==} + open@11.0.0: resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==} engines: {node: '>=20'} @@ -5396,6 +5449,9 @@ packages: pkg-types@1.3.1: resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + playwright-core@1.58.2: resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} engines: {node: '>=18'} @@ -5464,6 +5520,10 @@ packages: property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -8081,6 +8141,10 @@ snapshots: dependencies: langium: 4.2.1 + '@mintplex-labs/piper-tts-web@1.0.4(onnxruntime-web@1.24.3)': + dependencies: + onnxruntime-web: 1.24.3 + '@neon-rs/load@0.0.4': {} '@noble/ciphers@2.1.1': {} @@ -8103,6 +8167,29 @@ snapshots: dependencies: playwright: 1.58.2 + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + '@radix-ui/colors@3.0.0': {} '@radix-ui/number@1.1.1': {} @@ -10672,6 +10759,8 @@ snapshots: transitivePeerDependencies: - supports-color + flatbuffers@25.9.23: {} + form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -10736,6 +10825,8 @@ snapshots: graceful-fs@4.2.11: {} + guid-typescript@1.0.9: {} + hachure-fill@0.5.2: {} has-symbols@1.1.0: {} @@ -11041,6 +11132,8 @@ snapshots: lodash-es@4.17.23: {} + long@5.3.2: {} + longest-streak@3.1.0: {} loose-envify@1.4.0: @@ -11671,6 +11764,17 @@ snapshots: dependencies: wrappy: 1.0.2 + onnxruntime-common@1.24.3: {} + + onnxruntime-web@1.24.3: + dependencies: + flatbuffers: 25.9.23 + guid-typescript: 1.0.9 + long: 5.3.2 + onnxruntime-common: 1.24.3 + platform: 1.3.6 + protobufjs: 7.5.4 + open@11.0.0: dependencies: default-browser: 5.5.0 @@ -11808,6 +11912,8 @@ snapshots: mlly: 1.8.1 pathe: 2.0.3 + platform@1.3.6: {} + playwright-core@1.58.2: {} playwright@1.58.2: @@ -11868,6 +11974,21 @@ snapshots: property-information@7.1.0: {} + protobufjs@7.5.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 25.2.3 + long: 5.3.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 diff --git a/ui/package.json b/ui/package.json index 8584594d..c09d4ffe 100644 --- a/ui/package.json +++ b/ui/package.json @@ -31,6 +31,7 @@ "@dnd-kit/utilities": "^3.2.2", "@lexical/link": "0.35.0", "@mdxeditor/editor": "^3.52.4", + "@mintplex-labs/piper-tts-web": "^1.0.4", "@paperclipai/adapter-claude-local": "workspace:*", "@paperclipai/adapter-codex-local": "workspace:*", "@paperclipai/adapter-cursor-local": "workspace:*", diff --git a/ui/src/components/TtsButton.tsx b/ui/src/components/TtsButton.tsx new file mode 100644 index 00000000..b05e8569 --- /dev/null +++ b/ui/src/components/TtsButton.tsx @@ -0,0 +1,62 @@ +import { Volume2, VolumeX, Loader2 } from "lucide-react"; +import { Button } from "./ui/button"; +import type { TtsStatus } from "../hooks/usePiperTts"; + +interface TtsButtonProps { + status: TtsStatus; + progress: number; + onSpeak: () => void; + onStop: () => void; + onPrewarm: () => void; + disabled?: boolean; +} + +export function TtsButton({ status, progress, onSpeak, onStop, onPrewarm, disabled }: TtsButtonProps) { + if (status === "downloading") { + return ( + + ); + } + + if (status === "speaking") { + return ( + + ); + } + + // idle or error: clicking triggers prewarm then speak + // ready: clicking triggers speak directly + const handleClick = () => { + if (status === "ready") { + onSpeak(); + } else { + onPrewarm(); + } + }; + + return ( + + ); +} diff --git a/ui/src/hooks/usePiperTts.ts b/ui/src/hooks/usePiperTts.ts new file mode 100644 index 00000000..b2d67488 --- /dev/null +++ b/ui/src/hooks/usePiperTts.ts @@ -0,0 +1,66 @@ +import { useState, useCallback, useRef } from "react"; +import { tts } from "@mintplex-labs/piper-tts-web"; + +const DEFAULT_VOICE = "en_US-hfc_female-medium"; + +export type TtsStatus = "idle" | "downloading" | "ready" | "speaking" | "error"; + +export function usePiperTts() { + const [status, setStatus] = useState("idle"); + const [progress, setProgress] = useState(0); + const audioRef = useRef(null); + + const prewarm = useCallback(async () => { + if (status === "ready" || status === "downloading") return; + setStatus("downloading"); + setProgress(0); + try { + const stored = await tts.stored(); + if (!stored.includes(DEFAULT_VOICE)) { + await tts.download(DEFAULT_VOICE, (p: { loaded: number; total: number }) => { + setProgress(Math.round((p.loaded / p.total) * 100)); + }); + } + setStatus("ready"); + setProgress(100); + } catch { + setStatus("error"); + } + }, [status]); + + const speak = useCallback(async (text: string) => { + if (status !== "ready") return; + // Stop any currently playing audio + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current = null; + } + setStatus("speaking"); + try { + const wav = await tts.predict({ text, voiceId: DEFAULT_VOICE }); + const audio = new Audio(wav); + audioRef.current = audio; + audio.onended = () => { + audioRef.current = null; + setStatus("ready"); + }; + audio.onerror = () => { + audioRef.current = null; + setStatus("ready"); + }; + await audio.play(); + } catch { + setStatus("ready"); + } + }, [status]); + + const stop = useCallback(() => { + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current = null; + } + if (status === "speaking") setStatus("ready"); + }, [status]); + + return { status, progress, prewarm, speak, stop }; +}