mikkel/nexus
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(34-01): create usePiperTts hook and TtsButton component with piper-tts-web

Browse source 
- Install @mintplex-labs/piper-tts-web as UI dependency
- Create usePiperTts hook with prewarm/speak/stop/status/progress (VOICE-01, VOICE-02)
- tts.stored() checks IndexedDB cache to skip re-download
- tts.download() with progress callback for visible download progress
- tts.predict() returns WAV blob URL for CPU-safe WASM synthesis
- Create TtsButton component showing download progress during prewarm
- TtsButton shows Volume2/VolumeX icons for idle/speaking states
This commit is contained in:
Nexus Dev 2026-04-03 22:34:44 +00:00
parent 0d318a31d3
commit 8f8257e143
4 changed files with 250 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
pnpm-lock.yaml generated
View file

@ -615,6 +615,9 @@ importers:
'@mdxeditor/editor':
specifier: ^3.52.4
version: 3.52.4(@codemirror/language@6.12.1)(@lezer/highlight@1.2.3)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(yjs@13.6.29)
'@mintplex-labs/piper-tts-web':
specifier: ^1.0.4
version: 1.0.4(onnxruntime-web@1.24.3)
'@paperclipai/adapter-claude-local':
specifier: workspace:*
version: link:../packages/adapters/claude-local
@ -2130,6 +2133,11 @@ packages:
'@mermaid-js/parser@1.0.0':
resolution: {integrity: sha512-vvK0Hi/VWndxoh03Mmz6wa1KDriSPjS2XMZL/1l19HFwygiObEEoEwSDxOqyLzzAI6J2PU3261JjTMTO7x+BPw==}
'@mintplex-labs/piper-tts-web@1.0.4':
resolution: {integrity: sha512-Y24X+CJaGXoY5HFPSstHvJI6408OAtw3Pmq2OIYwpRpcwLLbgadWg8l1ODHNkgpB0Ps5fS9PAAQB60fHA3Bdag==}
peerDependencies:
onnxruntime-web: ^1.18.0
'@neon-rs/load@0.0.4':
resolution: {integrity: sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==}
@ -2162,6 +2170,36 @@ packages:
engines: {node: '>=18'}
hasBin: true
'@protobufjs/aspromise@1.1.2':
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
'@protobufjs/base64@1.1.2':
resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
'@protobufjs/codegen@2.0.4':
resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==}
'@protobufjs/eventemitter@1.1.0':
resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
'@protobufjs/fetch@1.1.0':
resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
'@protobufjs/float@1.0.2':
resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
'@protobufjs/inquire@1.1.0':
resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==}
'@protobufjs/path@1.1.2':
resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
'@protobufjs/pool@1.1.0':
resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
'@protobufjs/utf8@1.1.0':
resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==}
'@radix-ui/colors@3.0.0':
resolution: {integrity: sha512-FUOsGBkHrYJwCSEtWRCIfQbZG7q1e6DgxCIOe1SUQzDe/7rXXeA47s8yCn6fuTNQAj1Zq4oTFi9Yjp3wzElcxg==}
@ -4572,6 +4610,9 @@ packages:
resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==}
engines: {node: '>= 18.0.0'}
flatbuffers@25.9.23:
resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==}
form-data@4.0.5:
resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==}
engines: {node: '>= 6'}
@ -4639,6 +4680,9 @@ packages:
graceful-fs@4.2.11:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
guid-typescript@1.0.9:
resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}
hachure-fill@0.5.2:
resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==}
@ -4951,6 +4995,9 @@ packages:
lodash-es@4.17.23:
resolution: {integrity: sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==}
long@5.3.2:
resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}
longest-streak@3.1.0:
resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==}
@ -5289,6 +5336,12 @@ packages:
once@1.4.0:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
onnxruntime-common@1.24.3:
resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==}
onnxruntime-web@1.24.3:
resolution: {integrity: sha512-41dDq7fxtTm0XzGE7N0d6m8FcOY8EWtUA65GkOixJPB/G7DGzBmiDAnVVXHznRw9bgUZpb+4/1lQK/PNxGpbrQ==}
open@11.0.0:
resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==}
engines: {node: '>=20'}
@ -5396,6 +5449,9 @@ packages:
pkg-types@1.3.1:
resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==}
platform@1.3.6:
resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
playwright-core@1.58.2:
resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==}
engines: {node: '>=18'}
@ -5464,6 +5520,10 @@ packages:
property-information@7.1.0:
resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==}
protobufjs@7.5.4:
resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==}
engines: {node: '>=12.0.0'}
proxy-addr@2.0.7:
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
@ -8081,6 +8141,10 @@ snapshots:
dependencies:
langium: 4.2.1
'@mintplex-labs/piper-tts-web@1.0.4(onnxruntime-web@1.24.3)':
dependencies:
onnxruntime-web: 1.24.3
'@neon-rs/load@0.0.4': {}
'@noble/ciphers@2.1.1': {}
@ -8103,6 +8167,29 @@ snapshots:
dependencies:
playwright: 1.58.2
'@protobufjs/aspromise@1.1.2': {}
'@protobufjs/base64@1.1.2': {}
'@protobufjs/codegen@2.0.4': {}
'@protobufjs/eventemitter@1.1.0': {}
'@protobufjs/fetch@1.1.0':
dependencies:
'@protobufjs/aspromise': 1.1.2
'@protobufjs/inquire': 1.1.0
'@protobufjs/float@1.0.2': {}
'@protobufjs/inquire@1.1.0': {}
'@protobufjs/path@1.1.2': {}
'@protobufjs/pool@1.1.0': {}
'@protobufjs/utf8@1.1.0': {}
'@radix-ui/colors@3.0.0': {}
'@radix-ui/number@1.1.1': {}
@ -10672,6 +10759,8 @@ snapshots:
transitivePeerDependencies:
- supports-color
flatbuffers@25.9.23: {}
form-data@4.0.5:
dependencies:
asynckit: 0.4.0
@ -10736,6 +10825,8 @@ snapshots:
graceful-fs@4.2.11: {}
guid-typescript@1.0.9: {}
hachure-fill@0.5.2: {}
has-symbols@1.1.0: {}
@ -11041,6 +11132,8 @@ snapshots:
lodash-es@4.17.23: {}
long@5.3.2: {}
longest-streak@3.1.0: {}
loose-envify@1.4.0:
@ -11671,6 +11764,17 @@ snapshots:
dependencies:
wrappy: 1.0.2
onnxruntime-common@1.24.3: {}
onnxruntime-web@1.24.3:
dependencies:
flatbuffers: 25.9.23
guid-typescript: 1.0.9
long: 5.3.2
onnxruntime-common: 1.24.3
platform: 1.3.6
protobufjs: 7.5.4
open@11.0.0:
dependencies:
default-browser: 5.5.0
@ -11808,6 +11912,8 @@ snapshots:
mlly: 1.8.1
pathe: 2.0.3
platform@1.3.6: {}
playwright-core@1.58.2: {}
playwright@1.58.2:
@ -11868,6 +11974,21 @@ snapshots:
property-information@7.1.0: {}
protobufjs@7.5.4:
dependencies:
'@protobufjs/aspromise': 1.1.2
'@protobufjs/base64': 1.1.2
'@protobufjs/codegen': 2.0.4
'@protobufjs/eventemitter': 1.1.0
'@protobufjs/fetch': 1.1.0
'@protobufjs/float': 1.0.2
'@protobufjs/inquire': 1.1.0
'@protobufjs/path': 1.1.2
'@protobufjs/pool': 1.1.0
'@protobufjs/utf8': 1.1.0
'@types/node': 25.2.3
long: 5.3.2
proxy-addr@2.0.7:
dependencies:
forwarded: 0.2.0

1
ui/package.json
View file

@ -31,6 +31,7 @@
"@dnd-kit/utilities": "^3.2.2",
"@lexical/link": "0.35.0",
"@mdxeditor/editor": "^3.52.4",
"@mintplex-labs/piper-tts-web": "^1.0.4",
"@paperclipai/adapter-claude-local": "workspace:*",
"@paperclipai/adapter-codex-local": "workspace:*",
"@paperclipai/adapter-cursor-local": "workspace:*",

62
ui/src/components/TtsButton.tsx Normal file
View file

@ -0,0 +1,62 @@
import { Volume2, VolumeX, Loader2 } from "lucide-react";
import { Button } from "./ui/button";
import type { TtsStatus } from "../hooks/usePiperTts";
interface TtsButtonProps {
status: TtsStatus;
progress: number;
onSpeak: () => void;
onStop: () => void;
onPrewarm: () => void;
disabled?: boolean;
}
export function TtsButton({ status, progress, onSpeak, onStop, onPrewarm, disabled }: TtsButtonProps) {
if (status === "downloading") {
return (
<Button variant="ghost" size="icon" className="h-8 w-8 relative" disabled title={`Downloading voice model: ${progress}%`}>
<Loader2 className="h-4 w-4 animate-spin" />
<span className="absolute -bottom-1 text-[10px] text-muted-foreground">{progress}%</span>
</Button>
);
}
if (status === "speaking") {
return (
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-primary"
onClick={onStop}
aria-label="Stop speaking"
title="Stop speaking"
>
<VolumeX className="h-4 w-4" />
</Button>
);
}
// idle or error: clicking triggers prewarm then speak
// ready: clicking triggers speak directly
const handleClick = () => {
if (status === "ready") {
onSpeak();
} else {
onPrewarm();
}
};
return (
<Button
variant="ghost"
size="icon"
className="h-8 w-8"
onClick={handleClick}
disabled={disabled || status === "error"}
aria-label="Read aloud"
title={status === "error" ? "TTS unavailable" : status === "idle" ? "Download voice model and read aloud" : "Read aloud"}
>
<Volume2 className="h-4 w-4" />
</Button>
);
}

66
ui/src/hooks/usePiperTts.ts Normal file
View file

@ -0,0 +1,66 @@
import { useState, useCallback, useRef } from "react";
import { tts } from "@mintplex-labs/piper-tts-web";
const DEFAULT_VOICE = "en_US-hfc_female-medium";
export type TtsStatus = "idle" | "downloading" | "ready" | "speaking" | "error";
export function usePiperTts() {
const [status, setStatus] = useState<TtsStatus>("idle");
const [progress, setProgress] = useState(0);
const audioRef = useRef<HTMLAudioElement | null>(null);
const prewarm = useCallback(async () => {
if (status === "ready" || status === "downloading") return;
setStatus("downloading");
setProgress(0);
try {
const stored = await tts.stored();
if (!stored.includes(DEFAULT_VOICE)) {
await tts.download(DEFAULT_VOICE, (p: { loaded: number; total: number }) => {
setProgress(Math.round((p.loaded / p.total) * 100));
});
}
setStatus("ready");
setProgress(100);
} catch {
setStatus("error");
}
}, [status]);
const speak = useCallback(async (text: string) => {
if (status !== "ready") return;
// Stop any currently playing audio
if (audioRef.current) {
audioRef.current.pause();
audioRef.current = null;
}
setStatus("speaking");
try {
const wav = await tts.predict({ text, voiceId: DEFAULT_VOICE });
const audio = new Audio(wav);
audioRef.current = audio;
audio.onended = () => {
audioRef.current = null;
setStatus("ready");
};
audio.onerror = () => {
audioRef.current = null;
setStatus("ready");
};
await audio.play();
} catch {
setStatus("ready");
}
}, [status]);
const stop = useCallback(() => {
if (audioRef.current) {
audioRef.current.pause();
audioRef.current = null;
}
if (status === "speaking") setStatus("ready");
}, [status]);
return { status, progress, prewarm, speak, stop };
}