From c7974fa67ca0fb2380613a7d0b084de139bc57f7 Mon Sep 17 00:00:00 2001 From: Nexus Dev Date: Fri, 3 Apr 2026 22:42:45 +0000 Subject: [PATCH] feat(34-02): voice onboarding step + PersonalAssistant voice wiring Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/src/components/NexusOnboardingWizard.tsx | 67 +++++++++++++++++---- ui/src/components/onboarding/VoiceStep.tsx | 57 ++++++++++++++++++ ui/src/pages/PersonalAssistant.tsx | 23 ++++++- 3 files changed, 135 insertions(+), 12 deletions(-) create mode 100644 ui/src/components/onboarding/VoiceStep.tsx diff --git a/ui/src/components/NexusOnboardingWizard.tsx b/ui/src/components/NexusOnboardingWizard.tsx index 2c157cd8..2f86ebff 100644 --- a/ui/src/components/NexusOnboardingWizard.tsx +++ b/ui/src/components/NexusOnboardingWizard.tsx @@ -22,6 +22,7 @@ import { ModeSelector } from "./onboarding/ModeSelector"; import { HardwareSummaryStep } from "./onboarding/HardwareSummaryStep"; import { ProviderSelectionStep } from "./onboarding/ProviderSelectionStep"; import { OnboardingSummaryStep } from "./onboarding/OnboardingSummaryStep"; +import { VoiceStep } from "./onboarding/VoiceStep"; import { useHardwareInfo } from "../hooks/useHardwareInfo"; import { updateNexusSettings, type NexusMode } from "../api/hardware"; import { useChatPanel } from "../context/ChatPanelContext"; @@ -37,7 +38,7 @@ function deriveProviderLabel( return "None selected"; } -// [nexus] 5-step onboarding wizard: hardware detection → mode selection → provider selection → root directory → summary +// [nexus] 6-step onboarding wizard: hardware detection → mode selection → provider selection → voice → root directory → summary export function OnboardingWizard() { const { onboardingOpen, onboardingOptions, closeOnboarding } = useDialog(); const { companies, setSelectedCompanyId, loading: companiesLoading } = useCompany(); @@ -65,12 +66,15 @@ export function OnboardingWizard() { setRouteDismissed(false); }, [location.pathname]); - // Step state: 1 = hardware detection, 2 = mode selection, 3 = provider selection, 4 = root directory, 5 = summary + // Step state: 1 = hardware detection, 2 = mode selection, 3 = provider selection, 4 = voice, 5 = root directory, 6 = summary const [step, setStep] = useState(1); // Mode state: "both" pre-selected per UI-SPEC const [selectedMode, setSelectedMode] = useState("both"); + // Voice preference — captured in step 4 + const [voiceEnabled, setVoiceEnabled] = useState(false); + // [nexus] Provider credentials — captured in React state for post-company-creation submission const [puterToken, setPuterToken] = useState(null); const [googleOAuthStateId, setGoogleOAuthStateId] = useState(null); @@ -97,6 +101,7 @@ export function OnboardingWizard() { setLoading(false); setStep(1); setSelectedMode("both"); + setVoiceEnabled(false); setPuterToken(null); setGoogleOAuthStateId(null); setApiKeyData(null); @@ -210,6 +215,15 @@ export function OnboardingWizard() { // Non-blocking — mode defaults to "both" if save fails } + // Persist voice preference — non-blocking + if (voiceEnabled) { + try { + await updateNexusSettings({ voiceEnabled: true }); + } catch { + // Non-blocking + } + } + // [nexus] Store collected provider credentials after company creation — all non-blocking if (puterToken) { await puterProxyApi.storeToken(company.id, puterToken).catch(() => {}); @@ -248,7 +262,7 @@ export function OnboardingWizard() { async function handleStartChat() { // Guard: claude_local requires rootDir if (defaultAdapter === "claude_local" && !rootDir.trim()) { - setError("Root directory is required for Claude Code. Go back to step 4 to set it."); + setError("Root directory is required for Claude Code. Go back to step 5 to set it."); return; } @@ -288,7 +302,7 @@ export function OnboardingWizard() { > {/* Step indicator */}

- {step === 5 ? "Summary" : `Step ${step} of 4`} + {step === 6 ? "Summary" : `Step ${step} of 5`}

{/* Step 1 — Hardware Detection */} @@ -397,8 +411,39 @@ export function OnboardingWizard() { )} - {/* Step 4 — Root Directory (was step 3) */} + {/* Step 4 — Voice */} {step === 4 && ( + <> +
+

+ Voice features +

+

+ Speak to your assistant and hear responses read aloud. Runs entirely on your device. +

+
+ + { + setVoiceEnabled(true); + setStep(5); + }} + onSkip={() => setStep(5)} + /> + + + + )} + + {/* Step 5 — Root Directory (was step 4) */} + {step === 5 && ( <> {/* Header */}
@@ -442,7 +487,7 @@ export function OnboardingWizard() {
diff --git a/ui/src/components/onboarding/VoiceStep.tsx b/ui/src/components/onboarding/VoiceStep.tsx new file mode 100644 index 00000000..08b5656f --- /dev/null +++ b/ui/src/components/onboarding/VoiceStep.tsx @@ -0,0 +1,57 @@ +import { useEffect, useState } from "react"; +import { Mic, Volume2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +interface VoiceStepProps { + onEnable: () => void; + onSkip: () => void; +} + +export function VoiceStep({ onEnable, onSkip }: VoiceStepProps) { + const [micAvailable, setMicAvailable] = useState(null); + + useEffect(() => { + navigator.mediaDevices?.enumerateDevices() + .then(devices => setMicAvailable(devices.some(d => d.kind === "audioinput"))) + .catch(() => setMicAvailable(false)); + }, []); + + return ( +
+
+
+ +
+

Speech-to-Text (Whisper)

+

+ {micAvailable === false + ? "No microphone detected — unavailable" + : micAvailable === true + ? "Microphone detected — speak to your assistant" + : "Checking microphone..."} +

+
+
+ +
+ +
+

Text-to-Speech (Piper)

+

+ Hear responses read aloud. Runs entirely on your device — no server needed. +

+
+
+
+ +
+ + +
+
+ ); +} diff --git a/ui/src/pages/PersonalAssistant.tsx b/ui/src/pages/PersonalAssistant.tsx index 84018ebb..0e578420 100644 --- a/ui/src/pages/PersonalAssistant.tsx +++ b/ui/src/pages/PersonalAssistant.tsx @@ -8,6 +8,9 @@ import { useCompany } from "../context/CompanyContext"; import { useToast } from "../context/ToastContext"; import { chatApi } from "../api/chat"; import { Button } from "@/components/ui/button"; +import { VoiceRecordButton } from "@/components/VoiceRecordButton"; +import { TtsButton } from "@/components/TtsButton"; +import { usePiperTts } from "../hooks/usePiperTts"; import type { ChatConversationListItem, ChatMessage } from "@paperclipai/shared"; @@ -100,6 +103,7 @@ export function PersonalAssistant() { const queryClient = useQueryClient(); const navigate = useNavigate(); const { pushToast } = useToast(); + const { status: ttsStatus, progress: ttsProgress, prewarm, speak, stop } = usePiperTts(); const [selectedConvId, setSelectedConvId] = useState(routeConvId ?? null); const [isCreating, setIsCreating] = useState(false); @@ -320,7 +324,20 @@ export function PersonalAssistant() { )} {messages.map((msg) => ( - +
+ + {msg.role === "assistant" && msg.content && ( +
+ speak(msg.content)} + onStop={stop} + onPrewarm={prewarm} + /> +
+ )} +
))} {streamingContent !== null && ( @@ -349,6 +366,10 @@ export function PersonalAssistant() { el.style.height = `${Math.min(el.scrollHeight, 160)}px`; }} /> + setInputValue((prev) => prev ? prev + " " + text : text)} + disabled={isSending} + />