feat(34-02): voice onboarding step + PersonalAssistant voice wiring
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e7090f63c3
commit
c55b085caa
3 changed files with 135 additions and 12 deletions
|
|
@ -22,6 +22,7 @@ import { ModeSelector } from "./onboarding/ModeSelector";
|
||||||
import { HardwareSummaryStep } from "./onboarding/HardwareSummaryStep";
|
import { HardwareSummaryStep } from "./onboarding/HardwareSummaryStep";
|
||||||
import { ProviderSelectionStep } from "./onboarding/ProviderSelectionStep";
|
import { ProviderSelectionStep } from "./onboarding/ProviderSelectionStep";
|
||||||
import { OnboardingSummaryStep } from "./onboarding/OnboardingSummaryStep";
|
import { OnboardingSummaryStep } from "./onboarding/OnboardingSummaryStep";
|
||||||
|
import { VoiceStep } from "./onboarding/VoiceStep";
|
||||||
import { useHardwareInfo } from "../hooks/useHardwareInfo";
|
import { useHardwareInfo } from "../hooks/useHardwareInfo";
|
||||||
import { updateNexusSettings, type NexusMode } from "../api/hardware";
|
import { updateNexusSettings, type NexusMode } from "../api/hardware";
|
||||||
import { useChatPanel } from "../context/ChatPanelContext";
|
import { useChatPanel } from "../context/ChatPanelContext";
|
||||||
|
|
@ -37,7 +38,7 @@ function deriveProviderLabel(
|
||||||
return "None selected";
|
return "None selected";
|
||||||
}
|
}
|
||||||
|
|
||||||
// [nexus] 5-step onboarding wizard: hardware detection → mode selection → provider selection → root directory → summary
|
// [nexus] 6-step onboarding wizard: hardware detection → mode selection → provider selection → voice → root directory → summary
|
||||||
export function OnboardingWizard() {
|
export function OnboardingWizard() {
|
||||||
const { onboardingOpen, onboardingOptions, closeOnboarding } = useDialog();
|
const { onboardingOpen, onboardingOptions, closeOnboarding } = useDialog();
|
||||||
const { companies, setSelectedCompanyId, loading: companiesLoading } = useCompany();
|
const { companies, setSelectedCompanyId, loading: companiesLoading } = useCompany();
|
||||||
|
|
@ -65,12 +66,15 @@ export function OnboardingWizard() {
|
||||||
setRouteDismissed(false);
|
setRouteDismissed(false);
|
||||||
}, [location.pathname]);
|
}, [location.pathname]);
|
||||||
|
|
||||||
// Step state: 1 = hardware detection, 2 = mode selection, 3 = provider selection, 4 = root directory, 5 = summary
|
// Step state: 1 = hardware detection, 2 = mode selection, 3 = provider selection, 4 = voice, 5 = root directory, 6 = summary
|
||||||
const [step, setStep] = useState(1);
|
const [step, setStep] = useState(1);
|
||||||
|
|
||||||
// Mode state: "both" pre-selected per UI-SPEC
|
// Mode state: "both" pre-selected per UI-SPEC
|
||||||
const [selectedMode, setSelectedMode] = useState<NexusMode>("both");
|
const [selectedMode, setSelectedMode] = useState<NexusMode>("both");
|
||||||
|
|
||||||
|
// Voice preference — captured in step 4
|
||||||
|
const [voiceEnabled, setVoiceEnabled] = useState(false);
|
||||||
|
|
||||||
// [nexus] Provider credentials — captured in React state for post-company-creation submission
|
// [nexus] Provider credentials — captured in React state for post-company-creation submission
|
||||||
const [puterToken, setPuterToken] = useState<string | null>(null);
|
const [puterToken, setPuterToken] = useState<string | null>(null);
|
||||||
const [googleOAuthStateId, setGoogleOAuthStateId] = useState<string | null>(null);
|
const [googleOAuthStateId, setGoogleOAuthStateId] = useState<string | null>(null);
|
||||||
|
|
@ -97,6 +101,7 @@ export function OnboardingWizard() {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
setStep(1);
|
setStep(1);
|
||||||
setSelectedMode("both");
|
setSelectedMode("both");
|
||||||
|
setVoiceEnabled(false);
|
||||||
setPuterToken(null);
|
setPuterToken(null);
|
||||||
setGoogleOAuthStateId(null);
|
setGoogleOAuthStateId(null);
|
||||||
setApiKeyData(null);
|
setApiKeyData(null);
|
||||||
|
|
@ -210,6 +215,15 @@ export function OnboardingWizard() {
|
||||||
// Non-blocking — mode defaults to "both" if save fails
|
// Non-blocking — mode defaults to "both" if save fails
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Persist voice preference — non-blocking
|
||||||
|
if (voiceEnabled) {
|
||||||
|
try {
|
||||||
|
await updateNexusSettings({ voiceEnabled: true });
|
||||||
|
} catch {
|
||||||
|
// Non-blocking
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// [nexus] Store collected provider credentials after company creation — all non-blocking
|
// [nexus] Store collected provider credentials after company creation — all non-blocking
|
||||||
if (puterToken) {
|
if (puterToken) {
|
||||||
await puterProxyApi.storeToken(company.id, puterToken).catch(() => {});
|
await puterProxyApi.storeToken(company.id, puterToken).catch(() => {});
|
||||||
|
|
@ -248,7 +262,7 @@ export function OnboardingWizard() {
|
||||||
async function handleStartChat() {
|
async function handleStartChat() {
|
||||||
// Guard: claude_local requires rootDir
|
// Guard: claude_local requires rootDir
|
||||||
if (defaultAdapter === "claude_local" && !rootDir.trim()) {
|
if (defaultAdapter === "claude_local" && !rootDir.trim()) {
|
||||||
setError("Root directory is required for Claude Code. Go back to step 4 to set it.");
|
setError("Root directory is required for Claude Code. Go back to step 5 to set it.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -288,7 +302,7 @@ export function OnboardingWizard() {
|
||||||
>
|
>
|
||||||
{/* Step indicator */}
|
{/* Step indicator */}
|
||||||
<p className="text-xs text-muted-foreground text-center">
|
<p className="text-xs text-muted-foreground text-center">
|
||||||
{step === 5 ? "Summary" : `Step ${step} of 4`}
|
{step === 6 ? "Summary" : `Step ${step} of 5`}
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
{/* Step 1 — Hardware Detection */}
|
{/* Step 1 — Hardware Detection */}
|
||||||
|
|
@ -397,8 +411,39 @@ export function OnboardingWizard() {
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Step 4 — Root Directory (was step 3) */}
|
{/* Step 4 — Voice */}
|
||||||
{step === 4 && (
|
{step === 4 && (
|
||||||
|
<>
|
||||||
|
<div className="flex flex-col gap-2 text-center">
|
||||||
|
<h1 className="text-2xl font-semibold tracking-tight">
|
||||||
|
Voice features
|
||||||
|
</h1>
|
||||||
|
<p className="text-sm text-muted-foreground">
|
||||||
|
Speak to your assistant and hear responses read aloud. Runs entirely on your device.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<VoiceStep
|
||||||
|
onEnable={() => {
|
||||||
|
setVoiceEnabled(true);
|
||||||
|
setStep(5);
|
||||||
|
}}
|
||||||
|
onSkip={() => setStep(5)}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="ghost"
|
||||||
|
onClick={() => setStep(3)}
|
||||||
|
className="w-full"
|
||||||
|
>
|
||||||
|
Back
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Step 5 — Root Directory (was step 4) */}
|
||||||
|
{step === 5 && (
|
||||||
<>
|
<>
|
||||||
{/* Header */}
|
{/* Header */}
|
||||||
<div className="flex flex-col gap-2 text-center">
|
<div className="flex flex-col gap-2 text-center">
|
||||||
|
|
@ -442,7 +487,7 @@ export function OnboardingWizard() {
|
||||||
|
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => setStep(5)}
|
onClick={() => setStep(6)}
|
||||||
disabled={loading || probing}
|
disabled={loading || probing}
|
||||||
className="w-full"
|
className="w-full"
|
||||||
>
|
>
|
||||||
|
|
@ -452,7 +497,7 @@ export function OnboardingWizard() {
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
onClick={() => setStep(3)}
|
onClick={() => setStep(4)}
|
||||||
className="w-full"
|
className="w-full"
|
||||||
disabled={loading}
|
disabled={loading}
|
||||||
>
|
>
|
||||||
|
|
@ -462,7 +507,7 @@ export function OnboardingWizard() {
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
onClick={() => setStep(5)}
|
onClick={() => setStep(6)}
|
||||||
className="w-full"
|
className="w-full"
|
||||||
disabled={loading}
|
disabled={loading}
|
||||||
>
|
>
|
||||||
|
|
@ -472,8 +517,8 @@ export function OnboardingWizard() {
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Step 5 — Summary */}
|
{/* Step 6 — Summary (was step 5) */}
|
||||||
{step === 5 && (
|
{step === 6 && (
|
||||||
<OnboardingSummaryStep
|
<OnboardingSummaryStep
|
||||||
hardwareInfo={hardwareInfo}
|
hardwareInfo={hardwareInfo}
|
||||||
selectedMode={selectedMode}
|
selectedMode={selectedMode}
|
||||||
|
|
@ -482,7 +527,7 @@ export function OnboardingWizard() {
|
||||||
loading={loading}
|
loading={loading}
|
||||||
error={error}
|
error={error}
|
||||||
onStartChat={handleStartChat}
|
onStartChat={handleStartChat}
|
||||||
onBack={() => setStep(4)}
|
onBack={() => setStep(5)}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
57
ui/src/components/onboarding/VoiceStep.tsx
Normal file
57
ui/src/components/onboarding/VoiceStep.tsx
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { Mic, Volume2 } from "lucide-react";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
|
||||||
|
interface VoiceStepProps {
|
||||||
|
onEnable: () => void;
|
||||||
|
onSkip: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function VoiceStep({ onEnable, onSkip }: VoiceStepProps) {
|
||||||
|
const [micAvailable, setMicAvailable] = useState<boolean | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
navigator.mediaDevices?.enumerateDevices()
|
||||||
|
.then(devices => setMicAvailable(devices.some(d => d.kind === "audioinput")))
|
||||||
|
.catch(() => setMicAvailable(false));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-4">
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
<div className="flex items-center gap-3 rounded-lg border p-3">
|
||||||
|
<Mic className="h-5 w-5 text-primary shrink-0" />
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium">Speech-to-Text (Whisper)</p>
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
{micAvailable === false
|
||||||
|
? "No microphone detected — unavailable"
|
||||||
|
: micAvailable === true
|
||||||
|
? "Microphone detected — speak to your assistant"
|
||||||
|
: "Checking microphone..."}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center gap-3 rounded-lg border p-3">
|
||||||
|
<Volume2 className="h-5 w-5 text-primary shrink-0" />
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium">Text-to-Speech (Piper)</p>
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
Hear responses read aloud. Runs entirely on your device — no server needed.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex flex-col gap-2">
|
||||||
|
<Button onClick={onEnable} className="w-full">
|
||||||
|
Enable voice
|
||||||
|
</Button>
|
||||||
|
<Button variant="ghost" onClick={onSkip} className="w-full">
|
||||||
|
Skip
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -8,6 +8,9 @@ import { useCompany } from "../context/CompanyContext";
|
||||||
import { useToast } from "../context/ToastContext";
|
import { useToast } from "../context/ToastContext";
|
||||||
import { chatApi } from "../api/chat";
|
import { chatApi } from "../api/chat";
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { VoiceRecordButton } from "@/components/VoiceRecordButton";
|
||||||
|
import { TtsButton } from "@/components/TtsButton";
|
||||||
|
import { usePiperTts } from "../hooks/usePiperTts";
|
||||||
import type { ChatConversationListItem, ChatMessage } from "@paperclipai/shared";
|
import type { ChatConversationListItem, ChatMessage } from "@paperclipai/shared";
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -100,6 +103,7 @@ export function PersonalAssistant() {
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
const { pushToast } = useToast();
|
const { pushToast } = useToast();
|
||||||
|
const { status: ttsStatus, progress: ttsProgress, prewarm, speak, stop } = usePiperTts();
|
||||||
|
|
||||||
const [selectedConvId, setSelectedConvId] = useState<string | null>(routeConvId ?? null);
|
const [selectedConvId, setSelectedConvId] = useState<string | null>(routeConvId ?? null);
|
||||||
const [isCreating, setIsCreating] = useState(false);
|
const [isCreating, setIsCreating] = useState(false);
|
||||||
|
|
@ -320,7 +324,20 @@ export function PersonalAssistant() {
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{messages.map((msg) => (
|
{messages.map((msg) => (
|
||||||
<MessageBubble key={msg.id} message={msg} />
|
<div key={msg.id}>
|
||||||
|
<MessageBubble message={msg} />
|
||||||
|
{msg.role === "assistant" && msg.content && (
|
||||||
|
<div className="flex justify-start pl-10 -mt-1 mb-1">
|
||||||
|
<TtsButton
|
||||||
|
status={ttsStatus}
|
||||||
|
progress={ttsProgress}
|
||||||
|
onSpeak={() => speak(msg.content)}
|
||||||
|
onStop={stop}
|
||||||
|
onPrewarm={prewarm}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
))}
|
))}
|
||||||
|
|
||||||
{streamingContent !== null && (
|
{streamingContent !== null && (
|
||||||
|
|
@ -349,6 +366,10 @@ export function PersonalAssistant() {
|
||||||
el.style.height = `${Math.min(el.scrollHeight, 160)}px`;
|
el.style.height = `${Math.min(el.scrollHeight, 160)}px`;
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
<VoiceRecordButton
|
||||||
|
onTranscription={(text) => setInputValue((prev) => prev ? prev + " " + text : text)}
|
||||||
|
disabled={isSending}
|
||||||
|
/>
|
||||||
<Button
|
<Button
|
||||||
onClick={handleSend}
|
onClick={handleSend}
|
||||||
disabled={!inputValue.trim() || isSending}
|
disabled={!inputValue.trim() || isSending}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue