feat(34-02): voice onboarding step + PersonalAssistant voice wiring
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nexus Dev
parent
982d75d0b3
commit
c7974fa67c
3 changed files with 135 additions and 12 deletions
|
|
@ -22,6 +22,7 @@ import { ModeSelector } from "./onboarding/ModeSelector";
|
|||
import { HardwareSummaryStep } from "./onboarding/HardwareSummaryStep";
|
||||
import { ProviderSelectionStep } from "./onboarding/ProviderSelectionStep";
|
||||
import { OnboardingSummaryStep } from "./onboarding/OnboardingSummaryStep";
|
||||
import { VoiceStep } from "./onboarding/VoiceStep";
|
||||
import { useHardwareInfo } from "../hooks/useHardwareInfo";
|
||||
import { updateNexusSettings, type NexusMode } from "../api/hardware";
|
||||
import { useChatPanel } from "../context/ChatPanelContext";
|
||||
|
|
@ -37,7 +38,7 @@ function deriveProviderLabel(
|
|||
return "None selected";
|
||||
}
|
||||
|
||||
// [nexus] 5-step onboarding wizard: hardware detection → mode selection → provider selection → root directory → summary
|
||||
// [nexus] 6-step onboarding wizard: hardware detection → mode selection → provider selection → voice → root directory → summary
|
||||
export function OnboardingWizard() {
|
||||
const { onboardingOpen, onboardingOptions, closeOnboarding } = useDialog();
|
||||
const { companies, setSelectedCompanyId, loading: companiesLoading } = useCompany();
|
||||
|
|
@ -65,12 +66,15 @@ export function OnboardingWizard() {
|
|||
setRouteDismissed(false);
|
||||
}, [location.pathname]);
|
||||
|
||||
// Step state: 1 = hardware detection, 2 = mode selection, 3 = provider selection, 4 = root directory, 5 = summary
|
||||
// Step state: 1 = hardware detection, 2 = mode selection, 3 = provider selection, 4 = voice, 5 = root directory, 6 = summary
|
||||
const [step, setStep] = useState(1);
|
||||
|
||||
// Mode state: "both" pre-selected per UI-SPEC
|
||||
const [selectedMode, setSelectedMode] = useState<NexusMode>("both");
|
||||
|
||||
// Voice preference — captured in step 4
|
||||
const [voiceEnabled, setVoiceEnabled] = useState(false);
|
||||
|
||||
// [nexus] Provider credentials — captured in React state for post-company-creation submission
|
||||
const [puterToken, setPuterToken] = useState<string | null>(null);
|
||||
const [googleOAuthStateId, setGoogleOAuthStateId] = useState<string | null>(null);
|
||||
|
|
@ -97,6 +101,7 @@ export function OnboardingWizard() {
|
|||
setLoading(false);
|
||||
setStep(1);
|
||||
setSelectedMode("both");
|
||||
setVoiceEnabled(false);
|
||||
setPuterToken(null);
|
||||
setGoogleOAuthStateId(null);
|
||||
setApiKeyData(null);
|
||||
|
|
@ -210,6 +215,15 @@ export function OnboardingWizard() {
|
|||
// Non-blocking — mode defaults to "both" if save fails
|
||||
}
|
||||
|
||||
// Persist voice preference — non-blocking
|
||||
if (voiceEnabled) {
|
||||
try {
|
||||
await updateNexusSettings({ voiceEnabled: true });
|
||||
} catch {
|
||||
// Non-blocking
|
||||
}
|
||||
}
|
||||
|
||||
// [nexus] Store collected provider credentials after company creation — all non-blocking
|
||||
if (puterToken) {
|
||||
await puterProxyApi.storeToken(company.id, puterToken).catch(() => {});
|
||||
|
|
@ -248,7 +262,7 @@ export function OnboardingWizard() {
|
|||
async function handleStartChat() {
|
||||
// Guard: claude_local requires rootDir
|
||||
if (defaultAdapter === "claude_local" && !rootDir.trim()) {
|
||||
setError("Root directory is required for Claude Code. Go back to step 4 to set it.");
|
||||
setError("Root directory is required for Claude Code. Go back to step 5 to set it.");
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -288,7 +302,7 @@ export function OnboardingWizard() {
|
|||
>
|
||||
{/* Step indicator */}
|
||||
<p className="text-xs text-muted-foreground text-center">
|
||||
{step === 5 ? "Summary" : `Step ${step} of 4`}
|
||||
{step === 6 ? "Summary" : `Step ${step} of 5`}
|
||||
</p>
|
||||
|
||||
{/* Step 1 — Hardware Detection */}
|
||||
|
|
@ -397,8 +411,39 @@ export function OnboardingWizard() {
|
|||
</>
|
||||
)}
|
||||
|
||||
{/* Step 4 — Root Directory (was step 3) */}
|
||||
{/* Step 4 — Voice */}
|
||||
{step === 4 && (
|
||||
<>
|
||||
<div className="flex flex-col gap-2 text-center">
|
||||
<h1 className="text-2xl font-semibold tracking-tight">
|
||||
Voice features
|
||||
</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Speak to your assistant and hear responses read aloud. Runs entirely on your device.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<VoiceStep
|
||||
onEnable={() => {
|
||||
setVoiceEnabled(true);
|
||||
setStep(5);
|
||||
}}
|
||||
onSkip={() => setStep(5)}
|
||||
/>
|
||||
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
onClick={() => setStep(3)}
|
||||
className="w-full"
|
||||
>
|
||||
Back
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Step 5 — Root Directory (was step 4) */}
|
||||
{step === 5 && (
|
||||
<>
|
||||
{/* Header */}
|
||||
<div className="flex flex-col gap-2 text-center">
|
||||
|
|
@ -442,7 +487,7 @@ export function OnboardingWizard() {
|
|||
|
||||
<Button
|
||||
type="button"
|
||||
onClick={() => setStep(5)}
|
||||
onClick={() => setStep(6)}
|
||||
disabled={loading || probing}
|
||||
className="w-full"
|
||||
>
|
||||
|
|
@ -452,7 +497,7 @@ export function OnboardingWizard() {
|
|||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
onClick={() => setStep(3)}
|
||||
onClick={() => setStep(4)}
|
||||
className="w-full"
|
||||
disabled={loading}
|
||||
>
|
||||
|
|
@ -462,7 +507,7 @@ export function OnboardingWizard() {
|
|||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
onClick={() => setStep(5)}
|
||||
onClick={() => setStep(6)}
|
||||
className="w-full"
|
||||
disabled={loading}
|
||||
>
|
||||
|
|
@ -472,8 +517,8 @@ export function OnboardingWizard() {
|
|||
</>
|
||||
)}
|
||||
|
||||
{/* Step 5 — Summary */}
|
||||
{step === 5 && (
|
||||
{/* Step 6 — Summary (was step 5) */}
|
||||
{step === 6 && (
|
||||
<OnboardingSummaryStep
|
||||
hardwareInfo={hardwareInfo}
|
||||
selectedMode={selectedMode}
|
||||
|
|
@ -482,7 +527,7 @@ export function OnboardingWizard() {
|
|||
loading={loading}
|
||||
error={error}
|
||||
onStartChat={handleStartChat}
|
||||
onBack={() => setStep(4)}
|
||||
onBack={() => setStep(5)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
57
ui/src/components/onboarding/VoiceStep.tsx
Normal file
57
ui/src/components/onboarding/VoiceStep.tsx
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { useEffect, useState } from "react";
|
||||
import { Mic, Volume2 } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
interface VoiceStepProps {
|
||||
onEnable: () => void;
|
||||
onSkip: () => void;
|
||||
}
|
||||
|
||||
export function VoiceStep({ onEnable, onSkip }: VoiceStepProps) {
|
||||
const [micAvailable, setMicAvailable] = useState<boolean | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
navigator.mediaDevices?.enumerateDevices()
|
||||
.then(devices => setMicAvailable(devices.some(d => d.kind === "audioinput")))
|
||||
.catch(() => setMicAvailable(false));
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex items-center gap-3 rounded-lg border p-3">
|
||||
<Mic className="h-5 w-5 text-primary shrink-0" />
|
||||
<div>
|
||||
<p className="text-sm font-medium">Speech-to-Text (Whisper)</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{micAvailable === false
|
||||
? "No microphone detected — unavailable"
|
||||
: micAvailable === true
|
||||
? "Microphone detected — speak to your assistant"
|
||||
: "Checking microphone..."}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3 rounded-lg border p-3">
|
||||
<Volume2 className="h-5 w-5 text-primary shrink-0" />
|
||||
<div>
|
||||
<p className="text-sm font-medium">Text-to-Speech (Piper)</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Hear responses read aloud. Runs entirely on your device — no server needed.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
<Button onClick={onEnable} className="w-full">
|
||||
Enable voice
|
||||
</Button>
|
||||
<Button variant="ghost" onClick={onSkip} className="w-full">
|
||||
Skip
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -8,6 +8,9 @@ import { useCompany } from "../context/CompanyContext";
|
|||
import { useToast } from "../context/ToastContext";
|
||||
import { chatApi } from "../api/chat";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { VoiceRecordButton } from "@/components/VoiceRecordButton";
|
||||
import { TtsButton } from "@/components/TtsButton";
|
||||
import { usePiperTts } from "../hooks/usePiperTts";
|
||||
import type { ChatConversationListItem, ChatMessage } from "@paperclipai/shared";
|
||||
|
||||
|
||||
|
|
@ -100,6 +103,7 @@ export function PersonalAssistant() {
|
|||
const queryClient = useQueryClient();
|
||||
const navigate = useNavigate();
|
||||
const { pushToast } = useToast();
|
||||
const { status: ttsStatus, progress: ttsProgress, prewarm, speak, stop } = usePiperTts();
|
||||
|
||||
const [selectedConvId, setSelectedConvId] = useState<string | null>(routeConvId ?? null);
|
||||
const [isCreating, setIsCreating] = useState(false);
|
||||
|
|
@ -320,7 +324,20 @@ export function PersonalAssistant() {
|
|||
)}
|
||||
|
||||
{messages.map((msg) => (
|
||||
<MessageBubble key={msg.id} message={msg} />
|
||||
<div key={msg.id}>
|
||||
<MessageBubble message={msg} />
|
||||
{msg.role === "assistant" && msg.content && (
|
||||
<div className="flex justify-start pl-10 -mt-1 mb-1">
|
||||
<TtsButton
|
||||
status={ttsStatus}
|
||||
progress={ttsProgress}
|
||||
onSpeak={() => speak(msg.content)}
|
||||
onStop={stop}
|
||||
onPrewarm={prewarm}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
|
||||
{streamingContent !== null && (
|
||||
|
|
@ -349,6 +366,10 @@ export function PersonalAssistant() {
|
|||
el.style.height = `${Math.min(el.scrollHeight, 160)}px`;
|
||||
}}
|
||||
/>
|
||||
<VoiceRecordButton
|
||||
onTranscription={(text) => setInputValue((prev) => prev ? prev + " " + text : text)}
|
||||
disabled={isSending}
|
||||
/>
|
||||
<Button
|
||||
onClick={handleSend}
|
||||
disabled={!inputValue.trim() || isSending}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue