package main import ( "bufio" "context" "crypto/tls" "encoding/json" "errors" "flag" "fmt" "io" "log" "math" "net/http" "os" "os/exec" "os/signal" "strings" "sync" "syscall" "time" "github.com/gorilla/websocket" ) const rate = 16000 const frameSamples = 320 const hop = 160 type template struct { Frames [][]float64 `json:"frames"` } type message struct { Type string `json:"type"` Code string `json:"code,omitempty"` Wake bool `json:"wake,omitempty"` turnOptions } var hann [frameSamples]float64 var coefficients [20]float64 func init() { for i := range hann { hann[i] = 0.5 - 0.5*math.Cos(2*math.Pi*float64(i)/float64(frameSamples-1)) } for b := range coefficients { freq := 150 * math.Pow(7000.0/150.0, float64(b)/19) coefficients[b] = 2 * math.Cos(2*math.Pi*freq/rate) } } // features calculates log spectral energy in coarse bands, normalized against overall level. // It is a small experimental template matcher, not a production speech model. func features(samples []int16) []float64 { const bands = 20 out := make([]float64, bands) for b := 0; b < bands; b++ { coeff := coefficients[b] var a, c float64 for i, s := range samples { v := float64(s)*hann[i] + coeff*a - c c = a a = v } power := a*a + c*c - coeff*a*c out[b] = math.Log(1 + math.Max(0, power)) } var mean float64 for _, v := range out { mean += v } mean /= bands for i := range out { out[i] -= mean } return out } func rms(s []int16) float64 { var v float64 for _, x := range s { v += float64(x) * float64(x) } return math.Sqrt(v / float64(len(s))) } func distance(a, b [][]float64) float64 { if len(a) == 0 || len(b) == 0 { return math.Inf(1) } prev := make([]float64, len(b)+1) for i := 1; i < len(prev); i++ { prev[i] = math.Inf(1) } for _, x := range a { cur := make([]float64, len(b)+1) cur[0] = math.Inf(1) for j, y := range b { var cost float64 for k := range x { d := x[k] - y[k] cost += d * d } cost = math.Sqrt(cost / float64(len(x))) cur[j+1] = cost + math.Min(prev[j], math.Min(prev[j+1], cur[j])) } prev = cur } return prev[len(b)] / float64(len(a)+len(b)) } func loadTemplates(path string) ([]template, error) { f, e := os.Open(path) if e != nil { return nil, e } defer f.Close() b, e := io.ReadAll(io.LimitReader(f, 2<<20+1)) if len(b) > 2<<20 { return nil, errors.New("template file too large") } if e != nil { return nil, e } var t []template e = json.Unmarshal(b, &t) if e != nil { return nil, e } if len(t) == 0 || len(t) > 10 { return nil, errors.New("no templates") } for _, x := range t { if len(x.Frames) == 0 || len(x.Frames) > 400 { return nil, errors.New("empty template") } for _, f := range x.Frames { if len(f) != 20 { return nil, errors.New("invalid feature dimensions") } } } return t, nil } func waitRecord(r *liveReader, max time.Duration, silence time.Duration, threshold float64) ([][]float64, error) { var samples []int16 var frames [][]float64 start := time.Now() last := start heard := false for time.Since(start) < max { audio, e := r.next() if e != nil { return nil, e } if !unmuted() { samples = nil frames = nil heard = false last = time.Now() start = time.Now() continue } samples = append(samples, audio.pcm...) if len(samples) >= frameSamples { frame := samples[:frameSamples] if rms(frame) >= threshold { heard = true last = time.Now() } if heard { frames = append(frames, features(frame)) } samples = samples[hop:] } if len(frames) >= 250 { return frames, nil } if heard && time.Since(last) >= silence { return frames, nil } } if !heard { return nil, errors.New("no speech detected") } return frames, nil } func enroll(r *liveReader, path string, count int, threshold float64) error { var all []template in := bufio.NewReader(os.Stdin) for i := 0; i < count; i++ { fmt.Printf("Sample %d/%d: press Enter, then say the wake phrase.\n", i+1, count) if _, e := in.ReadString('\n'); e != nil { return e } for !unmuted() { time.Sleep(100 * time.Millisecond) } lights.set("listening") r.discardPending() f, e := waitRecord(r, 4*time.Second, 350*time.Millisecond, threshold) if e != nil { return e } lights.set("idle") all = append(all, template{f}) fmt.Printf("Recorded %d frames.\n", len(f)) } b, e := json.Marshal(all) if e != nil { return e } return os.WriteFile(path, b, 0600) } func connect(ctx context.Context, url, token string, options *turnOptions) (*websocket.Conn, error) { h := http.Header{} h.Set("Authorization", "Bearer "+token) d := websocket.Dialer{HandshakeTimeout: 2 * time.Second, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}} c, resp, e := d.DialContext(ctx, url, h) if e != nil { if resp != nil { return nil, fmt.Errorf("websocket handshake HTTP %d: %w", resp.StatusCode, e) } return nil, e } c.SetReadLimit(4 << 20) _ = c.SetReadDeadline(time.Now().Add(750 * time.Millisecond)) var m message if e = c.ReadJSON(&m); e != nil { c.Close() return nil, e } if m.Type != "ready" { c.Close() return nil, fmt.Errorf("expected ready, got %q", m.Type) } _ = c.SetReadDeadline(time.Time{}) if options != nil { *options = m.turnOptions } return c, nil } func playbackCommand(ctx context.Context, device string) *exec.Cmd { player := exec.CommandContext(ctx, "aplay", "-q", "-D", device, "-t", "raw", "-f", "S16_LE", "-r", "24000", "-c", "1") if path := os.Getenv("ALICE_PLAYER"); path != "" { player = exec.CommandContext(ctx, path) } player.Stderr = os.Stderr terminateWithClient(player) return player } func exchange(ctx context.Context, url, token string, live *liveReader, fileAudio []byte, wakeAudio []int16, threshold float64, max, silence time.Duration, playback string, followUp time.Duration, options *turnOptions) (result error) { defer func() { if ctx.Err() != nil { result = ctx.Err() } }() ducked := false duck := func() { if !ducked { ducked = true _ = musicCallContext(ctx, "duck", json.RawMessage(`{"active":true}`)) } } defer func() { lights.set("idle") if ducked { _ = musicCall("duck", json.RawMessage(`{"active":false}`)) } }() if live != nil { if !unmuted() { return errMuted } lights.set("listening") if followUp == 0 { duck() } if wakeAudio == nil { live.discardPending() } } c, e := connect(ctx, url, token, options) if e != nil { return e } defer c.Close() stopClose := context.AfterFunc(ctx, func() { c.Close() }) defer stopClose() if wakeAudio != nil && options != nil && options.ActivationSound { go activationTone(ctx, playback) } if fileAudio == nil && !unmuted() { return errMuted } _ = c.SetWriteDeadline(time.Now().Add(15 * time.Second)) if fileAudio != nil { if e = c.WriteJSON(message{Type: "start"}); e != nil { return e } for len(fileAudio) > 0 { n := 3200 if n > len(fileAudio) { n = len(fileAudio) } _ = c.SetWriteDeadline(time.Now().Add(15 * time.Second)) if e = c.WriteMessage(websocket.BinaryMessage, fileAudio[:n]); e != nil { return e } fileAudio = fileAudio[n:] } } else { // Preserve the utterance across activation, including commands spoken // together with the wake word. Count PCM frames, independent of TLS timing. const prerollSamples = rate pre := make([]int16, 0, prerollSamples+hop) pre = append(pre, wakeAudio...) var frames, voiced, sent int lastVoice := -9 var peakRMS float64 heard := false waitForSpeech := 4 * time.Second if followUp > 0 { waitForSpeech = followUp } for ; frames < int(max/(10*time.Millisecond)); frames++ { if !heard && time.Duration(frames)*10*time.Millisecond >= waitForSpeech { break } audio, err := live.nextContext(ctx) if err != nil { return err } if audio.muted || !unmuted() { return errMuted } s := audio.pcm level := rms(s) peakRMS = math.Max(peakRMS, level) voiceStart := false if level >= threshold { voiced++ // Short commands after activation can have quiet gaps between // syllables. Require two voiced frames within 80 ms or a strong // frame outside the immediate tail of the wake word. voiceStart = voiced >= 3 || (wakeAudio != nil || followUp > 0) && (frames-lastVoice <= 8 || frames >= 3 && level >= 2*threshold) lastVoice = frames } else { voiced = 0 } if !heard { pre = append(pre, s...) if len(pre) > prerollSamples { copy(pre, pre[len(pre)-prerollSamples:]) pre = pre[:prerollSamples] } if !voiceStart { continue } duck() _ = c.SetWriteDeadline(time.Now().Add(2 * time.Second)) if e = c.WriteJSON(message{Type: "start", Wake: wakeAudio != nil || followUp > 0}); e != nil { return e } heard = true s = pre // After a deliberate pause, send the command onset without // any remnant of the wake word. if frames+1-voiced >= 22 { s = pre[len(pre)-min(len(pre), rate/5):] } } _ = c.SetWriteDeadline(time.Now().Add(2 * time.Second)) if e = c.WriteMessage(websocket.BinaryMessage, pcm(s)); e != nil { return e } sent += len(s) if frames-lastVoice >= int(silence/(10*time.Millisecond)) { break } } log.Printf("capture: %d ms streamed, peak frame RMS %.0f", sent*1000/rate, peakRMS) if !unmuted() { return errMuted } if !heard { return errNoSpeech } } _ = c.SetWriteDeadline(time.Now().Add(15 * time.Second)) if e = c.WriteJSON(message{Type: "end"}); e != nil { return e } lights.set("processing") var toolWrites sync.Mutex var player *exec.Cmd var w io.WriteCloser defer func() { if player != nil { w.Close() player.Process.Kill() player.Wait() } }() _ = c.SetReadDeadline(time.Now().Add(90 * time.Second)) for { typ, data, e := c.ReadMessage() if e != nil { return e } if typ == websocket.BinaryMessage { if player == nil { cmd := playbackCommand(ctx, playback) w, e = cmd.StdinPipe() if e != nil { return e } if e = cmd.Start(); e != nil { w.Close() return e } player = cmd } lights.set("speaking") if _, e = w.Write(data); e != nil { return e } continue } if typ == websocket.TextMessage { var m message if e = json.Unmarshal(data, &m); e != nil { return e } switch m.Type { case "device_tool": var req deviceTool if e = json.Unmarshal(data, &req); e != nil { return e } go func(req deviceTool) { result := runDeviceTool(ctx, req) toolWrites.Lock() defer toolWrites.Unlock() if ctx.Err() != nil { return } _ = c.SetWriteDeadline(time.Now().Add(2 * time.Second)) if err := c.WriteJSON(map[string]interface{}{"type": "device_tool_result", "id": req.ID, "result": result}); err != nil { c.Close() } }(req) case "done": if player == nil { return nil } if e = w.Close(); e != nil { return e } e = player.Wait() player = nil if options != nil && e == nil { options.spoken = true } return e case "error": return fmt.Errorf("server error: %s", m.Code) } } } } func main() { mode := flag.String("mode", "ptt", "ptt, enroll, wake, or file") muteGPIO := flag.String("mute-gpio", "/sys/class/gpio/gpio414/value", "physical microphone mute value, 1 = unmuted") input := flag.String("input", "", "16 kHz mono PCM16LE file for file mode (max 30 seconds)") device := flag.String("capture", "hw:0,1", "ALSA capture device") vqeHelper := flag.String("vqe-helper", "", "Mini 2 firmware VQE capture processor") spotterModel := flag.String("vqe-wake-model", "", "firmware spotter model directory") playback := flag.String("playback", "default", "ALSA playback device") url := flag.String("url", "wss://alice.pepega.systems/v1/voice", "server websocket URL") keywordModel := flag.String("wake-model", "", "optional local microWakeWord model path") keywordHelper := flag.String("wake-helper", "/data/opt/alice-backend/wake-helper.sh", "local keyword inference executable") path := flag.String("templates", "wake-templates.json", "local enrollment file") th := flag.Float64("voice-threshold", 350, "speech RMS threshold") match := flag.Float64("match-threshold", 1.3, "experimental DTW maximum distance") oneTurn := flag.Bool("once", false, "exit wake mode after one successful response") count := flag.Int("samples", 4, "enrollment samples") max := flag.Duration("max", 12*time.Second, "maximum request duration") silence := flag.Duration("silence", 1200*time.Millisecond, "end after speech silence") flag.Parse() if *spotterModel != "" && (*mode != "wake" || *vqeHelper == "" || *keywordModel != "") { log.Fatal("-vqe-wake-model requires -mode wake and -vqe-helper, without -wake-model") } mutePath = *muteGPIO lights.start() defer lights.stop() if *th <= 0 || *max <= 0 || *silence <= 0 || *count < 1 || *count > 10 { log.Fatal("invalid threshold, duration, or sample count") } token := strings.TrimSpace(os.Getenv("DEVICE_TOKEN")) if *mode != "enroll" && token == "" { log.Fatal("DEVICE_TOKEN is required") } var ts []template var e error if *mode == "wake" { if *keywordModel == "" && *spotterModel == "" { ts, e = loadTemplates(*path) if e != nil { log.Fatal(e) } } } else if *mode != "ptt" && *mode != "enroll" && *mode != "file" { log.Fatal("unknown mode") } if *mode == "file" { f, e := os.Open(*input) if e != nil { log.Fatal(e) } b, e := io.ReadAll(io.LimitReader(f, 30*rate*2+1)) f.Close() if e != nil { log.Fatal(e) } if len(b) == 0 || len(b) > 30*rate*2 || len(b)%2 != 0 { log.Fatal("input must contain at most 30 seconds of PCM16LE and have an even nonzero size") } if e = exchange(context.Background(), *url, token, nil, b, nil, *th, *max, *silence, *playback, 0, nil); e != nil { log.Fatal(e) } return } if *mode == "ptt" { fmt.Println("Press Enter to start speaking.") if _, e = bufio.NewReader(os.Stdin).ReadString('\n'); e != nil { log.Fatal(e) } } dialog := &conversation{commands: make(chan byte, 4), panelWake: make(chan struct{}, 1)} go dialog.localCommands() if *mode == "wake" { go reportPlayback(*url, token) } live, stopCapture, e := capture(*device, *vqeHelper, *spotterModel, dialog.command) if e != nil { log.Fatal(e) } var once sync.Once cleanup := func() { once.Do(stopCapture) } defer cleanup() sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) go func() { <-sig cleanup() _ = musicCall("duck", json.RawMessage(`{"active":false}`)) lights.stop() os.Exit(0) }() if *mode == "enroll" { if e = enroll(live, *path, *count, *th); e != nil { cleanup() log.Fatal(e) } return } if *mode == "ptt" { for { for !unmuted() { time.Sleep(100 * time.Millisecond) } e = exchange(context.Background(), *url, token, live, nil, nil, *th, *max, *silence, *playback, 0, nil) if errors.Is(e, errMuted) { log.Println("microphone muted; waiting for unmute") continue } if e != nil { cleanup() log.Fatal(e) } return } } go listenPanel(*playback, dialog) if *keywordModel != "" || *spotterModel != "" { var detector *keywordDetector if *keywordModel != "" { detector, e = startKeywordDetector(*keywordHelper, *keywordModel) if e != nil { cleanup() log.Fatalf("keyword model: %v", e) } defer detector.close() } err := runKeywordWake(live, detector, dialog.panelWake, func(preroll []int16) error { return dialog.dialogue(*url, token, live, preroll, *th, *max, *silence, *playback, *oneTurn) }, *oneTurn) if err != nil { cleanup() log.Fatalf("keyword detector: %v", err) } return } // Detect whole speech segments locally before opening a voice connection. log.Println("Listening locally for enrolled phrase") var samples []int16 var f [][]float64 active := false quiet := 0 for { audio, e := live.next() if e != nil { cleanup() log.Fatal(e) } if !unmuted() { samples = nil f = nil active = false quiet = 0 continue } samples = append(samples, audio.pcm...) if len(samples) < frameSamples { continue } v := rms(samples[:frameSamples]) frame := samples[:frameSamples] if v >= *th { active = true quiet = 0 } samples = samples[hop:] if active { f = append(f, features(frame)) if v < *th { quiet++ } if quiet >= 35 || len(f) >= 250 { best := math.Inf(1) for _, t := range ts { // Enrollment and wake windows both include the final pause. if len(f)*3 < len(t.Frames)*2 || len(f)*2 > len(t.Frames)*3 { continue } if d := distance(f, t.Frames); d < best { best = d } } log.Printf("wake candidate distance=%.3f", best) if best <= *match { log.Println("wake detected") if e = exchange(context.Background(), *url, token, live, nil, nil, *th, *max, *silence, *playback, 0, nil); e != nil { log.Printf("session: %v", e) if !errors.Is(e, errMuted) && !errors.Is(e, errNoSpeech) { lights.flashError() } } else if *oneTurn { return } } live.discardPending() f = nil active = false quiet = 0 samples = nil } } } }