613 lines
15 KiB
Go
613 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"math"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/gorilla/websocket"
|
|
)
|
|
|
|
const rate = 16000
|
|
const frameSamples = 320
|
|
const hop = 160
|
|
|
|
type template struct {
|
|
Frames [][]float64 `json:"frames"`
|
|
}
|
|
type message struct {
|
|
Type string `json:"type"`
|
|
Code string `json:"code,omitempty"`
|
|
Wake bool `json:"wake,omitempty"`
|
|
}
|
|
|
|
var hann [frameSamples]float64
|
|
var coefficients [20]float64
|
|
|
|
func init() {
|
|
for i := range hann {
|
|
hann[i] = 0.5 - 0.5*math.Cos(2*math.Pi*float64(i)/float64(frameSamples-1))
|
|
}
|
|
for b := range coefficients {
|
|
freq := 150 * math.Pow(7000.0/150.0, float64(b)/19)
|
|
coefficients[b] = 2 * math.Cos(2*math.Pi*freq/rate)
|
|
}
|
|
}
|
|
|
|
// features calculates log spectral energy in coarse bands, normalized against overall level.
|
|
// It is a small experimental template matcher, not a production speech model.
|
|
func features(samples []int16) []float64 {
|
|
const bands = 20
|
|
out := make([]float64, bands)
|
|
for b := 0; b < bands; b++ {
|
|
coeff := coefficients[b]
|
|
var a, c float64
|
|
for i, s := range samples {
|
|
v := float64(s)*hann[i] + coeff*a - c
|
|
c = a
|
|
a = v
|
|
}
|
|
power := a*a + c*c - coeff*a*c
|
|
out[b] = math.Log(1 + math.Max(0, power))
|
|
}
|
|
var mean float64
|
|
for _, v := range out {
|
|
mean += v
|
|
}
|
|
mean /= bands
|
|
for i := range out {
|
|
out[i] -= mean
|
|
}
|
|
return out
|
|
}
|
|
func rms(s []int16) float64 {
|
|
var v float64
|
|
for _, x := range s {
|
|
v += float64(x) * float64(x)
|
|
}
|
|
return math.Sqrt(v / float64(len(s)))
|
|
}
|
|
func distance(a, b [][]float64) float64 {
|
|
if len(a) == 0 || len(b) == 0 {
|
|
return math.Inf(1)
|
|
}
|
|
prev := make([]float64, len(b)+1)
|
|
for i := 1; i < len(prev); i++ {
|
|
prev[i] = math.Inf(1)
|
|
}
|
|
for _, x := range a {
|
|
cur := make([]float64, len(b)+1)
|
|
cur[0] = math.Inf(1)
|
|
for j, y := range b {
|
|
var cost float64
|
|
for k := range x {
|
|
d := x[k] - y[k]
|
|
cost += d * d
|
|
}
|
|
cost = math.Sqrt(cost / float64(len(x)))
|
|
cur[j+1] = cost + math.Min(prev[j], math.Min(prev[j+1], cur[j]))
|
|
}
|
|
prev = cur
|
|
}
|
|
return prev[len(b)] / float64(len(a)+len(b))
|
|
}
|
|
func loadTemplates(path string) ([]template, error) {
|
|
f, e := os.Open(path)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer f.Close()
|
|
b, e := io.ReadAll(io.LimitReader(f, 2<<20+1))
|
|
if len(b) > 2<<20 {
|
|
return nil, errors.New("template file too large")
|
|
}
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
var t []template
|
|
e = json.Unmarshal(b, &t)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if len(t) == 0 || len(t) > 10 {
|
|
return nil, errors.New("no templates")
|
|
}
|
|
for _, x := range t {
|
|
if len(x.Frames) == 0 || len(x.Frames) > 400 {
|
|
return nil, errors.New("empty template")
|
|
}
|
|
for _, f := range x.Frames {
|
|
if len(f) != 20 {
|
|
return nil, errors.New("invalid feature dimensions")
|
|
}
|
|
}
|
|
}
|
|
return t, nil
|
|
}
|
|
|
|
func waitRecord(r *liveReader, max time.Duration, silence time.Duration, threshold float64) ([][]float64, error) {
|
|
var samples []int16
|
|
var frames [][]float64
|
|
start := time.Now()
|
|
last := start
|
|
heard := false
|
|
for time.Since(start) < max {
|
|
s, e := r.next()
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if !unmuted() {
|
|
samples = nil
|
|
frames = nil
|
|
heard = false
|
|
last = time.Now()
|
|
start = time.Now()
|
|
continue
|
|
}
|
|
samples = append(samples, s...)
|
|
if len(samples) >= frameSamples {
|
|
frame := samples[:frameSamples]
|
|
if rms(frame) >= threshold {
|
|
heard = true
|
|
last = time.Now()
|
|
}
|
|
if heard {
|
|
frames = append(frames, features(frame))
|
|
}
|
|
samples = samples[hop:]
|
|
}
|
|
if len(frames) >= 250 {
|
|
return frames, nil
|
|
}
|
|
if heard && time.Since(last) >= silence {
|
|
return frames, nil
|
|
}
|
|
}
|
|
if !heard {
|
|
return nil, errors.New("no speech detected")
|
|
}
|
|
return frames, nil
|
|
}
|
|
func enroll(r *liveReader, path string, count int, threshold float64) error {
|
|
var all []template
|
|
in := bufio.NewReader(os.Stdin)
|
|
for i := 0; i < count; i++ {
|
|
fmt.Printf("Sample %d/%d: press Enter, then say the wake phrase.\n", i+1, count)
|
|
if _, e := in.ReadString('\n'); e != nil {
|
|
return e
|
|
}
|
|
for !unmuted() {
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
lights.set("listening")
|
|
r.discardPending()
|
|
f, e := waitRecord(r, 4*time.Second, 350*time.Millisecond, threshold)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
lights.set("idle")
|
|
all = append(all, template{f})
|
|
fmt.Printf("Recorded %d frames.\n", len(f))
|
|
}
|
|
b, e := json.Marshal(all)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
return os.WriteFile(path, b, 0600)
|
|
}
|
|
func connect(url, token string) (*websocket.Conn, error) {
|
|
h := http.Header{}
|
|
h.Set("Authorization", "Bearer "+token)
|
|
d := websocket.Dialer{HandshakeTimeout: 2 * time.Second, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}
|
|
c, resp, e := d.Dial(url, h)
|
|
if e != nil {
|
|
if resp != nil {
|
|
return nil, fmt.Errorf("websocket handshake HTTP %d: %w", resp.StatusCode, e)
|
|
}
|
|
return nil, e
|
|
}
|
|
c.SetReadLimit(4 << 20)
|
|
_ = c.SetReadDeadline(time.Now().Add(750 * time.Millisecond))
|
|
var m message
|
|
if e = c.ReadJSON(&m); e != nil {
|
|
c.Close()
|
|
return nil, e
|
|
}
|
|
if m.Type != "ready" {
|
|
c.Close()
|
|
return nil, fmt.Errorf("expected ready, got %q", m.Type)
|
|
}
|
|
_ = c.SetReadDeadline(time.Time{})
|
|
return c, nil
|
|
}
|
|
func playbackCommand(ctx context.Context, device string) *exec.Cmd {
|
|
player := exec.CommandContext(ctx, "aplay", "-q", "-D", device, "-t", "raw", "-f", "S16_LE", "-r", "24000", "-c", "1")
|
|
if path := os.Getenv("ALICE_PLAYER"); path != "" {
|
|
player = exec.CommandContext(ctx, path)
|
|
}
|
|
player.Stderr = os.Stderr
|
|
terminateWithClient(player)
|
|
return player
|
|
}
|
|
|
|
func exchange(url, token string, live *liveReader, fileAudio []byte, wakeAudio []int16, threshold float64, max, silence time.Duration, playback string) error {
|
|
if live != nil {
|
|
if !unmuted() {
|
|
return errMuted
|
|
}
|
|
lights.set("listening")
|
|
_ = musicCall("duck", json.RawMessage(`{"active":true}`))
|
|
defer func() { _ = musicCall("duck", json.RawMessage(`{"active":false}`)); lights.set("idle") }()
|
|
if wakeAudio == nil {
|
|
live.discardPending()
|
|
}
|
|
} else {
|
|
defer lights.set("idle")
|
|
}
|
|
c, e := connect(url, token)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
defer c.Close()
|
|
if fileAudio == nil && !unmuted() {
|
|
return errMuted
|
|
}
|
|
_ = c.SetWriteDeadline(time.Now().Add(15 * time.Second))
|
|
if fileAudio != nil {
|
|
if e = c.WriteJSON(message{Type: "start"}); e != nil {
|
|
return e
|
|
}
|
|
for len(fileAudio) > 0 {
|
|
n := 3200
|
|
if n > len(fileAudio) {
|
|
n = len(fileAudio)
|
|
}
|
|
_ = c.SetWriteDeadline(time.Now().Add(15 * time.Second))
|
|
if e = c.WriteMessage(websocket.BinaryMessage, fileAudio[:n]); e != nil {
|
|
return e
|
|
}
|
|
fileAudio = fileAudio[n:]
|
|
}
|
|
} else {
|
|
// Preserve the utterance across activation, including commands spoken
|
|
// together with the wake word. Count PCM frames, independent of TLS timing.
|
|
const prerollSamples = rate
|
|
pre := make([]int16, 0, prerollSamples+hop)
|
|
pre = append(pre, wakeAudio...)
|
|
var frames, lastVoice, voiced, sent int
|
|
var peakRMS float64
|
|
heard := false
|
|
for ; frames < int(max/(10*time.Millisecond)); frames++ {
|
|
if !heard && frames >= 400 {
|
|
break
|
|
}
|
|
s, err := live.next()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !unmuted() {
|
|
return errMuted
|
|
}
|
|
level := rms(s)
|
|
peakRMS = math.Max(peakRMS, level)
|
|
if level >= threshold {
|
|
voiced++
|
|
lastVoice = frames
|
|
} else {
|
|
voiced = 0
|
|
}
|
|
if !heard {
|
|
pre = append(pre, s...)
|
|
if len(pre) > prerollSamples {
|
|
copy(pre, pre[len(pre)-prerollSamples:])
|
|
pre = pre[:prerollSamples]
|
|
}
|
|
if voiced < 3 {
|
|
continue
|
|
}
|
|
_ = c.SetWriteDeadline(time.Now().Add(2 * time.Second))
|
|
if e = c.WriteJSON(message{Type: "start", Wake: wakeAudio != nil}); e != nil {
|
|
return e
|
|
}
|
|
heard = true
|
|
s = pre
|
|
// After a deliberate pause, send the command onset without
|
|
// any remnant of the wake word.
|
|
if frames+1-voiced >= 22 {
|
|
s = pre[len(pre)-min(len(pre), rate/5):]
|
|
}
|
|
}
|
|
_ = c.SetWriteDeadline(time.Now().Add(2 * time.Second))
|
|
if e = c.WriteMessage(websocket.BinaryMessage, pcm(s)); e != nil {
|
|
return e
|
|
}
|
|
sent += len(s)
|
|
if frames-lastVoice >= int(silence/(10*time.Millisecond)) {
|
|
break
|
|
}
|
|
}
|
|
log.Printf("capture: %d ms streamed, peak frame RMS %.0f", sent*1000/rate, peakRMS)
|
|
if !unmuted() {
|
|
return errMuted
|
|
}
|
|
if !heard {
|
|
return errNoSpeech
|
|
}
|
|
}
|
|
_ = c.SetWriteDeadline(time.Now().Add(15 * time.Second))
|
|
if e = c.WriteJSON(message{Type: "end"}); e != nil {
|
|
return e
|
|
}
|
|
lights.set("processing")
|
|
var player *exec.Cmd
|
|
var w io.WriteCloser
|
|
defer func() {
|
|
if player != nil {
|
|
w.Close()
|
|
player.Process.Kill()
|
|
player.Wait()
|
|
}
|
|
}()
|
|
_ = c.SetReadDeadline(time.Now().Add(90 * time.Second))
|
|
for {
|
|
typ, data, e := c.ReadMessage()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if typ == websocket.BinaryMessage {
|
|
if player == nil {
|
|
cmd := playbackCommand(context.Background(), playback)
|
|
w, e = cmd.StdinPipe()
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if e = cmd.Start(); e != nil {
|
|
w.Close()
|
|
return e
|
|
}
|
|
player = cmd
|
|
}
|
|
lights.set("speaking")
|
|
if _, e = w.Write(data); e != nil {
|
|
return e
|
|
}
|
|
continue
|
|
}
|
|
if typ == websocket.TextMessage {
|
|
var m message
|
|
if e = json.Unmarshal(data, &m); e != nil {
|
|
return e
|
|
}
|
|
switch m.Type {
|
|
case "device_tool":
|
|
var req deviceTool
|
|
if e = json.Unmarshal(data, &req); e != nil {
|
|
return e
|
|
}
|
|
result := runDeviceTool(req)
|
|
_ = c.SetWriteDeadline(time.Now().Add(15 * time.Second))
|
|
if e = c.WriteJSON(map[string]interface{}{"type": "device_tool_result", "id": req.ID, "result": result}); e != nil {
|
|
return e
|
|
}
|
|
case "done":
|
|
if player == nil {
|
|
return nil
|
|
}
|
|
if e = w.Close(); e != nil {
|
|
return e
|
|
}
|
|
e = player.Wait()
|
|
player = nil
|
|
return e
|
|
case "error":
|
|
return fmt.Errorf("server error: %s", m.Code)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
func main() {
|
|
mode := flag.String("mode", "ptt", "ptt, enroll, wake, or file")
|
|
muteGPIO := flag.String("mute-gpio", "/sys/class/gpio/gpio414/value", "physical microphone mute value, 1 = unmuted")
|
|
input := flag.String("input", "", "16 kHz mono PCM16LE file for file mode (max 30 seconds)")
|
|
device := flag.String("capture", "hw:0,1", "ALSA capture device")
|
|
vqeHelper := flag.String("vqe-helper", "", "Mini 2 firmware VQE capture processor")
|
|
playback := flag.String("playback", "default", "ALSA playback device")
|
|
url := flag.String("url", "wss://alice.pepega.systems/v1/voice", "server websocket URL")
|
|
keywordModel := flag.String("wake-model", "", "optional local microWakeWord model path")
|
|
keywordHelper := flag.String("wake-helper", "/data/opt/alice-backend/wake-helper.sh", "local keyword inference executable")
|
|
path := flag.String("templates", "wake-templates.json", "local enrollment file")
|
|
th := flag.Float64("voice-threshold", 350, "speech RMS threshold")
|
|
match := flag.Float64("match-threshold", 1.3, "experimental DTW maximum distance")
|
|
oneTurn := flag.Bool("once", false, "exit wake mode after one successful response")
|
|
count := flag.Int("samples", 4, "enrollment samples")
|
|
max := flag.Duration("max", 12*time.Second, "maximum request duration")
|
|
silence := flag.Duration("silence", 1200*time.Millisecond, "end after speech silence")
|
|
flag.Parse()
|
|
mutePath = *muteGPIO
|
|
lights.start()
|
|
defer lights.stop()
|
|
if *th <= 0 || *max <= 0 || *silence <= 0 || *count < 1 || *count > 10 {
|
|
log.Fatal("invalid threshold, duration, or sample count")
|
|
}
|
|
token := strings.TrimSpace(os.Getenv("DEVICE_TOKEN"))
|
|
if *mode != "enroll" && token == "" {
|
|
log.Fatal("DEVICE_TOKEN is required")
|
|
}
|
|
var ts []template
|
|
var e error
|
|
if *mode == "wake" {
|
|
if *keywordModel == "" {
|
|
ts, e = loadTemplates(*path)
|
|
if e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
}
|
|
} else if *mode != "ptt" && *mode != "enroll" && *mode != "file" {
|
|
log.Fatal("unknown mode")
|
|
}
|
|
if *mode == "file" {
|
|
f, e := os.Open(*input)
|
|
if e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
b, e := io.ReadAll(io.LimitReader(f, 30*rate*2+1))
|
|
f.Close()
|
|
if e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
if len(b) == 0 || len(b) > 30*rate*2 || len(b)%2 != 0 {
|
|
log.Fatal("input must contain at most 30 seconds of PCM16LE and have an even nonzero size")
|
|
}
|
|
if e = exchange(*url, token, nil, b, nil, *th, *max, *silence, *playback); e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
return
|
|
}
|
|
if *mode == "ptt" {
|
|
fmt.Println("Press Enter to start speaking.")
|
|
if _, e = bufio.NewReader(os.Stdin).ReadString('\n'); e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
}
|
|
live, stopCapture, e := capture(*device, *vqeHelper)
|
|
if e != nil {
|
|
log.Fatal(e)
|
|
}
|
|
var once sync.Once
|
|
cleanup := func() { once.Do(stopCapture) }
|
|
defer cleanup()
|
|
sig := make(chan os.Signal, 1)
|
|
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
|
go func() {
|
|
<-sig
|
|
cleanup()
|
|
_ = musicCall("duck", json.RawMessage(`{"active":false}`))
|
|
lights.stop()
|
|
os.Exit(0)
|
|
}()
|
|
if *mode == "enroll" {
|
|
if e = enroll(live, *path, *count, *th); e != nil {
|
|
cleanup()
|
|
log.Fatal(e)
|
|
}
|
|
return
|
|
}
|
|
if *mode == "ptt" {
|
|
for {
|
|
for !unmuted() {
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
e = exchange(*url, token, live, nil, nil, *th, *max, *silence, *playback)
|
|
if errors.Is(e, errMuted) {
|
|
log.Println("microphone muted; waiting for unmute")
|
|
continue
|
|
}
|
|
if e != nil {
|
|
cleanup()
|
|
log.Fatal(e)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
go listenVolumeButtons(*playback)
|
|
if *keywordModel != "" {
|
|
detector, err := startKeywordDetector(*keywordHelper, *keywordModel)
|
|
if err != nil {
|
|
cleanup()
|
|
log.Fatalf("keyword model: %v", err)
|
|
}
|
|
err = runKeywordWake(live, detector, func(preroll []int16) error {
|
|
return exchange(*url, token, live, nil, preroll, *th, *max, *silence, *playback)
|
|
}, *oneTurn)
|
|
detector.close()
|
|
if err != nil {
|
|
cleanup()
|
|
log.Fatalf("keyword detector: %v", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
// Detect whole speech segments locally before opening a voice connection.
|
|
log.Println("Listening locally for enrolled phrase")
|
|
var samples []int16
|
|
var f [][]float64
|
|
active := false
|
|
quiet := 0
|
|
for {
|
|
s, e := live.next()
|
|
if e != nil {
|
|
cleanup()
|
|
log.Fatal(e)
|
|
}
|
|
if !unmuted() {
|
|
samples = nil
|
|
f = nil
|
|
active = false
|
|
quiet = 0
|
|
continue
|
|
}
|
|
samples = append(samples, s...)
|
|
if len(samples) < frameSamples {
|
|
continue
|
|
}
|
|
v := rms(samples[:frameSamples])
|
|
frame := samples[:frameSamples]
|
|
if v >= *th {
|
|
active = true
|
|
quiet = 0
|
|
}
|
|
samples = samples[hop:]
|
|
if active {
|
|
f = append(f, features(frame))
|
|
if v < *th {
|
|
quiet++
|
|
}
|
|
if quiet >= 35 || len(f) >= 250 {
|
|
best := math.Inf(1)
|
|
for _, t := range ts {
|
|
// Enrollment and wake windows both include the final pause.
|
|
if len(f)*3 < len(t.Frames)*2 || len(f)*2 > len(t.Frames)*3 {
|
|
continue
|
|
}
|
|
if d := distance(f, t.Frames); d < best {
|
|
best = d
|
|
}
|
|
}
|
|
log.Printf("wake candidate distance=%.3f", best)
|
|
if best <= *match {
|
|
log.Println("wake detected")
|
|
if e = exchange(*url, token, live, nil, nil, *th, *max, *silence, *playback); e != nil {
|
|
log.Printf("session: %v", e)
|
|
if !errors.Is(e, errMuted) && !errors.Is(e, errNoSpeech) {
|
|
lights.flashError()
|
|
}
|
|
} else if *oneTurn {
|
|
return
|
|
}
|
|
}
|
|
live.discardPending()
|
|
f = nil
|
|
active = false
|
|
quiet = 0
|
|
samples = nil
|
|
}
|
|
}
|
|
}
|
|
}
|