Control Windows audio streaming from Alice

This commit is contained in:
2026-11-14 15:26:40 +03:00
parent 44d191831f
commit 3e83d25410
12 changed files with 536 additions and 1 deletions
+11
View File
@@ -16,6 +16,17 @@ function volumeLevel(text) {
export function quickAction(transcript) {
const text = transcript.toLowerCase().replace(/[.,!?;:…\s]+/gu, ' ').trim()
.replace(/^алиса\s+/, '').replace(/^пожалуйста\s+|\s+пожалуйста$/g, '');
const pc = /^(включи|запусти|выключи|отключи|останови) (?:звук|трансляцию(?: звука)?) (?:с |из )?(?:пк|компьютера|компа)(?: на колонку| на колонке)?$/u.exec(text);
if (pc) {
const enabled = pc[1] === 'включи' || pc[1] === 'запусти';
return {
name: 'pc_audio',
args: { action: enabled ? 'start' : 'stop' },
done: enabled ? 'Звук с компьютера включён.' : 'Звук с компьютера выключен.',
failure: 'Не удалось переключить звук с компьютера. Проверь агент на Windows.',
success: result => result?.ok === true && result.enabled === enabled,
};
}
if (/^(?:пауза|на паузу|(?:поставь|поставить)(?: музыку| воспроизведение)? на паузу|(?:приостанови|выключи|останови) (?:музыку|воспроизведение))$/.test(text)) {
return {
name: 'spotify_control',
+3
View File
@@ -11,6 +11,7 @@ const actions = {
export const tools = [
{ type: 'openrouter:web_search' },
{ type: 'function', function: { name: 'pc_audio', description: 'Start, stop or check the Windows PC audio mirror on the speaker. Use for explicit requests about sound from the PC, independently of Spotify.', parameters: { type: 'object', properties: { action: { type: 'string', enum: ['start', 'stop', 'status'] } }, required: ['action'], additionalProperties: false } } },
{ type: 'function', function: { name: 'spotify_play', description: 'Play a track, artist, album or playlist on Spotify. Pass type when the user names a kind of content. Normalize known transliterations.', parameters: { type: 'object', properties: { query: { type: 'string', maxLength: 500 }, uri: { type: 'string', maxLength: 500 }, type: { type: 'string', enum: spotifyTypes } }, additionalProperties: false } } },
{ type: 'function', function: { name: 'spotify_control', description: 'Control Spotify on the station. Use status for what is playing, queue to read upcoming tracks, enqueue to add a track by name, seek with an absolute position in milliseconds, and repeat with track, context (album or playlist), or off.', parameters: { type: 'object', properties: { action: { type: 'string', enum: Object.keys(actions) }, ...Object.assign({}, ...Object.values(actions)) }, required: ['action'], additionalProperties: false } } },
{ type: 'function', function: { name: 'spotify_playlist', description: 'List own playlists, read tracks, create a private playlist, or add/remove a track. Use playlistUri for an existing playlist, trackUri or trackQuery for a track, and name when creating.', parameters: { type: 'object', properties: {
@@ -44,6 +45,8 @@ export function validateTool(name, args) {
if (args.playlistUri && !/^spotify:playlist:[A-Za-z0-9]{22}$/.test(args.playlistUri)) throw new Error('invalid_arguments');
if (args.trackUri && !/^spotify:track:[A-Za-z0-9]{22}$/.test(args.trackUri)) throw new Error('invalid_arguments');
if (['add', 'remove'].includes(args.action) && !args.trackUri && !args.trackQuery) throw new Error('invalid_arguments');
} else if (name === 'pc_audio') {
if (keys.length !== 1 || !['start', 'stop', 'status'].includes(args.action)) throw new Error('invalid_arguments');
} else if (name === 'station_volume') {
if (keys.length !== 1 || ('delta' in args ? ![-10, 10].includes(args.delta) : !Number.isInteger(args.level) || args.level < 0 || args.level > 100)) throw new Error('invalid_arguments');
} else throw new Error('unknown_tool');
+14
View File
@@ -101,3 +101,17 @@ CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -trimpath -ldflags='-s -w' -o scr
```
Приёмник пишет в журнал формат при первом пакете и освобождает вход PulseAudio через две секунды без звука. Базовый запрос к буферу PulseAudio составляет 30 мс. Полная задержка также зависит от Windows, сети и буфера аппаратного выхода. Остановить приёмник в терминале можно через Ctrl+C.
### Голосовое управление с Windows
Команды «Алиса, включи звук с компьютера» и «Алиса, выключи звук с компьютера» управляют отправителем. Агент работает в сеансе пользователя Windows и запускает Scream Sender по команде. При выходе из агента или потере связи со станцией отправитель завершается.
На Mini 2 положи в `/data/opt/alice-backend` файл `pc-audio-receiver.conf` с адресом приёмника `192.168.1.135:4010` на первой строке и IP компьютера `192.168.1.170` на второй. Соседний файл `pc-audio.json` содержит поля `listen` (адрес станции с TCP-портом, например `192.168.1.135:4011`), `peer` (IP компьютера) и `token` (случайные 32 байта в hex). Поставь файлам права `0600` и перезапусти `S99alice`.
Из каталога `station` собери агент командой:
```sh
CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -trimpath -ldflags='-s -w -H windowsgui' -o AlicePCAudio.exe ./cmd/pc-audio-agent
```
Положи рядом с EXE файл `ScreamSender.exe` и `agent.json` с полями `station` (TCP-адрес из `listen`), `port` (UDP-порт приёмника) и `token` (тот же ключ). Запусти `AlicePCAudio.exe` из сеанса Windows. Для автозапуска добавь ярлык в `shell:startup`. Журнал агента находится рядом с EXE в файле `agent.log`. Если отправитель ранее запускался вручную, закрой этот экземпляр перед первым запуском агента.
+209
View File
@@ -0,0 +1,209 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/netip"
"os"
"path/filepath"
"strconv"
"time"
"unsafe"
"alice-back/station/pcaudio"
"golang.org/x/sys/windows"
)
const agentLease = 5 * time.Second
type sender struct {
process windows.Handle
job windows.Handle
}
func startSender(path, host string, port uint16) (*sender, error) {
job, err := windows.CreateJobObject(nil, nil)
if err != nil {
return nil, err
}
var limits windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
windows.CloseHandle(job)
return nil, err
}
program, _ := windows.UTF16PtrFromString(path)
args, _ := windows.UTF16PtrFromString(windows.ComposeCommandLine([]string{path, host, strconv.Itoa(int(port))}))
dir, _ := windows.UTF16PtrFromString(filepath.Dir(path))
info := windows.StartupInfo{Cb: uint32(unsafe.Sizeof(windows.StartupInfo{}))}
var child windows.ProcessInformation
if err := windows.CreateProcess(program, args, nil, nil, false, windows.CREATE_SUSPENDED|windows.CREATE_NO_WINDOW, nil, dir, &info, &child); err != nil {
windows.CloseHandle(job)
return nil, err
}
defer windows.CloseHandle(child.Thread)
err = windows.AssignProcessToJobObject(job, child.Process)
if err == nil {
_, err = windows.ResumeThread(child.Thread)
}
if err != nil {
_ = windows.TerminateProcess(child.Process, 1)
windows.CloseHandle(child.Process)
windows.CloseHandle(job)
return nil, err
}
return &sender{process: child.Process, job: job}, nil
}
func (s *sender) stop() {
windows.CloseHandle(s.job)
_, _ = windows.WaitForSingleObject(s.process, 1000)
windows.CloseHandle(s.process)
}
func (s *sender) exited() bool {
status, err := windows.WaitForSingleObject(s.process, 0)
return err != nil || status != uint32(windows.WAIT_TIMEOUT)
}
func poll(client *http.Client, url, token string, state pcaudio.State) (pcaudio.Command, error) {
data, _ := json.Marshal(state)
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(data))
if err != nil {
return pcaudio.Command{}, err
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return pcaudio.Command{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return pcaudio.Command{}, fmt.Errorf("station returned HTTP %d", resp.StatusCode)
}
var command pcaudio.Command
err = json.NewDecoder(io.LimitReader(resp.Body, 1024)).Decode(&command)
if err == nil && len(command.Revision) != 32 {
err = errors.New("invalid station command")
}
return command, err
}
func run() error {
exe, err := os.Executable()
if err != nil {
return err
}
dir := filepath.Dir(exe)
path := filepath.Join(dir, "ScreamSender.exe")
if _, err := os.Stat(path); err != nil {
return err
}
data, err := os.ReadFile(filepath.Join(dir, "agent.json"))
if err != nil {
return err
}
var config struct {
Station string
Port uint16
Token string
}
if err := json.Unmarshal(data, &config); err != nil {
return err
}
host, err := netip.ParseAddrPort(config.Station)
if err != nil || !host.Addr().Is4() || config.Port == 0 || len(config.Token) != 64 {
return errors.New("invalid agent configuration")
}
name, _ := windows.UTF16PtrFromString("Local\\AlicePCAudio")
mutex, err := windows.CreateMutex(nil, false, name)
if mutex != 0 {
defer windows.CloseHandle(mutex)
}
if errors.Is(err, windows.ERROR_ALREADY_EXISTS) {
return nil
}
if err != nil {
return err
}
client := &http.Client{
Timeout: 900 * time.Millisecond,
Transport: &http.Transport{DialContext: (&net.Dialer{}).DialContext},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
url := "http://" + host.String() + pcaudio.Path
var state pcaudio.State
var child *sender
var lastSuccess time.Time
var previousError string
for {
if child != nil && child.exited() {
child.stop()
child = nil
state.Enabled = false
state.Error = "sender_exited"
log.Print("Scream Sender exited")
}
command, err := poll(client, url, config.Token, state)
now := time.Now()
if err != nil {
if err.Error() != previousError {
log.Printf("Station: %v", err)
previousError = err.Error()
}
if child != nil && now.Sub(lastSuccess) >= agentLease {
child.stop()
child = nil
state = pcaudio.State{}
log.Print("Station connection lost. Streaming stopped")
}
} else {
if lastSuccess.IsZero() || previousError != "" {
log.Print("Connected to Mini 2")
}
lastSuccess, previousError = now, ""
if command.Revision != state.Revision {
// Stop any sender from the previous command before applying a new one.
if child != nil && !command.Enabled {
child.stop()
child = nil
log.Print("Streaming stopped")
}
state = pcaudio.State{Command: command}
if command.Enabled && child == nil {
child, err = startSender(path, host.Addr().String(), config.Port)
if err != nil {
state.Enabled = false
state.Error = "sender_start_failed"
log.Printf("Scream Sender: %v", err)
} else {
log.Print("Streaming started")
}
}
}
}
time.Sleep(650 * time.Millisecond)
}
}
func main() {
if exe, err := os.Executable(); err == nil {
path := filepath.Join(filepath.Dir(exe), "agent.log")
if logs, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err == nil {
defer logs.Close()
log.SetOutput(logs)
}
}
if err := run(); err != nil {
log.Print(err)
os.Exit(1)
}
}
+3
View File
@@ -20,6 +20,9 @@ type deviceTool struct {
}
func runDeviceTool(ctx context.Context, req deviceTool) json.RawMessage {
if req.Name == "pc_audio" {
return pcAudioTool(ctx, req.Arguments)
}
if req.Name != "station_volume" {
return musicCallContext(ctx, req.Name, req.Arguments)
}
+2
View File
@@ -3,3 +3,5 @@ module alice-back/station
go 1.21
require github.com/gorilla/websocket v1.5.3
require golang.org/x/sys v0.30.0
+2
View File
@@ -1,2 +1,4 @@
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+1
View File
@@ -541,6 +541,7 @@ func main() {
dialog := &conversation{commands: make(chan byte, 4), panelWake: make(chan struct{}, 1)}
go dialog.localCommands()
if *mode == "wake" {
loadPCControl()
go reportPlayback(*url, token)
}
live, stopCapture, e := capture(*device, *vqeHelper, *spotterModel, dialog.command)
+178
View File
@@ -0,0 +1,178 @@
package main
import (
"context"
"crypto/rand"
"crypto/subtle"
"encoding/hex"
"encoding/json"
"log"
"net"
"net/http"
"net/netip"
"os"
"strings"
"sync"
"time"
"alice-back/station/pcaudio"
)
const pcAudioConfigPath = "/data/opt/alice-backend/pc-audio.json"
const pcAudioLease = 5 * time.Second
var windowsAudio *pcAudioControl
type pcAudioControl struct {
mu sync.Mutex
token string
peer netip.Addr
desired pcaudio.Command
actual pcaudio.State
last time.Time
}
func loadPCControl() {
data, err := os.ReadFile(pcAudioConfigPath)
if os.IsNotExist(err) {
return
}
if err != nil {
log.Printf("PC audio configuration: %v", err)
return
}
var config struct{ Listen, Peer, Token string }
if json.Unmarshal(data, &config) != nil || len(config.Token) != 64 {
log.Print("Invalid PC audio configuration")
return
}
local, localErr := netip.ParseAddrPort(config.Listen)
peer, peerErr := netip.ParseAddr(config.Peer)
if localErr != nil || peerErr != nil || !local.Addr().Is4() || local.Addr().IsUnspecified() || local.Port() == 0 || !peer.Is4() {
log.Print("Invalid addresses for PC audio control")
return
}
listener, err := net.Listen("tcp4", local.String())
if err != nil {
log.Printf("PC audio control: %v", err)
return
}
windowsAudio = &pcAudioControl{token: config.Token, peer: peer, desired: newPCCommand(false)}
server := &http.Server{Handler: windowsAudio, ReadTimeout: 2 * time.Second, WriteTimeout: 2 * time.Second, MaxHeaderBytes: 4096}
go func() {
if err := server.Serve(listener); err != nil {
log.Printf("PC audio control: %v", err)
}
}()
log.Printf("PC audio control listening on %s", listener.Addr())
}
func newPCCommand(enabled bool) pcaudio.Command {
var id [16]byte
if _, err := rand.Read(id[:]); err != nil {
panic(err)
}
return pcaudio.Command{Revision: hex.EncodeToString(id[:]), Enabled: enabled}
}
func (c *pcAudioControl) expire(now time.Time) {
if c.desired.Enabled && now.Sub(c.last) >= pcAudioLease {
c.desired = newPCCommand(false)
}
}
func (c *pcAudioControl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != pcaudio.Path {
http.NotFound(w, r)
return
}
remote, err := netip.ParseAddrPort(r.RemoteAddr)
token, bearer := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
if err != nil || !bearer || remote.Addr().Unmap() != c.peer || subtle.ConstantTimeCompare([]byte(token), []byte(c.token)) != 1 {
http.Error(w, "forbidden", http.StatusForbidden)
return
}
var state pcaudio.State
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1024)).Decode(&state); err != nil {
http.Error(w, "invalid state", http.StatusBadRequest)
return
}
c.mu.Lock()
now := time.Now()
c.expire(now)
c.last = now
c.actual = state
desired := c.desired
c.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(desired)
}
func (c *pcAudioControl) run(ctx context.Context, action string) json.RawMessage {
c.mu.Lock()
now := time.Now()
c.expire(now)
if action == "stop" && c.desired.Enabled {
c.desired = newPCCommand(false)
}
if now.Sub(c.last) >= pcAudioLease {
c.mu.Unlock()
return json.RawMessage(`{"error":"pc_agent_offline","hint":"Запусти агент Алисы на компьютере Windows."}`)
}
if action == "status" {
data, _ := json.Marshal(map[string]any{"ok": true, "enabled": c.actual.Enabled, "error": c.actual.Error})
c.mu.Unlock()
return data
}
if c.desired.Enabled != (action == "start") || c.actual.Error != "" {
c.desired = newPCCommand(action == "start")
}
command := c.desired
c.mu.Unlock()
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
c.mu.Lock()
current, actual := c.desired, c.actual
c.mu.Unlock()
if current != command {
return json.RawMessage(`{"error":"pc_audio_command_replaced"}`)
}
if actual.Revision == command.Revision {
if actual.Error != "" || actual.Enabled != command.Enabled {
c.mu.Lock()
if command.Enabled && c.desired == command {
c.desired = newPCCommand(false)
}
c.mu.Unlock()
return json.RawMessage(`{"error":"pc_audio_sender_failed"}`)
}
data, _ := json.Marshal(map[string]any{"ok": true, "enabled": actual.Enabled})
return data
}
select {
case <-ctx.Done():
c.mu.Lock()
if command.Enabled && c.desired == command {
c.desired = newPCCommand(false)
}
c.mu.Unlock()
return json.RawMessage(`{"error":"pc_audio_timeout"}`)
case <-ticker.C:
}
}
}
func pcAudioTool(ctx context.Context, arguments json.RawMessage) json.RawMessage {
var args struct{ Action string }
if json.Unmarshal(arguments, &args) != nil || (args.Action != "start" && args.Action != "stop" && args.Action != "status") {
return json.RawMessage(`{"error":"invalid_pc_audio_action"}`)
}
if windowsAudio == nil {
return json.RawMessage(`{"error":"pc_audio_unconfigured"}`)
}
return windowsAudio.run(ctx, args.Action)
}
+83
View File
@@ -0,0 +1,83 @@
package main
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"testing"
"time"
"alice-back/station/pcaudio"
)
func TestPCAudioWaitsForAgentAndExpires(t *testing.T) {
const token = "test-token"
control := &pcAudioControl{
token: token, peer: netip.MustParseAddr("192.0.2.10"), desired: newPCCommand(false),
}
poll := func(state pcaudio.State, key string) (pcaudio.Command, int) {
data, _ := json.Marshal(state)
req := httptest.NewRequest(http.MethodPost, pcaudio.Path, bytes.NewReader(data))
req.RemoteAddr = "192.0.2.10:42000"
req.Header.Set("Authorization", "Bearer "+key)
response := httptest.NewRecorder()
control.ServeHTTP(response, req)
var cmd pcaudio.Command
if response.Code == http.StatusOK {
if err := json.Unmarshal(response.Body.Bytes(), &cmd); err != nil {
t.Fatal(err)
}
}
return cmd, response.Code
}
if _, code := poll(pcaudio.State{}, "wrong"); code != http.StatusForbidden {
t.Fatalf("accepted unauthorized request: %d", code)
}
if result := control.run(context.Background(), "start"); !strings.Contains(string(result), "offline") {
t.Fatalf("started while the PC was offline: %s", result)
}
initial, _ := poll(pcaudio.State{}, token)
if initial.Enabled {
t.Fatal("initial command enabled audio")
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
done := make(chan json.RawMessage, 1)
go func() { done <- control.run(ctx, "start") }()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
seen := initial
for !seen.Enabled {
select {
case <-ctx.Done():
t.Fatal("start command did not reach the agent")
case <-ticker.C:
seen, _ = poll(pcaudio.State{Command: initial}, token)
}
}
select {
case result := <-done:
t.Fatalf("start reported success before acknowledgment: %s", result)
default:
}
poll(pcaudio.State{Command: seen}, token)
select {
case result := <-done:
if string(result) != `{"enabled":true,"ok":true}` {
t.Fatalf("start result: %s", result)
}
case <-ctx.Done():
t.Fatal("start acknowledgment was not returned")
}
control.mu.Lock()
control.last = time.Now().Add(-pcAudioLease)
control.mu.Unlock()
expired, _ := poll(pcaudio.State{Command: seen}, token)
if expired.Enabled || expired.Revision == seen.Revision {
t.Fatal("reconnecting agent would automatically restart streaming")
}
}
+13
View File
@@ -0,0 +1,13 @@
package pcaudio
const Path = "/v1/pc-audio"
type Command struct {
Revision string `json:"revision"`
Enabled bool `json:"enabled"`
}
type State struct {
Command
Error string `json:"error,omitempty"`
}
+17 -1
View File
@@ -15,11 +15,12 @@ if [ ! -d /sys/class/gpio/gpio414 ]; then
fi
fi
child=
pc_receiver=
timer=
stopping=0
stop() {
stopping=1
for p in "$child" "$timer"; do [ -z "$p" ] || kill -TERM "$p" 2>/dev/null || :; done
for p in "$child" "$pc_receiver" "$timer"; do [ -z "$p" ] || kill -TERM "$p" 2>/dev/null || :; done
}
trap stop TERM INT HUP
reset_duck() {
@@ -37,17 +38,32 @@ while [ "$stopping" -eq 0 ]; do
fi
DEVICE_TOKEN=$(cat device-token)
export DEVICE_TOKEN
if [ -x ./scream-receiver ] && [ -r pc-audio-receiver.conf ]; then
: >/tmp/alice-pc-audio.log
(
read -r listen
read -r source
exec ./scream-receiver -listen "$listen" -source "$source"
) <pc-audio-receiver.conf >>/tmp/alice-pc-audio.log 2>&1 9>&- &
pc_receiver=$!
fi
: > /tmp/alice-station.log
./alice-station -mode wake -voice-threshold 12 -vqe-helper /data/opt/alice-backend/vqe-capture.sh -vqe-wake-model /system/vendor/quasar/spotter_models/activation/alisa >> /tmp/alice-station.log 2>&1 9>&- &
child=$!
while [ "$stopping" -eq 0 ] && kill -0 "$child" 2>/dev/null; do
if [ "$(wc -c < /tmp/alice-station.log)" -gt 524288 ]; then : >/tmp/alice-station.log; fi
if [ -n "$pc_receiver" ] && [ "$(wc -c < /tmp/alice-pc-audio.log)" -gt 524288 ]; then : >/tmp/alice-pc-audio.log; fi
sleep 3 & timer=$!; wait "$timer" || :; timer=
done
wait "$child" || :
# A signal interrupts wait; reap the station before releasing the microphone lock.
[ "$stopping" -eq 0 ] || wait "$child" 2>/dev/null || :
child=
if [ -n "$pc_receiver" ]; then
kill -TERM "$pc_receiver" 2>/dev/null || :
wait "$pc_receiver" 2>/dev/null || :
pc_receiver=
fi
[ "$stopping" -ne 0 ] && break
sleep 5 & timer=$!; wait "$timer" || :; timer=
done