Files
alice-backend/station/vqe/capture.c
T

262 lines
9.9 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Mini 2 armhf firmware ABI, verified against libyandex_evil BuildID
// d1d0f12f8f8389c0ca72672c61530161b85353c9 (Quasar 3.21).
// stdin: ALSA plug:default, 48 kHz S16_LE, four mics followed by stereo speaker feedback.
// stdout: vqe_0, 16 kHz mono S16_LE. In spotter mode, each 10 ms block
// is followed by one byte of flags (bits 0–1: wake and mute, bits 2–4: command).
#include <dlfcn.h>
#include <math.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
enum { INPUT = 1536, OUTPUT = 512, HOP = 160, MICS = 4, SPEAKERS = 2,
RAW_MIC_CHANNEL = 1, ASR_CHANNEL = 6, WAKE = 1, MUTED = 2 };
static const char *(*error_message)(void *);
static void (*error_free)(void *);
static void check(void *error) {
if (!error) return;
fprintf(stderr, "VQE: %s\n", error_message(error));
error_free(error);
exit(1);
}
static void *symbol(void *lib, const char *name) {
void *address = dlsym(lib, name);
if (!address) {
fprintf(stderr, "VQE: %s\n", dlerror());
exit(1);
}
return address;
}
static int16_t pcm(float sample) {
if (!isfinite(sample)) return 0;
float value = sample * 32768.0f;
if (value >= 32767) return 32767;
if (value <= -32768) return -32768;
return (int16_t)value;
}
struct spotter {
void *config, *model;
bool raw_channel;
void *(*create)(void *, int, size_t, const int *, int, void **);
void (*destroy)(void *, void **);
void (*free_config)(void *, void **);
void (*submit)(void *, const int16_t *, size_t, void **);
int (*next_event)(void *, void **);
const char *(*phrase)(void *, void **);
const char *(*error_message)(void *);
void (*error_free)(void *);
};
static void check_spotter(struct spotter *spot, void *error) {
if (!error) return;
fprintf(stderr, "Spotter: %s\n", spot->error_message(error));
spot->error_free(error);
exit(1);
}
static void start_spotter(struct spotter *spot) {
// The bundled activation model consumes the cleaned stream and one raw mic.
const int channels[] = {0, 4};
void *error = NULL;
spot->model = spot->create(spot->config, 16000, spot->raw_channel ? 2 : 1, channels, 0, &error);
check_spotter(spot, error);
if (!spot->model) exit(1);
}
static void stop_spotter(struct spotter *spot) {
if (!spot->model) return;
void *error = NULL;
spot->destroy(spot->model, &error);
check_spotter(spot, error);
spot->model = NULL;
}
static void load_spotter(struct spotter *spot, void *lib, const char *path, bool raw_channel) {
spot->raw_channel = raw_channel;
#define SPOT(name) symbol(lib, "spotter_" #name)
void *(*read_config)(const char *, void **) = SPOT(conf_read);
spot->create = SPOT(create_multichannel);
spot->destroy = SPOT(destroy);
spot->free_config = SPOT(conf_free);
spot->submit = SPOT(submit_raw_data);
spot->next_event = SPOT(get_next_event);
spot->phrase = SPOT(get_current_phrase);
spot->error_message = SPOT(error_get_message);
spot->error_free = SPOT(error_free);
#undef SPOT
void *error = NULL;
spot->config = read_config(path, &error);
check_spotter(spot, error);
if (!spot->config) exit(1);
start_spotter(spot);
}
static uint8_t detect(struct spotter *spot, const int16_t *clean, const float *raw) {
int16_t audio[OUTPUT][2];
const int16_t *input = clean;
size_t length = OUTPUT * sizeof(int16_t);
if (raw) {
for (size_t i = 0; i < OUTPUT; ++i) {
audio[i][0] = clean[i];
audio[i][1] = pcm(raw[i]);
}
input = &audio[0][0];
length = sizeof(audio);
}
void *error = NULL;
spot->submit(spot->model, input, length, &error);
check_spotter(spot, error);
uint8_t result = 0;
for (;;) {
int event = spot->next_event(spot->model, &error);
check_spotter(spot, error);
if (event == 0) return result;
if (event != 2) continue;
const char *phrase = spot->phrase(spot->model, &error);
check_spotter(spot, error);
if (!phrase) continue;
if (raw) {
if (strcmp(phrase, "алиса") == 0) result = WAKE;
} else {
if (strcmp(phrase, "стоп") == 0 || strcmp(phrase, "хватит") == 0) result = 1;
else if (strcmp(phrase, "громче") == 0) result = 2;
else if (strcmp(phrase, "тише") == 0) result = 3;
else if (strcmp(phrase, "пауза") == 0) result = 4;
else if (strcmp(phrase, "продолжи") == 0 || strcmp(phrase, "играй") == 0) result = 5;
else if (strcmp(phrase, "следующий") == 0 || strcmp(phrase, "дальше") == 0) result = 6;
else if (strcmp(phrase, "назад") == 0) result = 7;
}
}
}
static bool mic_unmuted(const char *path) {
FILE *file = fopen(path, "r");
if (!file) return false;
int value = fgetc(file);
fclose(file);
return value == '1';
}
int main(int argc, char **argv) {
if (argc != 1 && argc != 3) {
fprintf(stderr, "Usage: alice-vqe [SPOTTER_MODEL MUTE_GPIO]\n");
return 1;
}
const bool use_spotter = argc == 3;
void *lib = dlopen("/system/vendor/quasar/libs/libyandex_evil.so", RTLD_NOW | RTLD_LOCAL);
if (!lib) {
fprintf(stderr, "VQE: %s\n", dlerror());
return 1;
}
#define VQE(name) symbol(lib, "YandexVqeC_" #name)
error_message = VQE(Error_GetMessage);
error_free = VQE(Error_Free);
void *(*preset)(const char *, void **) = VQE(Config_CreateFromPreset);
void (*free_config)(void *) = VQE(Config_Free);
int (*input_rate)(void *) = VQE(Config_GetInputSamplingRate);
int (*output_rate)(void *) = VQE(Config_GetOutputSamplingRate);
int (*input_chunk)(void *) = VQE(Config_GetInputChunkSize);
int (*mic_count)(void *) = VQE(Config_GetMicsCount);
int (*speaker_count)(void *) = VQE(Config_GetSpeakersCount);
void *(*create)(void *, const char *, size_t, const char **, void **) = VQE(Create);
void (*destroy)(void *) = VQE(Free);
void *(*process)(void *, const float *, size_t, const float *, size_t) = VQE(Process);
void *(*channel)(void *, int, size_t, const float **, size_t *) = VQE(GetOutputChannelData);
void *(*omni_mic)(void *, size_t *) = VQE(GetOmniMicIndex);
#undef VQE
void *config = NULL, *vqe = NULL;
check(preset("yandexmini_2", &config));
if (input_rate(config) != 48000 || output_rate(config) != 16000 ||
input_chunk(config) != INPUT || mic_count(config) != MICS || speaker_count(config) != SPEAKERS) {
fprintf(stderr, "VQE: incompatible Mini 2 preset\n");
return 1;
}
check(create(config, "/system/vendor/quasar/vqe_models", 0, NULL, &vqe));
free_config(config);
struct spotter spot = {0}, commands = {0};
if (use_spotter) {
load_spotter(&spot, lib, argv[1], true);
load_spotter(&commands, lib, "/system/vendor/quasar/spotter_models/command", false);
}
int16_t input[INPUT][MICS + SPEAKERS], output[OUTPUT];
float mics[INPUT][MICS], speakers[INPUT][SPEAKERS];
// Suppress the PDM startup transient while warming the processor (512 ms).
unsigned warmup = 16;
uint8_t packet[HOP * sizeof(int16_t) + 1];
size_t packet_samples = 0;
bool partial_muted = false;
while (fread(input, sizeof(input), 1, stdin) == 1) {
for (size_t i = 0; i < INPUT; ++i) {
for (size_t j = 0; j < MICS; ++j) mics[i][j] = input[i][j] / 32768.0f;
for (size_t j = 0; j < SPEAKERS; ++j) speakers[i][j] = input[i][MICS+j] / 32768.0f;
}
check(process(vqe, &mics[0][0], INPUT * MICS, &speakers[0][0], INPUT * SPEAKERS));
const float *samples = NULL;
size_t count = 0;
check(channel(vqe, ASR_CHANNEL, 0, &samples, &count));
if (count != OUTPUT || !samples) {
fprintf(stderr, "VQE: incompatible output frame\n");
return 1;
}
if (warmup) {
--warmup;
continue;
}
for (size_t i = 0; i < OUTPUT; ++i) output[i] = pcm(samples[i]);
if (!use_spotter) {
if (fwrite(output, sizeof(output), 1, stdout) != 1 || fflush(stdout) != 0) return 1;
continue;
}
uint8_t detected = 0;
bool chunk_muted = !mic_unmuted(argv[2]);
if (!chunk_muted) {
if (!spot.model) start_spotter(&spot);
if (!commands.model) start_spotter(&commands);
size_t mic = 0;
check(omni_mic(vqe, &mic));
check(channel(vqe, RAW_MIC_CHANNEL, mic, &samples, &count));
if (count != OUTPUT || !samples) return 1;
// The last 10 ms packet in this chunk carries the spotter event.
detected = detect(&spot, output, samples) | (detect(&commands, output, NULL) << 2);
} else {
stop_spotter(&spot);
stop_spotter(&commands);
memset(output, 0, sizeof(output));
}
for (size_t i = 0; i < OUTPUT; ++i) {
memcpy(packet + packet_samples++ * sizeof(int16_t), &output[i], sizeof(int16_t));
if (packet_samples != HOP) continue;
bool last = OUTPUT - i - 1 < HOP;
packet[sizeof(packet) - 1] =
(last ? detected : 0) | (chunk_muted || partial_muted ? MUTED : 0);
if (fwrite(packet, sizeof(packet), 1, stdout) != 1) return 1;
packet_samples = 0;
partial_muted = false;
}
// If the final packet straddles two chunks, carry its mute state.
partial_muted = chunk_muted && packet_samples != 0;
if (fflush(stdout) != 0) return 1;
}
if (use_spotter) {
stop_spotter(&spot);
void *error = NULL;
spot.free_config(spot.config, &error);
check_spotter(&spot, error);
stop_spotter(&commands);
commands.free_config(commands.config, &error);
check_spotter(&commands, error);
}
destroy(vqe);
return ferror(stdin) ? 1 : 0;
}