262 lines
9.9 KiB
C
262 lines
9.9 KiB
C
// Mini 2 armhf firmware ABI, verified against libyandex_evil BuildID
|
||
// d1d0f12f8f8389c0ca72672c61530161b85353c9 (Quasar 3.21).
|
||
// stdin: ALSA plug:default, 48 kHz S16_LE, four mics followed by stereo speaker feedback.
|
||
// stdout: vqe_0, 16 kHz mono S16_LE. In spotter mode, each 10 ms block
|
||
// is followed by one byte of flags (bits 0–1: wake and mute, bits 2–4: command).
|
||
#include <dlfcn.h>
|
||
#include <math.h>
|
||
#include <stdbool.h>
|
||
#include <stdint.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
|
||
enum { INPUT = 1536, OUTPUT = 512, HOP = 160, MICS = 4, SPEAKERS = 2,
|
||
RAW_MIC_CHANNEL = 1, ASR_CHANNEL = 6, WAKE = 1, MUTED = 2 };
|
||
|
||
static const char *(*error_message)(void *);
|
||
static void (*error_free)(void *);
|
||
|
||
static void check(void *error) {
|
||
if (!error) return;
|
||
fprintf(stderr, "VQE: %s\n", error_message(error));
|
||
error_free(error);
|
||
exit(1);
|
||
}
|
||
|
||
static void *symbol(void *lib, const char *name) {
|
||
void *address = dlsym(lib, name);
|
||
if (!address) {
|
||
fprintf(stderr, "VQE: %s\n", dlerror());
|
||
exit(1);
|
||
}
|
||
return address;
|
||
}
|
||
|
||
static int16_t pcm(float sample) {
|
||
if (!isfinite(sample)) return 0;
|
||
float value = sample * 32768.0f;
|
||
if (value >= 32767) return 32767;
|
||
if (value <= -32768) return -32768;
|
||
return (int16_t)value;
|
||
}
|
||
|
||
struct spotter {
|
||
void *config, *model;
|
||
bool raw_channel;
|
||
void *(*create)(void *, int, size_t, const int *, int, void **);
|
||
void (*destroy)(void *, void **);
|
||
void (*free_config)(void *, void **);
|
||
void (*submit)(void *, const int16_t *, size_t, void **);
|
||
int (*next_event)(void *, void **);
|
||
const char *(*phrase)(void *, void **);
|
||
const char *(*error_message)(void *);
|
||
void (*error_free)(void *);
|
||
};
|
||
|
||
static void check_spotter(struct spotter *spot, void *error) {
|
||
if (!error) return;
|
||
fprintf(stderr, "Spotter: %s\n", spot->error_message(error));
|
||
spot->error_free(error);
|
||
exit(1);
|
||
}
|
||
|
||
static void start_spotter(struct spotter *spot) {
|
||
// The bundled activation model consumes the cleaned stream and one raw mic.
|
||
const int channels[] = {0, 4};
|
||
void *error = NULL;
|
||
spot->model = spot->create(spot->config, 16000, spot->raw_channel ? 2 : 1, channels, 0, &error);
|
||
check_spotter(spot, error);
|
||
if (!spot->model) exit(1);
|
||
}
|
||
|
||
static void stop_spotter(struct spotter *spot) {
|
||
if (!spot->model) return;
|
||
void *error = NULL;
|
||
spot->destroy(spot->model, &error);
|
||
check_spotter(spot, error);
|
||
spot->model = NULL;
|
||
}
|
||
|
||
static void load_spotter(struct spotter *spot, void *lib, const char *path, bool raw_channel) {
|
||
spot->raw_channel = raw_channel;
|
||
#define SPOT(name) symbol(lib, "spotter_" #name)
|
||
void *(*read_config)(const char *, void **) = SPOT(conf_read);
|
||
spot->create = SPOT(create_multichannel);
|
||
spot->destroy = SPOT(destroy);
|
||
spot->free_config = SPOT(conf_free);
|
||
spot->submit = SPOT(submit_raw_data);
|
||
spot->next_event = SPOT(get_next_event);
|
||
spot->phrase = SPOT(get_current_phrase);
|
||
spot->error_message = SPOT(error_get_message);
|
||
spot->error_free = SPOT(error_free);
|
||
#undef SPOT
|
||
void *error = NULL;
|
||
spot->config = read_config(path, &error);
|
||
check_spotter(spot, error);
|
||
if (!spot->config) exit(1);
|
||
start_spotter(spot);
|
||
}
|
||
|
||
static uint8_t detect(struct spotter *spot, const int16_t *clean, const float *raw) {
|
||
int16_t audio[OUTPUT][2];
|
||
const int16_t *input = clean;
|
||
size_t length = OUTPUT * sizeof(int16_t);
|
||
if (raw) {
|
||
for (size_t i = 0; i < OUTPUT; ++i) {
|
||
audio[i][0] = clean[i];
|
||
audio[i][1] = pcm(raw[i]);
|
||
}
|
||
input = &audio[0][0];
|
||
length = sizeof(audio);
|
||
}
|
||
void *error = NULL;
|
||
spot->submit(spot->model, input, length, &error);
|
||
check_spotter(spot, error);
|
||
uint8_t result = 0;
|
||
for (;;) {
|
||
int event = spot->next_event(spot->model, &error);
|
||
check_spotter(spot, error);
|
||
if (event == 0) return result;
|
||
if (event != 2) continue;
|
||
const char *phrase = spot->phrase(spot->model, &error);
|
||
check_spotter(spot, error);
|
||
if (!phrase) continue;
|
||
if (raw) {
|
||
if (strcmp(phrase, "алиса") == 0) result = WAKE;
|
||
} else {
|
||
if (strcmp(phrase, "стоп") == 0 || strcmp(phrase, "хватит") == 0) result = 1;
|
||
else if (strcmp(phrase, "громче") == 0) result = 2;
|
||
else if (strcmp(phrase, "тише") == 0) result = 3;
|
||
else if (strcmp(phrase, "пауза") == 0) result = 4;
|
||
else if (strcmp(phrase, "продолжи") == 0 || strcmp(phrase, "играй") == 0) result = 5;
|
||
else if (strcmp(phrase, "следующий") == 0 || strcmp(phrase, "дальше") == 0) result = 6;
|
||
else if (strcmp(phrase, "назад") == 0) result = 7;
|
||
}
|
||
}
|
||
}
|
||
|
||
static bool mic_unmuted(const char *path) {
|
||
FILE *file = fopen(path, "r");
|
||
if (!file) return false;
|
||
int value = fgetc(file);
|
||
fclose(file);
|
||
return value == '1';
|
||
}
|
||
|
||
int main(int argc, char **argv) {
|
||
if (argc != 1 && argc != 3) {
|
||
fprintf(stderr, "Usage: alice-vqe [SPOTTER_MODEL MUTE_GPIO]\n");
|
||
return 1;
|
||
}
|
||
const bool use_spotter = argc == 3;
|
||
void *lib = dlopen("/system/vendor/quasar/libs/libyandex_evil.so", RTLD_NOW | RTLD_LOCAL);
|
||
if (!lib) {
|
||
fprintf(stderr, "VQE: %s\n", dlerror());
|
||
return 1;
|
||
}
|
||
#define VQE(name) symbol(lib, "YandexVqeC_" #name)
|
||
error_message = VQE(Error_GetMessage);
|
||
error_free = VQE(Error_Free);
|
||
void *(*preset)(const char *, void **) = VQE(Config_CreateFromPreset);
|
||
void (*free_config)(void *) = VQE(Config_Free);
|
||
int (*input_rate)(void *) = VQE(Config_GetInputSamplingRate);
|
||
int (*output_rate)(void *) = VQE(Config_GetOutputSamplingRate);
|
||
int (*input_chunk)(void *) = VQE(Config_GetInputChunkSize);
|
||
int (*mic_count)(void *) = VQE(Config_GetMicsCount);
|
||
int (*speaker_count)(void *) = VQE(Config_GetSpeakersCount);
|
||
void *(*create)(void *, const char *, size_t, const char **, void **) = VQE(Create);
|
||
void (*destroy)(void *) = VQE(Free);
|
||
void *(*process)(void *, const float *, size_t, const float *, size_t) = VQE(Process);
|
||
void *(*channel)(void *, int, size_t, const float **, size_t *) = VQE(GetOutputChannelData);
|
||
void *(*omni_mic)(void *, size_t *) = VQE(GetOmniMicIndex);
|
||
#undef VQE
|
||
|
||
void *config = NULL, *vqe = NULL;
|
||
check(preset("yandexmini_2", &config));
|
||
if (input_rate(config) != 48000 || output_rate(config) != 16000 ||
|
||
input_chunk(config) != INPUT || mic_count(config) != MICS || speaker_count(config) != SPEAKERS) {
|
||
fprintf(stderr, "VQE: incompatible Mini 2 preset\n");
|
||
return 1;
|
||
}
|
||
check(create(config, "/system/vendor/quasar/vqe_models", 0, NULL, &vqe));
|
||
free_config(config);
|
||
|
||
struct spotter spot = {0}, commands = {0};
|
||
if (use_spotter) {
|
||
load_spotter(&spot, lib, argv[1], true);
|
||
load_spotter(&commands, lib, "/system/vendor/quasar/spotter_models/command", false);
|
||
}
|
||
|
||
int16_t input[INPUT][MICS + SPEAKERS], output[OUTPUT];
|
||
float mics[INPUT][MICS], speakers[INPUT][SPEAKERS];
|
||
// Suppress the PDM startup transient while warming the processor (512 ms).
|
||
unsigned warmup = 16;
|
||
uint8_t packet[HOP * sizeof(int16_t) + 1];
|
||
size_t packet_samples = 0;
|
||
bool partial_muted = false;
|
||
while (fread(input, sizeof(input), 1, stdin) == 1) {
|
||
for (size_t i = 0; i < INPUT; ++i) {
|
||
for (size_t j = 0; j < MICS; ++j) mics[i][j] = input[i][j] / 32768.0f;
|
||
for (size_t j = 0; j < SPEAKERS; ++j) speakers[i][j] = input[i][MICS+j] / 32768.0f;
|
||
}
|
||
check(process(vqe, &mics[0][0], INPUT * MICS, &speakers[0][0], INPUT * SPEAKERS));
|
||
const float *samples = NULL;
|
||
size_t count = 0;
|
||
check(channel(vqe, ASR_CHANNEL, 0, &samples, &count));
|
||
if (count != OUTPUT || !samples) {
|
||
fprintf(stderr, "VQE: incompatible output frame\n");
|
||
return 1;
|
||
}
|
||
if (warmup) {
|
||
--warmup;
|
||
continue;
|
||
}
|
||
for (size_t i = 0; i < OUTPUT; ++i) output[i] = pcm(samples[i]);
|
||
if (!use_spotter) {
|
||
if (fwrite(output, sizeof(output), 1, stdout) != 1 || fflush(stdout) != 0) return 1;
|
||
continue;
|
||
}
|
||
uint8_t detected = 0;
|
||
bool chunk_muted = !mic_unmuted(argv[2]);
|
||
if (!chunk_muted) {
|
||
if (!spot.model) start_spotter(&spot);
|
||
if (!commands.model) start_spotter(&commands);
|
||
size_t mic = 0;
|
||
check(omni_mic(vqe, &mic));
|
||
check(channel(vqe, RAW_MIC_CHANNEL, mic, &samples, &count));
|
||
if (count != OUTPUT || !samples) return 1;
|
||
// The last 10 ms packet in this chunk carries the spotter event.
|
||
detected = detect(&spot, output, samples) | (detect(&commands, output, NULL) << 2);
|
||
} else {
|
||
stop_spotter(&spot);
|
||
stop_spotter(&commands);
|
||
memset(output, 0, sizeof(output));
|
||
}
|
||
for (size_t i = 0; i < OUTPUT; ++i) {
|
||
memcpy(packet + packet_samples++ * sizeof(int16_t), &output[i], sizeof(int16_t));
|
||
if (packet_samples != HOP) continue;
|
||
bool last = OUTPUT - i - 1 < HOP;
|
||
packet[sizeof(packet) - 1] =
|
||
(last ? detected : 0) | (chunk_muted || partial_muted ? MUTED : 0);
|
||
if (fwrite(packet, sizeof(packet), 1, stdout) != 1) return 1;
|
||
packet_samples = 0;
|
||
partial_muted = false;
|
||
}
|
||
// If the final packet straddles two chunks, carry its mute state.
|
||
partial_muted = chunk_muted && packet_samples != 0;
|
||
if (fflush(stdout) != 0) return 1;
|
||
}
|
||
if (use_spotter) {
|
||
stop_spotter(&spot);
|
||
void *error = NULL;
|
||
spot.free_config(spot.config, &error);
|
||
check_spotter(&spot, error);
|
||
stop_spotter(&commands);
|
||
commands.free_config(commands.config, &error);
|
||
check_spotter(&commands, error);
|
||
}
|
||
destroy(vqe);
|
||
return ferror(stdin) ? 1 : 0;
|
||
}
|