jeans/frontend
jeans/frontend
1
0
Fork 0
mirror of https://github.com/home-assistant/frontend.git synced 2025-07-25 18:26:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix audio recorder (#16280)

Browse Source 
FIx audio recorder
This commit is contained in:
Bram Kragten 2023-04-23 19:48:46 +02:00 committed by GitHub
parent 8d1ae71741
commit 35baf4c779
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 35 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/panels/config/voice-assistants/debug/assist-pipeline-run-debug.ts
View File

@ -189,9 +189,7 @@ export class AssistPipelineRunDebug extends LitElement {
}); });
this._audioBuffer = []; this._audioBuffer = [];
audioRecorder.start(); await audioRecorder.start();
this.hass.connection.socket!.binaryType = "arraybuffer";
let run: PipelineRun | undefined; let run: PipelineRun | undefined;
@ -288,6 +286,7 @@ export class AssistPipelineRunDebug extends LitElement {
} }
private _sendAudioChunk(chunk: Int16Array) { private _sendAudioChunk(chunk: Int16Array) {
this.hass.connection.socket!.binaryType = "arraybuffer";
// Turn into 8 bit so we can prefix our handler ID. // Turn into 8 bit so we can prefix our handler ID.
const data = new Uint8Array(1 + chunk.length * 2); const data = new Uint8Array(1 + chunk.length * 2);
data[0] = this._pipelineRuns[0].run.runner_data.stt_binary_handler_id!; data[0] = this._pipelineRuns[0].run.runner_data.stt_binary_handler_id!;

65
src/util/audio-recorder.ts
View File

@ -7,6 +7,10 @@ export class AudioRecorder {
private _stream: MediaStream | undefined; private _stream: MediaStream | undefined;
private _source: MediaStreamAudioSourceNode | undefined;
private _recorder: AudioWorkletNode | undefined;
constructor(callback: (data: Int16Array) => void) { constructor(callback: (data: Int16Array) => void) {
this._callback = callback; this._callback = callback;
} }
@ -28,30 +32,19 @@ export class AudioRecorder {
} }
public async start() { public async start() {
this._active = true; if (!this._context || !this._stream || !this._source || !this._recorder) {
try {
if (!this._context || !this._stream) { await this._createContext();
await this._createContext(); } catch (err) {
} else { // eslint-disable-next-line no-console
this._context.resume(); console.error(err);
this._stream.getTracks()[0].enabled = true; this._active = false;
}
if (!this._context || !this._stream) {
this._active = false;
return;
}
const source = this._context.createMediaStreamSource(this._stream);
const recorder = new AudioWorkletNode(this._context, "recorder.worklet");
source.connect(recorder).connect(this._context.destination);
recorder.port.onmessage = (e) => {
if (!this._active) {
return;
} }
this._callback(e.data); } else {
}; this._stream.getTracks()[0].enabled = true;
await this._context.resume();
this._active = true;
}
} }
public async stop() { public async stop() {
@ -68,21 +61,29 @@ export class AudioRecorder {
this._context?.close(); this._context?.close();
this._stream = undefined; this._stream = undefined;
this._context = undefined; this._context = undefined;
this._source = undefined;
this._recorder = undefined;
} }
private async _createContext() { private async _createContext() {
try { // @ts-ignore-next-line
// @ts-ignore-next-line this._context = new (window.AudioContext || window.webkitAudioContext)();
this._context = new (window.AudioContext || window.webkitAudioContext)(); this._stream = await navigator.mediaDevices.getUserMedia({ audio: true });
this._stream = await navigator.mediaDevices.getUserMedia({ audio: true });
} catch (err) {
// eslint-disable-next-line no-console
console.error(err);
return;
}
await this._context.audioWorklet.addModule( await this._context.audioWorklet.addModule(
new URL("./recorder.worklet.js", import.meta.url) new URL("./recorder.worklet.js", import.meta.url)
); );
this._source = this._context.createMediaStreamSource(this._stream);
this._recorder = new AudioWorkletNode(this._context, "recorder.worklet");
this._recorder.port.onmessage = (e) => {
if (!this._active) {
return;
}
this._callback(e.data);
};
this._active = true;
this._source.connect(this._recorder);
} }
} }