maxmustermann/godot
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[HTML5] Run Audio process in thread when available

Browse source 
This should fix some of the audio stuttering issues when the HTML5
export is compiled with threads support.
The API should be ported to AudioWorklet to (hopefully) be perfect.
That though, cannot be backported to 3.2 due to extra restriction of
AudioWorklet (which only runs in SecureContext, and needs a polyfill for
Safari).
This commit is contained in:
Fabio Alessandrelli 2020-09-28 15:43:48 +02:00
parent a57bd798cd
commit 61d4b8045c
5 changed files with 331 additions and 186 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
platform/javascript/SCsub
View file

@ -18,6 +18,7 @@ build = env.add_program(build_targets, javascript_files)
js_libraries = [
"native/http_request.js",
"native/library_godot_audio.js",
]
for lib in js_libraries:
env.Append(LINKFLAGS=["--js-library", env.File(lib).path])

266
platform/javascript/audio_driver_javascript.cpp
View file

@ -34,31 +34,55 @@
#include <emscripten.h>
#include "godot_audio.h"
AudioDriverJavaScript *AudioDriverJavaScript::singleton = NULL;
bool AudioDriverJavaScript::is_available() {
return EM_ASM_INT({
if (!(window.AudioContext || window.webkitAudioContext)) {
return 0;
}
return 1;
}) != 0;
return godot_audio_is_available() != 0;
}
const char *AudioDriverJavaScript::get_name() const {
return "JavaScript";
}
extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_js_mix() {
AudioDriverJavaScript::singleton->mix_to_js();
#ifndef NO_THREADS
void AudioDriverJavaScript::_audio_thread_func(void *p_data) {
AudioDriverJavaScript *obj = static_cast<AudioDriverJavaScript *>(p_data);
while (!obj->quit) {
obj->lock();
if (!obj->needs_process) {
obj->unlock();
OS::get_singleton()->delay_usec(1000); // Give the browser some slack.
continue;
}
obj->_js_driver_process();
obj->needs_process = false;
obj->unlock();
}
}
#endif
extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_start() {
#ifndef NO_THREADS
AudioDriverJavaScript::singleton->lock();
#else
AudioDriverJavaScript::singleton->_js_driver_process();
#endif
}
extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_end() {
#ifndef NO_THREADS
AudioDriverJavaScript::singleton->needs_process = true;
AudioDriverJavaScript::singleton->unlock();
#endif
}
extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_capture(float sample) {
AudioDriverJavaScript::singleton->process_capture(sample);
}
void AudioDriverJavaScript::mix_to_js() {
int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
void AudioDriverJavaScript::_js_driver_process() {
int sample_count = memarr_len(internal_buffer) / channel_count;
int32_t *stream_buffer = reinterpret_cast<int32_t *>(internal_buffer);
audio_server_process(sample_count, stream_buffer);
@ -73,37 +97,12 @@ void AudioDriverJavaScript::process_capture(float sample) {
}
Error AudioDriverJavaScript::init() {
int mix_rate = GLOBAL_GET("audio/mix_rate");
mix_rate = GLOBAL_GET("audio/mix_rate");
int latency = GLOBAL_GET("audio/output_latency");
/* clang-format off */
_driver_id = EM_ASM_INT({
const MIX_RATE = $0;
const LATENCY = $1 / 1000;
return Module.IDHandler.add({
'context': new (window.AudioContext || window.webkitAudioContext)({ sampleRate: MIX_RATE, latencyHint: LATENCY}),
'input': null,
'stream': null,
'script': null
});
}, mix_rate, latency);
/* clang-format on */
int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode());
channel_count = godot_audio_init(mix_rate, latency);
buffer_length = closest_power_of_2((latency * mix_rate / 1000) * channel_count);
/* clang-format off */
buffer_length = EM_ASM_INT({
var ref = Module.IDHandler.get($0);
const ctx = ref['context'];
const BUFFER_LENGTH = $1;
const CHANNEL_COUNT = $2;
var script = ctx.createScriptProcessor(BUFFER_LENGTH, 2, CHANNEL_COUNT);
script.connect(ctx.destination);
ref['script'] = script;
return script.bufferSize;
}, _driver_id, buffer_length, channel_count);
/* clang-format on */
buffer_length = godot_audio_create_processor(buffer_length, channel_count);
if (!buffer_length) {
return FAILED;
}
@ -114,134 +113,67 @@ Error AudioDriverJavaScript::init() {
internal_buffer = memnew_arr(float, buffer_length *channel_count);
}
return internal_buffer ? OK : ERR_OUT_OF_MEMORY;
if (!internal_buffer) {
return ERR_OUT_OF_MEMORY;
}
return OK;
}
void AudioDriverJavaScript::start() {
/* clang-format off */
EM_ASM({
const ref = Module.IDHandler.get($0);
var INTERNAL_BUFFER_PTR = $1;
var audioDriverMixFunction = cwrap('audio_driver_js_mix');
var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
ref['script'].onaudioprocess = function(audioProcessingEvent) {
audioDriverMixFunction();
var input = audioProcessingEvent.inputBuffer;
var output = audioProcessingEvent.outputBuffer;
var internalBuffer = HEAPF32.subarray(
INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT,
INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);
for (var channel = 0; channel < output.numberOfChannels; channel++) {
var outputData = output.getChannelData(channel);
// Loop through samples.
for (var sample = 0; sample < outputData.length; sample++) {
outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
}
}
if (ref['input']) {
var inputDataL = input.getChannelData(0);
var inputDataR = input.getChannelData(1);
for (var i = 0; i < inputDataL.length; i++) {
audioDriverProcessCapture(inputDataL[i]);
audioDriverProcessCapture(inputDataR[i]);
}
}
};
}, _driver_id, internal_buffer);
/* clang-format on */
#ifndef NO_THREADS
mutex = Mutex::create();
thread = Thread::create(_audio_thread_func, this);
#endif
godot_audio_start(internal_buffer);
}
void AudioDriverJavaScript::resume() {
/* clang-format off */
EM_ASM({
const ref = Module.IDHandler.get($0);
if (ref && ref['context'] && ref['context'].resume)
ref['context'].resume();
}, _driver_id);
/* clang-format on */
godot_audio_resume();
}
float AudioDriverJavaScript::get_latency() {
/* clang-format off */
return EM_ASM_DOUBLE({
const ref = Module.IDHandler.get($0);
var latency = 0;
if (ref && ref['context']) {
const ctx = ref['context'];
if (ctx.baseLatency) {
latency += ctx.baseLatency;
}
if (ctx.outputLatency) {
latency += ctx.outputLatency;
}
}
return latency;
}, _driver_id);
/* clang-format on */
return godot_audio_get_latency();
}
int AudioDriverJavaScript::get_mix_rate() const {
/* clang-format off */
return EM_ASM_INT({
const ref = Module.IDHandler.get($0);
return ref && ref['context'] ? ref['context'].sampleRate : 0;
}, _driver_id);
/* clang-format on */
return mix_rate;
}
AudioDriver::SpeakerMode AudioDriverJavaScript::get_speaker_mode() const {
/* clang-format off */
return get_speaker_mode_by_total_channels(EM_ASM_INT({
const ref = Module.IDHandler.get($0);
return ref && ref['context'] ? ref['context'].destination.channelCount : 0;
}, _driver_id));
/* clang-format on */
return get_speaker_mode_by_total_channels(channel_count);
}
// No locking, as threads are not supported.
void AudioDriverJavaScript::lock() {
#ifndef NO_THREADS
if (mutex) {
mutex->lock();
}
#endif
}
void AudioDriverJavaScript::unlock() {
#ifndef NO_THREADS
if (mutex) {
mutex->unlock();
}
#endif
}
void AudioDriverJavaScript::finish_async() {
// Close the context, add the operation to the async_finish list in module.
int id = _driver_id;
_driver_id = 0;
/* clang-format off */
EM_ASM({
const id = $0;
var ref = Module.IDHandler.get(id);
Module.async_finish.push(new Promise(function(accept, reject) {
if (!ref) {
console.log("Ref not found!", id, Module.IDHandler);
setTimeout(accept, 0);
} else {
Module.IDHandler.remove(id);
const context = ref['context'];
// Disconnect script and input.
ref['script'].disconnect();
if (ref['input'])
ref['input'].disconnect();
ref = null;
context.close().then(function() {
accept();
}).catch(function(e) {
accept();
});
}
}));
}, id);
/* clang-format on */
#ifndef NO_THREADS
quit = true; // Ask thread to quit.
#endif
godot_audio_finish_async();
}
void AudioDriverJavaScript::finish() {
#ifndef NO_THREADS
Thread::wait_to_finish(thread);
memdelete(thread);
thread = NULL;
memdelete(mutex);
mutex = NULL;
#endif
if (internal_buffer) {
memdelete_arr(internal_buffer);
internal_buffer = NULL;
@ -250,62 +182,28 @@ void AudioDriverJavaScript::finish() {
Error AudioDriverJavaScript::capture_start() {
input_buffer_init(buffer_length);
/* clang-format off */
EM_ASM({
function gotMediaInput(stream) {
var ref = Module.IDHandler.get($0);
ref['stream'] = stream;
ref['input'] = ref['context'].createMediaStreamSource(stream);
ref['input'].connect(ref['script']);
}
function gotMediaInputError(e) {
out(e);
}
if (navigator.mediaDevices.getUserMedia) {
navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
} else {
if (!navigator.getUserMedia)
navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
}
}, _driver_id);
/* clang-format on */
godot_audio_capture_start();
return OK;
}
Error AudioDriverJavaScript::capture_stop() {
/* clang-format off */
EM_ASM({
var ref = Module.IDHandler.get($0);
if (ref['stream']) {
const tracks = ref['stream'].getTracks();
for (var i = 0; i < tracks.length; i++) {
tracks[i].stop();
}
ref['stream'] = null;
}
if (ref['input']) {
ref['input'].disconnect();
ref['input'] = null;
}
}, _driver_id);
/* clang-format on */
godot_audio_capture_stop();
input_buffer.clear();
return OK;
}
AudioDriverJavaScript::AudioDriverJavaScript() {
_driver_id = 0;
internal_buffer = NULL;
buffer_length = 0;
mix_rate = 0;
channel_count = 0;
#ifndef NO_THREADS
mutex = NULL;
thread = NULL;
quit = false;
needs_process = true;
#endif
singleton = this;
}

19
platform/javascript/audio_driver_javascript.h
View file

@ -31,18 +31,33 @@
#ifndef AUDIO_DRIVER_JAVASCRIPT_H
#define AUDIO_DRIVER_JAVASCRIPT_H
#include "core/os/mutex.h"
#include "core/os/thread.h"
#include "servers/audio_server.h"
class AudioDriverJavaScript : public AudioDriver {
private:
float *internal_buffer;
int _driver_id;
int buffer_length;
int mix_rate;
int channel_count;
public:
#ifndef NO_THREADS
Mutex *mutex;
Thread *thread;
bool quit;
bool needs_process;
static void _audio_thread_func(void *p_data);
#endif
void _js_driver_process();
static bool is_available();
void mix_to_js();
void process_capture(float sample);
static AudioDriverJavaScript *singleton;

58
platform/javascript/godot_audio.h Normal file
View file

@ -0,0 +1,58 @@
/*************************************************************************/
/* godot_audio.h */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
#ifndef GODOT_AUDIO_H
#define GODOT_AUDIO_H
#ifdef __cplusplus
extern "C" {
#endif
#include "stddef.h"
extern int godot_audio_is_available();
extern int godot_audio_init(int p_mix_rate, int p_latency);
extern int godot_audio_create_processor(int p_buffer_length, int p_channel_count);
extern void godot_audio_start(float *r_buffer_ptr);
extern void godot_audio_resume();
extern void godot_audio_finish_async();
extern float godot_audio_get_latency();
extern void godot_audio_capture_start();
extern void godot_audio_capture_stop();
#ifdef __cplusplus
}
#endif
#endif /* GODOT_AUDIO_H */

173
platform/javascript/native/library_godot_audio.js Normal file
View file

@ -0,0 +1,173 @@
/*************************************************************************/
/* library_godot_audio.js */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
var GodotAudio = {
$GodotAudio: {
ctx: null,
input: null,
script: null,
},
godot_audio_is_available__proxy: 'sync',
godot_audio_is_available: function () {
if (!(window.AudioContext || window.webkitAudioContext)) {
return 0;
}
return 1;
},
godot_audio_init: function(mix_rate, latency) {
GodotAudio.ctx = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: mix_rate,
latencyHint: latency
});
return GodotAudio.ctx.destination.channelCount;
},
godot_audio_create_processor: function(buffer_length, channel_count) {
GodotAudio.script = GodotAudio.ctx.createScriptProcessor(buffer_length, 2, channel_count);
GodotAudio.script.connect(GodotAudio.ctx.destination);
return GodotAudio.script.bufferSize;
},
godot_audio_start: function(buffer_ptr) {
var audioDriverProcessStart = cwrap('audio_driver_process_start');
var audioDriverProcessEnd = cwrap('audio_driver_process_end');
var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']);
GodotAudio.script.onaudioprocess = function(audioProcessingEvent) {
audioDriverProcessStart();
var input = audioProcessingEvent.inputBuffer;
var output = audioProcessingEvent.outputBuffer;
var internalBuffer = HEAPF32.subarray(
buffer_ptr / HEAPF32.BYTES_PER_ELEMENT,
buffer_ptr / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels);
for (var channel = 0; channel < output.numberOfChannels; channel++) {
var outputData = output.getChannelData(channel);
// Loop through samples.
for (var sample = 0; sample < outputData.length; sample++) {
outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel];
}
}
if (GodotAudio.input) {
var inputDataL = input.getChannelData(0);
var inputDataR = input.getChannelData(1);
for (var i = 0; i < inputDataL.length; i++) {
audioDriverProcessCapture(inputDataL[i]);
audioDriverProcessCapture(inputDataR[i]);
}
}
audioDriverProcessEnd();
};
},
godot_audio_resume: function() {
if (GodotAudio.ctx && GodotAudio.ctx.state != 'running') {
GodotAudio.ctx.resume();
}
},
godot_audio_finish_async: function() {
Module.async_finish.push(new Promise(function(accept, reject) {
if (!GodotAudio.ctx) {
setTimeout(accept, 0);
} else {
if (GodotAudio.script) {
GodotAudio.script.disconnect();
GodotAudio.script = null;
}
if (GodotAudio.input) {
GodotAudio.input.disconnect();
GodotAudio.input = null;
}
GodotAudio.ctx.close().then(function() {
accept();
}).catch(function(e) {
accept();
});
GodotAudio.ctx = null;
}
}));
},
godot_audio_get_latency__proxy: 'sync',
godot_audio_get_latency: function() {
var latency = 0;
if (GodotAudio.ctx) {
if (GodotAudio.ctx.baseLatency) {
latency += GodotAudio.ctx.baseLatency;
}
if (GodotAudio.ctx.outputLatency) {
latency += GodotAudio.ctx.outputLatency;
}
}
return latency;
},
godot_audio_capture_start__proxy: 'sync',
godot_audio_capture_start: function() {
if (GodotAudio.input) {
return; // Already started.
}
function gotMediaInput(stream) {
GodotAudio.input = GodotAudio.ctx.createMediaStreamSource(stream);
GodotAudio.input.connect(GodotAudio.script);
}
function gotMediaInputError(e) {
out(e);
}
if (navigator.mediaDevices.getUserMedia) {
navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError);
} else {
if (!navigator.getUserMedia)
navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError);
}
},
godot_audio_capture_stop__proxy: 'sync',
godot_audio_capture_stop: function() {
if (GodotAudio.input) {
const tracks = GodotAudio.input.mediaStream.getTracks();
for (var i = 0; i < tracks.length; i++) {
tracks[i].stop();
}
GodotAudio.input.disconnect();
GodotAudio.input = null;
}
},
};
autoAddDeps(GodotAudio, "$GodotAudio");
mergeInto(LibraryManager.library, GodotAudio);