Моя доповідь на JSGroup.

1. JS: Audio Data Processing
Ingvar Stepanyan
Web Developer, MSP
@RReverser (http://rreverser.com/)

2. Native implementations:
[Mozilla] Audio Data API (low-level processing)
[WebKit] Web Audio API (+ some high-level sugar)
[Flash] Sound API (low-level processing)
Unifying wrappers:
dynamicaudio.js (Audio Data API + Flash)
XAudioJS (Audio Data API + Web Audio API + Flash)
sink.js & co. (Audio Data API + Web Audio API)
Current status
Low
Level

3. <audio id=“audio” ... />
....................................
var audio = document.getElementById(‘audio’),
channels, rate, bufferLength;
audio.addEventListener('loadedmetadata', function () {
channels = audio.mozChannels;
rate = audio.mozSampleRate;
bufferLength = audio.mozFrameBufferLength;
// ...
});
Reading audio data

4. audio.addEventListener('MozAudioAvailable', function () {
var buffer = event.frameBuffer,
time = event.time;
for (var i = 0; i < bufferLength / channels; i++ ) {
// use buffer[i * channels + channelNumber]
}
});
Reading audio data

5. 0.409862111297023, 0.259322117331362, -0.008777887762307, -0.984780922682458, -0.739880137997811, -0.393993360534229, -
0.698508825498069, -0.393512581774658, 0.073465971474621, -0.774029495474732, -0.068926253092157, 0.698052197232118,
0.450745893413682, 0.731625352383412, -0.633931102362129, 0.882738533608791, -0.493848309462582, -0.668645515465273,
0.271953939580002, 0.246314079650827, -0.645711462859856, 0.736913698005765, 0.872459503967142, -0.482284551106221,
0.661431378511110, -0.291951234239424, 0.669216884142705, -0.341110225783700, -0.165754738005617, -0.986556649848787,
0.803597140689702, -0.813939097068497, 0.297290844923532, -0.434373616658307, -0.820145776933725, -0.351005880456336,
0.380713105392354, -0.721515454501945, 0.066484189654290, 0.642101001764223, -0.597205735590506, -0.200460217995576,
0.491272422617811, 0.480988687186248, 0.227128301563507, 0.685168802023231, 0.075880282193273, -0.749673188605156,
0.702223692201174, -0.184544172051190, -0.506130812375945, -0.686331843733017, -0.791173541823365, 0.886386883125611,
0.042702665517532, 0.573126318821646, 0.576012637739739, -0.329392317046949, -0.560594547513062, -0.223420931279321,
0.652487040645743, 0.403985003945232, 0.881201502984618, 0.992330609078359, 0.157417714187382, 0.133725348839930,
0.343408728622162, 0.005933505310489, -0.914475827505964, 0.154820823526593, 0.148971641643792, 0.841135222645066,
0.095968894389784, -0.234891692775066, -0.055960466199662, -0.133320485223315, -0.945514113185999, 0.485242740877991,
0.933864235656065, 0.022960704882891, -0.139667183659487, 0.618401839172406, -0.195349340573667, 0.587291626042564,
0.158475194222853, 0.565554351499445, 0.101859803696333, 0.593813833398595, -0.349654793639986, 0.665640345371317,
0.814509967148215, 0.533843529155366, -0.631997214437279, -0.503973088121093, -0.180033876163095, -0.715287970025233,
0.739453218066262, -0.452880703839526, -0.112778265491663, -0.650074534874509, -0.925708286102557, 0.976021772837429,
0.768202886746198, -0.559265634778471, 0.004451442257993, 0.747659266125223, -0.692840193163558, -0.811031246931423,
0.064833012476722, -0.112213849236344, -0.045264398268911, 0.353260515196007, 0.150881652218966, -0.199176332305722,
0.680000507648084, 0.265540158726514, -0.692481536406930, -0.353516025899132, 0.548221585427517, 0.461960884430420,
0.983277137292909, 0.622653444743655, 0.480297639150934, 0.452168428463840, -0.823544922845037, -0.331975377216007,
0.358569546837979, -0.596899627797886, 0.301496127117938, 0.576480504549117, 0.150743674405126, -0.851734411492563,
0.127175797857987, 0.806251938739738, 0.360201361643523, 0.380717789492329, 0.759847187611911, -0.629616985016364,
0.179681096205048, 0.147705467636446, 0.793359439243774, -0.888943630149254, -0.768200353474536, 0.837243925007694,
0.601330942636268, -0.611155192762695, 0.176371159373866, 0.731621908979866, 0.294121659309740, 0.758972133758613,
0.226735505828943, -0.099879270352797, 0.716827038775917, -0.439818898014638, -0.845310989782625, 0.712989203634548,
0.040939664050535, -0.492177661587544, -0.611892969959734, -0.419016380589113, 0.526549046576405, -0.638706291747430,
0.611630316089939, -0.340452683930807, 0.833273571181310, 0.526974765301767, 0.462530036158461, -0.796029844105523,
0.604456871305559, 0.061317570172945, 0.642369077263937, -0.978365729885714, -0.468053997163355, 0.954769747262937,
0.446717706860744, -0.960283148953476, -0.751094692845392, 0.805721468166348, -0.290676951057867, 0.391488310207055,
0.099343420640492, 0.034684411420110, 0.463771493683684, -0.378392748975848, -0.830120263808777, -0.357574609519144,
0.755175701837563, -0.379417718802803, 0.295501263679325, 0.181793810178941, -0.116752805641118, -0.279166790263324,
0.938534135910232, 0.091496919686446, -0.107904426032210, -0.978447352332630, -0.180865072441111, 0.831093166130408,
0.099057141660298, 0.456832343000489, -0.652587456595693, 0.359989313981682, -0.951405226971930, 0.381285879196237,
0.182679971280062, -0.580757635269123, 0.879272667221741, -0.480336023774189, 0.833228235340136, .......................................
Low-level Data Format
WTF???

6. Samples ( not “examples” :) )
𝑆(𝑡) =
𝐴(𝑡)
𝐴 𝑚𝑎𝑥
𝑇𝑠𝑎𝑚𝑝𝑙𝑒 =
1
𝑓𝑠𝑎𝑚𝑝𝑙𝑒
𝑡 =
𝑁𝑠𝑎𝑚𝑝𝑙𝑒𝑠
𝑓𝑠𝑎𝑚𝑝𝑙𝑒

7. Sample analysis
https://github.com/corbanbrook/dsp.js/

8. Nyquist–Shannon sampling theorem:
𝑓𝑠𝑎𝑚𝑝𝑙𝑒 ≥ 2𝑓𝑚𝑎𝑥
Sample rate
http://bartus.org/akustyk/adc.html

9. audio.mozSetup(channels, sampleRate);
// …
audio.mozWriteAudio(samples1);
// …
audio.mozWriteAudio(samples2);
Writing audio

10. function AudioStream(readFn, bufSize) {
var audio = new Audio();
audio.mozSetup(1, sampleRate);
readFn = readFn.bind(audio, bufSize);
audio.writeAudio = function(soundData, callback) {
var written = this.mozWriteAudio(soundData),
length = soundData.length;
written < length
? setTimeout(this.writeAudio.bind(this, soundData.subarray(written), callback),
written ? 1000 * written / sampleRate : 0)
: callback(readFn());
}
readFn();
return audio;
}
Buffered writing

11. Main
Thread
Write
Audio
setTimeout
Read
Function
Calculation
Worker
Concurrency

12. Demonstration
http://rreverser.com/dev/piano/