Talk about Node.js streams and related

1. Streams
FunctionalJs - London - May 2014
@darachennis

3. A <3able Stream
// by @TooTallNate - https://gist.github.com/TooTallNate/3947591
// npm install lame ; npm install speaker ; node mp3player.js
var fs = require('fs');
var lame = require('lame');
var Speaker = require('speaker');
fs.createReadStream(process.argv[2])
.pipe(new lame.Decoder())
.on('format', function (format) {
this.pipe(new Speaker(format));
});

4. A <3able Stream
// by @TooTallNate - https://gist.github.com/TooTallNate/3947591
// npm install lame ; npm install speaker ; node mp3player.js
var fs = require('fs');
var lame = require('lame');
var Speaker = require('speaker');
fs.createReadStream(process.argv[2])
.pipe(new lame.Decoder())
.on('format', function (format) {
this.pipe(new Speaker(format));
});

5. Sound Soup
var fs = require('fs');
var lame = require('lame');
var Speaker = require('speaker');
!
// Cache names of mp3's in cwd
function isMp3(e) { return e.indexOf("mp3") > 1 };
var soundz = fs.readdirSync('.').filter(isMp3);
!
// Every half second, play a random sound
setInterval(function(){
blend() },500);
function rand(min, max) {
var offset = min;
var range = (max - min) + 1;
return Math.floor( Math.random() * range) +
offset;
}
function blend() {
fs.createReadStream(soundz[rand(0,soundz.length-1)])
.pipe(new lame.Decoder())
.on('format', function (format) {
this.pipe(new Speaker(format));
});
}

6. A brief history in
Node.js streams

7. UNIXy
• UNIX $ a | b | c | d
• NODE.js> a.pipe(b).pipe(c).pipe(d)
• same as:
• a.pipe(b);
b.pipe(c);
c.pipe(d);
// but, less typing for the win o/

8. Streams 0
• Util.pump(Readable,Writable)

9. Streams 1
• EventEmitter and a pipe() function.
• pipe() is an instance of the Builder Pattern
• EventEmitter is an instance of Subject/Observer
Pattern

10. Streams 1: Example
var Stream = require(‘stream').Stream
var util = require('util')
function MyStream () {
Stream.call(this)
}
util.inherits(MyStream, Stream)
// stream logic and state management and back-pressure
// you’re on your own (with the rest of the pipeline)
// this was error prone and painful …

11. Streams 2
• EventEmitter and a pipe() function.
• Readable, Writable, Duplex, Transform,
Passthrough (and classic) ‘behaviours'
• New ‘behaviours’ encourage good ‘backpressure'
handling disciplines

12. Streams 2: Using Readable
// From: Stream Handbook, Readable with Push
var Readable = require('stream').Readable;
var rs = new Readable;
rs.push(‘beep ‘);
rs.push(‘boopn’);
rs.push(null);
rs.pipe(process.stdout);

13. Streams 2: Using Readable
// From: Stream Handbook
var Readable = require('stream').Readable;
var rs = Readable();
var c = 97;
rs._read = function () {
rs.push(String.fromCharCode(c++));
if (c > 'z'.charCodeAt(0)) rs.push(null);
};
rs.pipe(process.stdout); // _read called once, prints ‘a-z'

14. Streams 2: Pop Quiz!!!
$ node readable-popquiz.js
abcdefghijklmnopqrstuvwxyz%
$
!

15. Streams 2: Pop Quiz!!!
// From: Stream Handbook (tweaked)
var Readable = require('stream').Readable;
var rs = Readable();
var c = 97;
rs._read = function () {
rs.push(String.fromCharCode(c++));
if (c > 'z'.charCodeAt(0)) rs.push(null);
};
rs.pipe(process.stdout); // _read called once
rs.pipe(process.stdout); // _read called twice
rs.pipe(process.stdout); // _read called three times
rs.pipe(process.stdout); // _read called four times

16. Streams 2: Pop Quiz!!!
$ node readable-popquiz.js
aaaabbbbccccddddeeeeffffgggghhhhiiiijjjjkkkkl
lllmmmmnnnnooooppppqqqqrrrrssssttttuuuuvvvvww
wwxxxxyyyyzzzz%
$
… WAT? (Think about it)
!

17. Streams 2: Using Writable
// From: Stream Handbook
var fs = require('fs');
var ws = fs.createWriteStream('message.txt');
ws.write('beep ‘); // Write …
setTimeout(function () {
ws.end(‘boopn'); // We’re done writing
}, 1000);

18. Streams 2: Using Writable
var Writable = require('stream').Writable;
var src = process.stdin;
var dst = new Writable;
dst._write = function (chunk, enc, next) {
console.log(chunk.toString());
next();
};
src.push("hit any key!");
src.pipe(dst);

19. Streams 2: Transform
var Transform = stream.Transform || require('readable-stream').Transform;
function Hash(algorithm, encoding, options) {
Transform.call(this, options);
this.digester = crypto.createHash(algorithm);
this.encoding = encoding;
}
util.inherits(Hash, Transform);
Hash.prototype._transform = function (chunk, enc, cb) {
var bf = (Buffer.isBuffer(chunk)) ? chunk : new Buffer(chunk, enc);
this.digester.update(bf);
cb();
};
Hash.prototype._flush = function (cb) {
this.push(this.digester.digest(this.encoding));
cb();
};

20. Streams 2: Transform
var hash = require('./hash.js');
var fs = require('fs');
var rs = fs.createReadStream(‘file.js’);
var md5Hash = hash.Hash('md5', 'base64');
// var sha1Hash = hash.Hash('sha1', 'base64');
rs.pipe(md5Hash).pipe(process.stdout);

21. Streams 2: Duplex
* Independent Readable and Writable channels
* Implement _read and _write
* Example use
- network protocols
- serial communications with hardware
- …

22. Know a classic stream
// From: Stream Handbook
process.stdin.on('data', function (buf) {
console.log(buf);
});
process.stdin.on('end', function () {
console.log('__END__');
});

23. Streams 3
• Streams2 tweaked
• API the same as Streams2

24. Useful modules
• @DominicTarr’s event-stream
https://github.com/dominictarr/event-stream
• @RVagg’s through2
https://github.com/rvagg/through2
• @Raynos’s duplexer
https://github.com/Raynos/duplexer

25. • https://github.com/substack/stream-handbook -
@SubStack’s Stream handbook
• http://www.nearform.com/nodecrunch/dont-use-
nodes-core-stream-module#.U4I71JRdWuk - Why
@Rvagg doesn’t use node’s core stream module (&
nor should we!)
Good to know …

26. Meta Programming with Streams
and Pipes
Great British Node Conference
October 8th 2013
London
!
Darach Ennis

27. Streams 101
! Readable Streams
! IO / throughput oriented
! Events – data, end, error, close
! Methods – pause, resume, end, destroy
! Writeable Streams
! IO / throughput oriented
! Events - drain, pause, resume, error, close
! Methods – write, end, destroy

28. Streams … 4IO
! IO Oriented
! Finite, not infinite
! Asynchronous
! Lazy
! Assumes in transit data, not in memory forms
! Great for IO. Not efficient for compute

29. CSV Reader .@maxogden

30. Beams
Streams for Compute

31. Beams 101
! Sources
! Where data pours in
! Sinks
! Where results pop out
! Operators
! Filters – Drop or pass on data based on a UDF
! Transforms – Mutate data based on a UDF
! Branch with Filters
! Combine with Sinks

32. Beams… 4CPU
! Compute Oriented
! Infinite, not finite
! Synchronous
! Eager
! Assumes in memory form, not encoded or wire data
! Convenient for compute. Not good for IO

33. Beams - Branch

34. Beams - Combine

35. Ex – Fly NodeCopter with Streams!

36. Extend Games to Robots!

37. Extend?
Meta Programming

38. A minor problem
! Eager: a.pipe(operation).pipe(z).pipe(a);
! Implies:
RangeError: Maximum call stack size exceeded

39. Goto + Modules = Magic

40. Extend Beam with Modules

41. Questions & Thanks
! npm install beam
! Compute oriented streams
! Branch, Combine, Filter, Transform data
! Extend with goto and modules.
! npm install eep
! Beam will be integrated into eep soon
! Eep provides aggregate windows
! Slide, Tumble, Monotonic, Periodic
! Stats Library
! Ordering now supported
! npm install nodesapiens
! Robosapiens V1 for node.js
! Code on github
! https://github.com/darach

42. Dojo - nodeschool.io
• Basics
• Stream Adventure
• Functional Javascript