Real-time Insights, powered by Reactive Programming5. Jay Phelps | @_jayphelps
127+ million
person-hours lost
per year
$500k — $1+ million
per hour
average hourly cost of
critical failure
https://devops.com/real-cost-downtime/
Fortune 500
8. Jay Phelps | @_jayphelps
debugging, testing, and even
Information Security too
11. Jay Phelps | @_jayphelps
debugging, testing, and InfoSec
12. Jay Phelps | @_jayphelps
InfoSec
“preventing unauthorized access, use, or
disruption of information”
17. Jay Phelps | @_jayphelps
We can block exploits
using our gateway proxy
18. Jay Phelps | @_jayphelps
...we need to know it's working...
19. Jay Phelps | @_jayphelps
...we want to watch attackers try...
21. Jay Phelps | @_jayphelps
We need real-time insights
For debugging, testing, and InfoSec
23. Jay Phelps | @_jayphelps
LOG ALL THE THINGSLOG ALL THE THINGSLOG ALL THE THINGSLOG ALL THE THINGSLOG ALL THE THINGS
26. Jay Phelps | @_jayphelps
We have thousands of servers...
27. Jay Phelps | @_jayphelps
We need to process them in real-time
28. Jay Phelps | @_jayphelps
“Netflix is a log generating company that
happens to stream movies” - Adrian Cockroft
29. Jay Phelps | @_jayphelps
Massiveamount of streaming logs
32. Jay Phelps | @_jayphelps
The best ideas from the Observer pattern, the
Iterator pattern, and functional programming
37. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
38. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
39. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
40. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
41. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
42. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
// 10
43. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
// 10
// 20
44. Jay Phelps | @_jayphelps
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
[1, 2, 3]
// 10
// 20
// 30
45. Jay Phelps | @_jayphelps
[1, 2, 3]
// 10
// 20
// 30
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
46. Jay Phelps | @_jayphelps
[1, 2, 3]
// 10
// 20
// 30
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
47. Jay Phelps | @_jayphelps
Observable.of(1, 2, 3)
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
48. Jay Phelps | @_jayphelps
Observable.of(1, 2, 3)
.map(value => value * 10)
.forEach(value => {
console.log(value);
});
49. Jay Phelps | @_jayphelps
Observable.of(1, 2, 3)
.map(value => value * 10)
.subscribe(value => {
console.log(value);
});
50. Jay Phelps | @_jayphelps
Observable.of(1, 2, 3)
.map(value => value * 10)
.subscribe(value => {
console.log(value);
});
51. Jay Phelps | @_jayphelps
Observable.of(1, 2, 3)
// 10
.map(value => value * 10)
.subscribe(value => {
console.log(value);
});
52. Jay Phelps | @_jayphelps
Observable.of(1, 2, 3)
// 10
// 20
.map(value => value * 10)
.subscribe(value => {
console.log(value);
});
53. Jay Phelps | @_jayphelps
Observable.of(1, 2, 3)
// 10
// 20
// 30
.map(value => value * 10)
.subscribe(value => {
console.log(value);
});
55. Jay Phelps | @_jayphelps
Observable is a collection that arrives over time
57. Jay Phelps | @_jayphelps
button.addEventListener('click', event => {
console.log('you clicked!');
});
58. Jay Phelps | @_jayphelps
Observable.fromEvent(button, 'click')
.subscribe(event => {
console.log('you clicked!');
});
61. Jay Phelps | @_jayphelps
Observable.fromEvent(button, 'click')
.debounceTime(500)
62. Jay Phelps | @_jayphelps
Observable.fromEvent(button, 'click')
.debounceTime(500)
63. Jay Phelps | @_jayphelps
Observable.fromEvent(button, 'click')
.debounceTime(500)
64. Jay Phelps | @_jayphelps
Observable.fromEvent(button, 'click')
.debounceTime(500)
65. Jay Phelps | @_jayphelps
Observable.fromEvent(button, 'click')
.debounceTime(500)
87. Jay Phelps | @_jayphelps
Observable.fromEvent(button, 'click')
.debounceTime(500)
89. Jay Phelps | @_jayphelps
{
"path": "/something",
"status": 500
}
Log message in JSON
90. Jay Phelps | @_jayphelps
import { webSocket } from "rxjs/observable/webSocket";
let streamOfLogs = webSocket("ws://logs.netflix.com")
93. 500 200 404 301
Jay Phelps | @_jayphelps
500
streamOfLogs
.filter(msg => msg.status !== 200)
404
97. Jay Phelps | @_jayphelps
Case Study
Scaling high volume logs in real-time
99. Jay Phelps | @_jayphelps
{
"path": "/something",
"status": 200
}
Device
{
"event": "StartPlay",
"device": "XBox 360"
}
Server
100. Jay Phelps | @_jayphelps
Stream them all through a single pipeline
101. Jay Phelps | @_jayphelps
How do you find and process the logs you want
104. Jay Phelps | @_jayphelps
{
"path": "/something",
"status": 500,
"duration": 135,
// etc ...
}
Server log message
105. Jay Phelps | @_jayphelps
streamOfLogs
.filter(event -> event.get("status") != 200)
.map(event ->
new JSONObject()
.put("path", event.get("path"))
.put("status", event.get("status"))
);
108. Jay Phelps | @_jayphelps
SELECT path,status WHERE status != 200
109. Jay Phelps | @_jayphelps
Just-in-time (JIT) Compilation
110. Jay Phelps | @_jayphelps
SELECT path,status,duration WHERE status != 200
111. Jay Phelps | @_jayphelps
SELECT path,status,duration WHERE status != 200
112. Jay Phelps | @_jayphelps
streamOfLogs
.filter(event -> event.get("status") != 200)
.map(event ->
new JSONObject()
.put("path", event.get("path"))
.put("status", event.get("status"))
);
SELECT path,status,duration WHERE status != 200
114. Jay Phelps | @_jayphelps
8+ million messages per second, peakmessagespersecond
Time
116. Jay Phelps | @_jayphelps
Distribute work via autoscaling
117. Jay Phelps | @_jayphelps
A
Source
B
Server
B
Server
50%
50%
Load balancing a job
119. Jay Phelps | @_jayphelps
SELECT path,status WHERE source == "API"
Chain jobs together
120. Jay Phelps | @_jayphelps
A
Job
B C
Job Job
Chain jobs together
121. Jay Phelps | @_jayphelps
High-volume, distributed systems have a problem...
123. Jay Phelps | @_jayphelps
“pressure opposed to the desired flow of gases in
confined places such as a pipe” - wikipedia.org
124. Jay Phelps | @_jayphelps
A B C
Server Server Server
100 rps 75 rps
125. Jay Phelps | @_jayphelps
A B C
Server Server Server
60 sec * 25 rps = 1,500 rpm!
100 rps 75 rps
126. Jay Phelps | @_jayphelps
A B C
Server Server Server
100 rps 75 rps
60 sec * 60 min * 25 rps = 90,000 rph!
132. Jay Phelps | @_jayphelps
1 2 3 4 5
.onBackpressureBuffer()
[3, 4, 5]
1 2
137. Jay Phelps | @_jayphelps
1 2 3 4 5
.onBackpressureDrop()
1 4
139. Jay Phelps | @_jayphelps
700+ jobs running 24/7
8+ million events/second
Autoscaling
Mantis
Low-latency, high throughput stream-processing job platform
http://techblog.netflix.com
140. Jay Phelps | @_jayphelps
We need somewhere to submit and view query results
142. Jay Phelps | @_jayphelps
SELECT path,status WHERE status != 200
153. Jay Phelps | @_jayphelps
Can be extremely high-volume
100k+ rps
154. Jay Phelps | @_jayphelps
We simply can’t render that fast!
155. Jay Phelps | @_jayphelps
Even if we could, it would be bad UX
156. Jay Phelps | @_jayphelps
Performance solutions are often driven by UX
158. Jay Phelps | @_jayphelps
UI virtualization aka Virtual Table?
160. Jay Phelps | @_jayphelps
We still can’t update that virtual
table 100k per second
164. Jay Phelps | @_jayphelps
getWebSocket()
.bufferTime(1000)
166. Jay Phelps | @_jayphelps
See what your users actually do
169. Jay Phelps | @_jayphelps
Drop after reaching a certain threshold
174. Jay Phelps | @_jayphelps
let buffer = getWebSocket()
.bufferTime(1000);
let gate = new BehaviorSubject(true);
let batchSize = 50;
let batchSizeCounter = 0;
let results = gate$
.switchMap(enabled => enabled ? buffer : Observable.never())
.do(buffer => {
batchSizeCounter += buffer.length;
if (batchSizeCounter >= batchSize) {
// truncates the array, if it's over batchSize
buffer.length = batchSize;
// turns on the gate, pausing the stream
gate.next(false);
batchSizeCounter = 0;
}
});
https://goo.gl/DMOqBA
175. Jay Phelps | @_jayphelps
let buffer = getWebSocket()
.bufferTime(1000);
let gate = new BehaviorSubject(true);
let batchSize = 50;
let batchSizeCounter = 0;
let results = gate$
.switchMap(enabled => enabled ? buffer : Observable.never())
.do(buffer => {
batchSizeCounter += buffer.length;
if (batchSizeCounter >= batchSize) {
// truncates the array, if it's over batchSize
buffer.length = batchSize;
// turns on the gate, pausing the stream
gate.next(false);
batchSizeCounter = 0;
}
});
https://goo.gl/DMOqBA
176. Jay Phelps | @_jayphelps
let buffer = getWebSocket()
.bufferTime(1000);
let gate = new BehaviorSubject(true);
let batchSize = 50;
let batchSizeCounter = 0;
let results = gate$
.switchMap(enabled => enabled ? buffer : Observable.never())
.do(buffer => {
batchSizeCounter += buffer.length;
if (batchSizeCounter >= batchSize) {
// truncates the array, if it's over batchSize
buffer.length = batchSize;
// turns on the gate, pausing the stream
gate.next(false);
batchSizeCounter = 0;
}
});
https://goo.gl/DMOqBA
177. Jay Phelps | @_jayphelps
let buffer = getWebSocket()
.bufferTime(1000);
let gate = new BehaviorSubject(true);
let batchSize = 50;
let batchSizeCounter = 0;
let results = gate$
.switchMap(enabled => enabled ? buffer : Observable.never())
.do(buffer => {
batchSizeCounter += buffer.length;
if (batchSizeCounter >= batchSize) {
// truncates the array, if it's over batchSize
buffer.length = batchSize;
// turns on the gate, pausing the stream
gate.next(false);
batchSizeCounter = 0;
}
});
https://goo.gl/DMOqBA
178. Jay Phelps | @_jayphelps
let buffer = getWebSocket()
.bufferTime(1000);
let gate = new BehaviorSubject(true);
let batchSize = 50;
let batchSizeCounter = 0;
let results = gate$
.switchMap(enabled => enabled ? buffer : Observable.never())
.do(buffer => {
batchSizeCounter += buffer.length;
if (batchSizeCounter >= batchSize) {
// truncates the array, if it's over batchSize
buffer.length = batchSize;
// turns on the gate, pausing the stream
gate.next(false);
batchSizeCounter = 0;
}
});
https://goo.gl/DMOqBA
179. Jay Phelps | @_jayphelps
let buffer = getWebSocket()
.bufferTime(1000);
let gate = new BehaviorSubject(true);
let batchSize = 50;
let batchSizeCounter = 0;
let results = gate$
.switchMap(enabled => enabled ? buffer : Observable.never())
.do(buffer => {
batchSizeCounter += buffer.length;
if (batchSizeCounter >= batchSize) {
// truncates the array, if it's over batchSize
buffer.length = batchSize;
// turns on the gate, pausing the stream
gate.next(false);
batchSizeCounter = 0;
}
});
https://goo.gl/DMOqBA
180. Jay Phelps | @_jayphelps
let buffer = getWebSocket()
.bufferTime(1000);
let gate = new BehaviorSubject(true);
let batchSize = 50;
let batchSizeCounter = 0;
let results = gate$
.switchMap(enabled => enabled ? buffer : Observable.never())
.do(buffer => {
batchSizeCounter += buffer.length;
if (batchSizeCounter >= batchSize) {
// truncates the array, if it's over batchSize
buffer.length = batchSize;
// turns on the gate, pausing the stream
gate.next(false);
batchSizeCounter = 0;
}
});
https://goo.gl/DMOqBA
181. Jay Phelps | @_jayphelps
Works for low-volume queries too
184. Jay Phelps | @_jayphelps
improved debugging, testing, and InfoSec
186. Jay Phelps | @_jayphelps
Rx is powerful, cross-platform