This is the extended (full) version of the talk given at HTML5DevConf 2014. A condensed version of this talk was previously given at NodeConfEU 2014. At Netflix we run hundreds of A/B tests every year. Maintaining multivariate experiences quickly adds strain to any UI engineering team. Join us to explore the patterns we’ve built in Node.js to tame this beast - ultimately enabling quick feature development and rapid test iteration on our service used by over 50 million people around the world. Video from NodeConfEU: https://www.youtube.com/watch?v=gtjzjiTI96c

1. Scaling A/B testing
on Netflix.com with
_________ Alex Liu
@stinkydofu

2. data driven
product development

7. Test 1 Test 2 Test 3 Test 4 Test 5 Test 6 Test 7
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G
A
B
C
D
E
F
G

8. 2,097,152
unique experiences across
seven tests

9. hundreds
of new A/B tests per year

10. 43351892955034
94860861172181
85493567650…
72061153709996

11. 2105 566 685
templates CSS JS

12. 2.5M
unique packages
every week

13. <html/>
<link/>
<script/>

14. problem:
conditional dependencies

16. ▶ Templating
▶ Packaging
▶ Bonus Round

17. Templating

19. payment.dust
<div id="payments">
<input id="first-name"><input id="last-name">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="card-number"><input id="security-code">
<select name="month"></select><select name="year"></select>
<checkbox id="agree-to-terms"/>
<button>Start Your Trial</button>
</div>

22. <input id=“first-name"><input id=“last-name">
{@inTest id="10" cell=“2a"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class=“payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
{@inTest id="10" cell=“3"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
{@inTest id="10" cell=“4"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
{@inTest id="10" cell=“5"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
<checkbox id="agree-to-terms"></checkbox>
<button>Start Your Trial</button>
</div>

23. payment.dust
if if if if if
Control Cell 2 Cell 3 Cell 4 Cell 5

24. <div class="payment-types">
<div id="CC">
{> payment_type_cc /}
</div>
<div id="DD">
{> payment_type_dd /}
</div>
</div>

25. payment.dust
if if if if if
Control Cell 2 Cell 3 Cell 4 Cell 5
payment_type_cc.dust payment_type_dd.dust

26. payment.dust
Control.dust
if if if if if
Cell2.dust Cell3.dust Cell4.dust Cell5.dust
payment_type_cc.dust payment_type_dd.dust

27. <div id="payments">
<input id="first-name">
<input id="last-name">
{> payment_method /}
<input type="checkbox" id="terms">
<button>Start Your Trial</button>
</div>

28. payment.dust
?
payment.json
Control.dust Cell2.dust Cell3.dust Cell4.dust Cell5.dust
payment_type_cc.dust payment_type_dd.dust

29. payment.json
{
"rules": [],
"templateName": "control"
},
{
"rules": ["PaymentTest(2)"],
"templateName": "payment_cell2"
},
{
"rules": ["PaymentTest(3)"],
"templateName": "payment_cell3"
},
{
"rules": ["PaymentTest(4)"],
"templateName": "payment_cell4"
},
{
"rules": ["PaymentTest(5)"],
"templateName": "payment_cell5"
}

30. require('nf-rule-infrastructure')

31. anatomy of a rule
var Rule = require('nf-rule-infrastructure'),
PaymentTest;
PaymentTest = new Rule('PaymentTest', function(context, params, cb) {
var test = context.abtests.get(10);
cb(test && test.cell(params.id));
});
module.exports = PaymentTest;

32. require('nf-template-resolver')

33. payment.dust dustjs partial

34. resolver payment.json
(mappings)
rule

35. rules
control.dust
cell2.dust
cell3.dust

36. payment.dust dustjs resolver

37. <input id=“first-name"><input id=“last-name">
{@inTest id="10" cell=“2a"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class=“payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
{@inTest id="10" cell=“3"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
{@inTest id="10" cell=“4"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
{@inTest id="10" cell=“5"}
<div class="payment-types">
<div id="CC">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id=“card-number"><input id=“security-code”>
</div>
<div id="DD">
<span class="payment-logos">
<img src="logo1.png"><img src="logo2.png">
<img src="logo3.png"><img src="logo4.png">
</span>
<input id="CPF"><input id="bank-name">
<input id="branch-number"><input id="account-number">
<input id="digit">
</div>
</div>
{/@inTest}
<checkbox id="agree-to-terms"></checkbox>
<button>Start Your Trial</button>
</div>

38. <div id="payments">
<input id="first-name">
<input id="last-name">
{> payment_method /}
<input type="checkbox" id="terms">
<button>Start Your Trial</button>
</div>

39. Wins
▶ combine rules
▶ improve template legibility
▶ increase template reuse

40. ▶ Templating
▶ Packaging
▶ Bonus Round

41. Packaging

42. everything
is a module

45. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

46. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

47. app.js
import jquery from 'jquery';
import oldSearch from 'oldSearch';
import newSearch from 'newSearch';
export ...

48. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

50. 685 files…?
2.5M packages…?

51. oldSearch
app.js
newSearch
dep1 dep2 dep3 dep4 dep5
sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep

52. problem:
conditional dependencies

54. build request

55. require('nf-include-when')

56. oldSearch.js
/*
* @includewhen rule.notInNewSearch
*/

57. newSearch.js
/*
* @includewhen rule.inNewSearch
*/

58. anatomy of a rule
var Rule = require('nf-rule-infrastructure'),
inNewSearch;
inNewSearch = new Rule('inNewSearch', function(context, cb) {
var test = context.abtests.get(1534);
cb(test && test.cell(1));
});
module.exports = inNewSearch;

59. require('nf-asset-registry')

60. app.js
import jquery from 'jquery';
import oldSearch from 'oldSearch';
import newSearch from 'newSearch';
export ...

61. app.js
jquery
oldSearch.js
newSearch.js
registry

62. "app.js": {
"deps": [
"jquery",
"oldSearch.js",
"newSearch.js",
],
"depsFull": [
"jquery",
"oldSearchDep2.js",
"oldSearchDep1.js",
"oldSearch.js",
"newSearchDep2.js",
"newSearchDep1.js",
"newSearch.js"
]
}

63. "newSearch.js": {
"rule": "inNewSearch",
"deps": [
"jquery",
"newSearchDep2.js",
"newSearchDep1.js",
],
"depsFull": [
"jquery",
"newSearchSubDep3.js",
"newSearchSubDep2.js"
"newSearchSubDep1.js"
"newSearchDep2.js",
"newSearchDep1.js"
]
}
nf-include-when

64. require('nf-packager')

65. var packager = require('nf-packager'),
includeWhen = require('nf-include-when'),
registries = require('nf-asset-registry');
function getScriptUrl()
return packager.getPackageDefinition('app.js',
registries,
includeWhen);
}

66. "app.js": {
"deps": [
"jquery",
"oldSearch.js",
"newSearch.js",
],
"depsFull": [
"jquery",
"oldSearchDep2.js",
"oldSearchDep1.js",
"oldSearch.js",
"newSearchDep2.js",
"newSearchDep1.js",
"newSearch.js"
],
"fileSize": "4.41 kB",
"fileSizeFull": "120.52 kB"
}
Step 1:
Get the full dependency tree for the
requested package from the registry.

67. [
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 2:
Determine which files have rules.

68. [
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 3:
Run the rules. Filter out all deps that
resolved false.
✓

69. [
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 4:
Filter out all extraneous
sub deps.
✓

70. Step 5:
Concatenate the files.
[
"jquery", /* no rule */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]

71. build
javascript
registry

72. request registry
packager rules

73. Wins
▶ leverage build time tools
▶ leverage the server
▶ divide and conquer with modules

74. ▶ Templating
▶ Packaging
▶ Bonus Round

75. Bonus Round

76. be creative
with the registry

77. "account/bb/models/ratingHistoryModel.js": {
"rule": null,
"deps": [...],
"depsFull": [...],
"depsCount": {
"underscore": 2,
"backbone": 1,
"jquery": 2,
"common/requirejs-plugins.js": 4,
"requirejs-text": 4,
"utils/contextData.js": 1,
"common/nfNamespace.js": 1
},
"hash": "dd23b163",
"fileSize": "1.21 kB",
"fileSizeFull": "173.04 kB"
}
dependency
counting
dependency
pruning
file sizes

78. @import modules
@import (reference) "/common/_nf_defs.less";
@import (reference) "/member/memberCore.less";
@import (reference) "/components/menu.less";
@import (reference) "/components/breadcrumbs.less";

79. "account/containerResponsive.css": {
"rule": null,
"deps": [...],
"depsFull": [...],
"depsCount": [...],
"hash": "65a431f3",
"fileSize": "709 B",
"fileSizeFull": "709 B",
"css": {
"selectors": 8,
"declarationBlocks": 6,
"declarations": 17,
"mediaQueries": 3
}
}
css
analysis

80. the
best part

81. "cache": {
"account/pin.js": "define('account/pin.js', ['member/memberC…",
"account/bb/models/changePlanModel.js": "define('account/b…",
"account/bb/models/ratingHistoryModel.js": "define('account…",
"account/bb/models/viewingActivityModel.js": "define('account…",
"account/bb/views/changePlanView.js": "define('account/bb/vi…",
"account/bb/views/changePlanView.js": "define('account/bb/vi…",
"account/bb/views/emailSubView.js": "define('account/bb/views…",
"account/bb/views/viewingActivityView.js": "define('account…",
"common/UITracking.js": "define('common/UITracking.js, ['me…",
"common/UITrackingOverlay.js": "define('common/UITrackingOve…",
…
…
…

82. templates templates
mappings
javascript
css
mappings
javascript
css

83. templates
mappings
javascript
css
UI Bundle

85. deploy UI bundles
anytime

86. never
touch the file system

87. < 5ms
package response times

88. Wins
▶ static analysis FTW
▶ independent UI deployments
▶ requests never touch the fs
▶ fast package response times

89. Our Learnings

90. learn by doing

91. fail fast
move faster

92. “I have not failed.
I’ve just found 10,000 ways
that won’t work.”
Thomas Edison

93. simplify

95. thank you
Alex Liu
@stinkydofu

96. questions?
Alex Liu
@stinkydofu