Visual Exploration of Large Data sets with D3, crossfilter and dc.js

Florian Georg - IBM Switzerland
florian.georg@ch.ibm.com

Software Solution Architect @ IBM
Software Delivery - practices and tools
Software-driven Product Innovation
Agile, Lean and Design Thinking
Cloud development (IBM Bluemix)
Analytics, Systems and IoT

https://internetofthings.ibmcloud.com/

not just static reporting
not just basic charting
not a full-fledged BI suite
even on larger datasets

Data-Driven Documents
http://d3js.org/

https://github.com/mbostock/d3/wiki/Gallery
http://bl.ocks.org/mbostock
(... lots of !)

http://bl.ocks.org/mbostock/1005873

var theData = [ "Jazoon", "D3", "IBM", "Bluemix" ];
var p = d3.select("body")
.selectAll("p") // DOM element selector
.data(theData) // join/bind to data
.enter() // for each "new" data point
.append("p") // ... add some DOM elements
.text(function (d){
return "Hello " + d;
})
.style("background-color", "steelblue");
<html>
<body>
<script src="http://d3js.org/d3.v3.min.js">

http://bost.ocks.org/mike/join/

var data = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
var nodesLinear = d3.select("div#linear").selectAll("p");
// Scales map domain into a range.
// domains / ranges can be numbers, colors, timestamps etc.
// D3 supports various scales and interpolators for the mapping,
// and you can write your own...
var color = d3.scale.linear()
.domain([0, Math.max.apply(Math, data)])
.range(["yellow", "red"]);
nodesLinear.data(data).enter()
.append("p")
.text(function(d){
return "linear interpolation " + d + " ==> " + color(d);})
.style("background-color", color);
http://jsbin.com/segoyu/3/edit

// make sure you understand the data structure that is
// expected by the layout
var root = {
children: [
{children: [
{value: 42},
{value: 10, omit: 1}
]
},{value: 42},{value: 10, omit: 1}
]
};
var width = 480,height = 300;
var svg = d3.select("body").append("svg")
.attr("width", width)
.attr("height", height)
// pack layout (can be used for bubblecharts etc.)
var pack = d3.layout.pack().size([width, height]);
svg.selectAll("circle")
// bind to your data, "decorated" by the layout.
.data( pack.nodes(root).filter(function(d) { return !d.omit; }) )
.enter().append("circle")
// the layout knows where to position elements,
// and adds positioning attributes like "x", "y" etc. to your data
// (read the docs to avoid surprises)
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; })
.attr("r", function(d) { return d.r; });
http://jsbin.com/dequma/8/edit

Pure HTML5 / JavaScript
Supports SVG and
Canvas
Very flexible
Mature project
Excellent documentation
Wide adoption

learning curve
quite low-level
Integration with UI Frameworks
(JQuery, Angular ...)
not scaling well to large datasets

http://square.github.io/crossfilter/

// A small sample of the classic cars dataset
var csv = "name,economy (mpg),cylinders,displacement (cc),power (hp),weight (lb),0-60 mph (s),yearn" +
"AMC Ambassador Brougham,13,8,360,175,3821,11,73n" +
"AMC Ambassador DPL,15,8,390,190,3850,8.5,70n" +
"AMC Ambassador SST,17,8,304,150,3672,11.5,72n" +
"AMC Concord DL 6,20.2,6,232,90,3265,18.2,79n" +
"AMC Concord DL,18.1,6,258,120,3410,15.1,78n" +
"Audi Fox,29,4,98,83,2219,16.5,74n" +
"BMW 2002,26,4,121,113,2234,12.5,70n" +
"BMW 320i,21.5,4,121,110,2600,12.8,77n" +
"Buick Century 350,13,8,350,175,4100,13,73n" +
"Buick Century Limited,25,6,181,110,2945,16.4,82n" +
"Chrysler Newport Royal,13,8,400,190,4422,12.5,72n" +
"Citroen DS-21 Pallas,6,4,133,115,3090,17.5,70n" +
"Datsun 1200,35,4,72,69,1613,18,71";

var cf = crossfilter(data);
var byCylinders = cf.dimension(function(p) {
return p.cylinders;
});
// create a groups per number of cylinders
var groupByCylinders = byCylinders.group();
var visCylinders = d3.select("body")
.select("div#byCylinders")
.selectAll("div")
.data(groupByCylinders.top(Infinity)) // data: all elements inside group
.enter()
.append("div")
.text(function(d){
// group is 'key', 'value' depends on the reduce function (default: count)
return "Cars with " + d.key + " cylinders: " + d.value;
});
http://jsbin.com/biruro/6/edit

Coordinated views on data ("brushing")
Incremental filtering
Extremely fast for histograms, top-n lists etc.

Dimensional data (not timeseries, continuous)
One filter per dimension
slow write / fast read operations
Few examples out there
Load & Explore (e.g. not "realtime analysis")
Dimensions are expensive (32 max)
Doesn't like NaN

// [ ... ]
var dateDim = crossfilter.dimension(function(d) {return d.date;});
var hits = dateDim.group().reduceSum(function(d) {return d.total;});
var minDate = dateDim.bottom(1)[0].date;
var maxDate = dateDim.top(1)[0].date;
var status_200=dateDim.group().reduceSum(function(d) {return d.http_200;});
dc.lineChart("#chart-line-hitsperday")
.width(500).height(200)
.dimension(dateDim)
.group(status_200,"200")
.stack(status_302,"302")
.stack(status_404,"404")
.renderArea(true)
.x(d3.time.scale().domain([minDate,maxDate]))
.brushOn(false)
.legend(dc.legend().x(50).y(10).itemHeight(13).gap(5))
.yAxisLabel("Hits per day");
http://jsbin.com/zapupe/4/edit

JSON is verbose
server-side preprocessing
dimension.dispose()
DC uses SVG (not Canvas)
multiple "simple" visualizations > one "complex"

Demo, slides, links etc:
http://datavisual.mybluemix.net

http://www.ibm.com/developerworks/
@florian_georg
florian.georg@ch.ibm.com

Visual Exploration of Large Data sets with D3, crossfilter and dc.js

Recommended

Recommended

More Related Content

What's hot

What's hot (20)

Similar to Visual Exploration of Large Data sets with D3, crossfilter and dc.js

Similar to Visual Exploration of Large Data sets with D3, crossfilter and dc.js (20)

More from Florian Georg

More from Florian Georg (9)

Recently uploaded

Recently uploaded (20)

Visual Exploration of Large Data sets with D3, crossfilter and dc.js