1. STABILIZER
[ASPLOS 2013]
Emery Berger, Charlie Curtsinger
Statistically Sound
Performance Evaluation

2. We all care about performance evaluation
We’ve been doing it wrong
STABILIZER
Repeated runs and error bars not enough
We’re not measuring what we thought

3. changing a program changes its layout
STABILIZER
Memory layout affects performance
STABILIZER eliminates the effect of layout
no way to measure effect of change in isolation
evaluation of LLVM’s optimizations with STABILIZER
Case Studies
enables sound performance evaluation
We’ve been doing it wrong

4. STABILIZER
We’ve been doing it wrong
changing a program changes its layout
Memory layout affects performance
STABILIZER eliminates the effect of layout
no way to measure effect of change in isolation
evaluation of LLVM’s optimizations with STABILIZER
Case Studies
enables sound performance evaluation

5. A
Unsound performance
evaluation
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}

6. int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
A
Unsound performance
evaluation
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}

7. int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
A
Unsound performance
evaluation
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
size_t meaning_of_life=42;
for (size_t i = 0; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
for (size_t i = 16; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}

8. int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
A
Unsound performance
evaluation
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
size_t meaning_of_life=42;
for (size_t i = 0; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
for (size_t i = 16; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}

9. int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
A
Unsound performance
evaluation
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
size_t meaning_of_life=42;
for (size_t i = 0; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
for (size_t i = 16; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
asm("isync");
}

10. int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
A
Unsound performance
evaluation
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
size_t meaning_of_life=42;
for (size_t i = 0; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
for (size_t i = 16; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
asm("isync");
}

11. A′
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
A
Unsound performance
evaluation
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
size_t meaning_of_life=42;
for (size_t i = 0; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
for (size_t i = 16; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
asm("isync");
}

12. A′A
Unsound performance
evaluation
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
size_t meaning_of_life=42;
for (size_t i = 0; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
for (size_t i = 16; i < size; i += 32) {
asm("icbi 0,%0" : : "r"(p));
p += 32;
}
asm("isync");
}

13. 0.00
0.25
0.50
0.75
1.00
85.0 87.5 90.0 92.5 95.0
Time (s)
Numberofruns
Version
A
A'
A′A ×1 ×1
Which is faster?

14. 0.00
0.25
0.50
0.75
1.00
85.0 87.5 90.0 92.5 95.0
Time (s)
Numberofruns
Version
A
A'
Is faster than ?A′ A

15. 0.00
0.25
0.50
0.75
1.00
85.0 87.5 90.0 92.5 95.0
Time (s)
Numberofruns
Version
A
A'
Is faster than ?A′ A

16. 0.00
0.25
0.50
0.75
1.00
85.0 87.5 90.0 92.5 95.0
Time (s)
Numberofruns
Version
A
A'
Is faster than ?A′ A

17. 0.00
0.25
0.50
0.75
1.00
85.0 87.5 90.0 92.5 95.0
Time (s)
Numberofruns
Version
A
A'
Is faster than ?A′ A
what about
variance?
2.8% faster

18. 0
5
10
15
85.0 87.5 90.0 92.5 95.0
Time (s)
Numberofruns
Version
A
A'
Which is faster?
A′A ×30 ×30

19. 0
5
10
15
85.0 87.5 90.0 92.5 95.0
Time (s)
Numberofruns
Version
A
A'
Is faster than ?A′ A
still
2.8% faster

20. Why is faster than ?A′ A

21. Why is faster than ?A′ A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}

22. Why is faster than ?A′ A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
Was it the
code change?

23. Why is faster than ?A′ A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
Or was it the
new layout?

24. Why is faster than ?A′ A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
Mytkowicz et al. (ASPLOS’09)
Layout biases measurement

25. Layout biases measurement
Mytkowicz et al. (ASPLOS’09)
Link Order
Environment
Variable Size
Changes function addresses
Moves the program stack

26. Layout biases measurement
Mytkowicz et al. (ASPLOS’09)
Link Order
Environment
Variable Size
Changes function addresses
Moves the program stack
Larger than
impact of -O3

27. Blame the cache
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
A

28. Blame the cache
A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
conflict
map to same
cache set

29. A′
Blame the cache
A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}

30. Blame the cache
A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
map to
same set
A′
Nothing here
no conflict

31. A′
Blame the cache
A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
or branch predictor
or TLB
or prefetcher
or branch
target predictor

32. Blame the hash
A
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
int main(int argc, char **argv) {
topFrame = (void**)__builtin_frame_address(0);
setHandler(Trap::TrapSignal, onTrap);
setHandler(SIGALRM, onTimer);
setHandler(SIGSEGV, onFault);
for(Function* f: functions) {
f->setTrap();
}
setTimer(interval);
int r = stabilizer_main(argc, argv);
return r;
}
void setTimer(int msec) {
struct itimerval timer;
timer.it_value.tv_sec = (msec - msec % 1000) / 1000;
timer.it_value.tv_usec = 1000 * (msec % 1000);
timer.it_interval.tv_sec = 0;
timer.it_interval.tv_usec = 0;
setitimer(ITIMER_REAL, &timer, 0);
}
static void flush_icache(void* begin, size_t size) {
uintptr_t p = (uintptr_t)begin & ~15UL;
for (size_t i = 0; i < size; i += 16) {
asm("icbi 0,%0" : : "r"(p));
p += 16;
}
asm("isync");
}
DataHeapType* getDataHeap() {
static char buf[sizeof(DataHeapType)];
static DataHeapType* _theDataHeap = new (buf) DataHeapType;
return _theDataHeap;
}
A′

33. it’s faster it’s faster
it’s faster
it’s faster
Is faster than ?A′ A
Let’s do a poll

34. Do we trust this?
it’s faster it’s faster
it’s faster
it’s faster

35. it’s faster it’s slower they’re the same
Is faster than ?A′ A

36. it’s faster it’s slower they’re the same
But it ran faster!
What if we only talk to Bob?

37. it’s faster
But it ran faster!
What if we only use this layout?
it’s slower they’re the same

38. it’s faster
But it ran faster!
What if we only use this layout?

39. it’s faster
But it ran faster!
What if we only use this layout?
Upgrade libc
Changes layout

40. it’s faster
But it ran faster!
What if we only use this layout?
Change Username
Changes layout

41. Layout is Brittle
it’s faster
But it ran faster!
What if we only use this layout?
Run in a new
directory
Changes layout

42. Layout is Brittle
But it ran faster!
What if we only use this layout?
Layout biases measurement
Mytkowicz et al. (ASPLOS’09)
Can we eliminate the
effect of layout?

43. But it ran faster!
What if we only use this layout?
Layout biases measurement
Can we eliminate the
effect of layout?
YES

44. STABILIZER
Memory layout affects performance
STABILIZER eliminates the effect of layout
enables sound performance evaluation
evaluation of LLVM’s optimizations with STABILIZER
Case Studies
makes performance evaluation difficult

45. STABILIZER
Memory layout affects performance
STABILIZER eliminates the effect of layout
enables sound performance evaluation
evaluation of LLVM’s optimizations with STABILIZER
Case Studies
makes performance evaluation difficult

46. Layout biases measurement
STABILIZER
function addresses stack frame sizes
heap allocations
randomizes layout

47. STABILIZER
randomizes layout
function addresses stack frame sizes
heap allocations
repeatedly
Layout biases measurement
during
execution

48. STABILIZER
randomizes layout
function addresses stack frame sizes
heap allocations
repeatedly
Layout biases measurement
a completely random layout
cannot bias results

49. A′A ×30 ×30
Sound Performance
Evaluation

50. A ×30 ×30
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
A′
Sound Performance
Evaluation

51. Is faster than ?A′ A
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes

52. Is faster than ?A
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
A′

53. Is faster than ?A
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
A′

54. Is faster than ?A
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
A′

55. AA′
what is the probability of measuring
a speedup this large by chance?
The Statistical Approach
If =
hypothesis testing

56. what is the probability of measuring
a speedup this large by chance?
easy to compute for
the normal distribution
AA′If =

57. easy to compute for
the normal distribution
AA′If =

58. STABILIZER
randomizes layoutrepeatedly
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
what is the probability of measuring
a speedup this large by chance?
if there is a
low probability

59. STABILIZER
randomizes layoutrepeatedly
this speedup is real
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
not due to the effect
on memory layout

60. this speedup is real
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
Sound Performance
Evaluation
STABILIZER
randomizes layoutrepeatedly
what does
re-randomization do?
not due to the effect
on memory layout

61. 0.0
0.2
0.4
350 360 370 380 390 400
Time (s)
ProbabilityDensity
STABILIZER
randomizes layoutrepeatedly
one random
layout per-run

62. STABILIZER
randomizes layoutrepeatedly
0.0
0.2
0.4
350 360 370 380 390 400
Time (s)
ProbabilityDensity
many random
layouts in each run
one random
layout per-run

63. STABILIZER generates a new
random layout every ½ second
Total execution time is
the sum of all periods
STABILIZER
randomizes layoutrepeatedly

64. STABILIZER generates a new
random layout every ½ second
Total execution time is
the sum of all periods
The sum of a sufficient number of
independent, identically distributed random
variables is approximately normally distributed.
STABILIZER
randomizes layoutrepeatedly

65. STABILIZER generates a new
random layout every ½ second
Total execution time is
the sum of all periods
The sum of a sufficient number of
independent, identically distributed random
variables is approximately normally distributed.
STABILIZER
randomizes layoutrepeatedly

66. STABILIZER generates a new
random layout every ½ second
Total execution time is
the sum of all periods
The sum of a sufficient number of
independent, identically distributed random
variables is approximately normally distributed.
STABILIZER
randomizes layoutrepeatedly

67. Central Limit Theorem
execution times are
normally distributed
The sum of a sufficient number of
independent, identically distributed random
variables is approximately normally distributed.

68. STABILIZER
Memory layout affects performance
STABILIZER eliminates the effect of layout
enables sound performance evaluation
evaluation of LLVM’s optimizations with STABILIZER
Case Studies
makes performance evaluation difficult

69. STABILIZER
makes performance evaluation difficult
Memory layout affects performance
STABILIZER eliminates the effect of layout
enables sound performance evaluation
evaluation of LLVM’s optimizations with STABILIZER
Case Studies

70. Case Studies
on each benchmark
across the whole
benchmark suite
evaluation of LLVM’s optimizations with STABILIZER
first, build benchmarks with STABILIZER

71. Build programs with
STABILIZER
> szc main.c

72. > szc main.c
Build programs with
STABILIZER

73. > szc main.c-Rcode
Build programs with
STABILIZER

74. > szc main.c-Rcode -Rheap -Rstack
Build programs with
STABILIZER
now run the benchmarks

75. 0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
Run benchmarks as usual
A′A ×30 ×30
drop the results into R

76. Is faster than ?A
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
A′

77. A′
0%
10%
20%
30%
40%
85.0 87.5 90.0 92.5 95.0
Time (s)
PercentofObservedRuntimes
If = A

78. If =A′ A

79. A′If = A
what is the probability of measuring a
difference at least this large?

80. what is the probability of measuring a
difference at least this large?
A′
The Student’s t-test
If = A
t.test(times.A′, times.A)

81. what is the probability of measuring a
difference at least this large?
A′If = A
The Student’s t-test
If p-value

82. A′
The Student’s t-test
If p-value ≤ 5%
If = A
95% Confidence
what is the probability of measuring a
difference at least this large?

83. If =A′ A
The Student’s t-test
If p-value ≤ 5%
we reject the null hypothesis
what is the probability of measuring a
difference at least this large?

84. ≠ A
The Student’s t-test
If p-value ≤ 5%
we reject the null hypothesis
Random chance not responsible for
the measured difference
A′

85. ≠ A
The difference is real
A′

86. −10%
0%
10%
20%
libquantum
m
ilc
bzip2sphinx3
nam
d
lbmperlbenchhm
m
erh264ref
cactusA
DM
w
rf
sjenggobm
kgrom
acszeusm
p
m
cf
gcc
astar
Speedup
Significant
Yes
No
Speedup of -O2 over -O1

87. −10%
0%
10%
20%
libquantum
m
ilc
bzip2sphinx3
nam
d
lbmperlbenchhm
m
erh264ref
cactusA
DM
w
rf
sjenggobm
kgrom
acszeusm
p
m
cf
gcc
astar
Speedup
Significant
Yes
No
Speedup of -O2 over -O1

88. −10%
0%
10%
20%
libquantum
m
ilc
bzip2sphinx3
nam
d
lbmperlbenchhm
m
erh264ref
cactusA
DM
w
rf
sjenggobm
kgrom
acszeusm
p
m
cf
gcc
astar
Speedup
Significant
Yes
No
Speedup of -O2 over -O1

89. −10%
0%
10%
20%
libquantum
m
ilc
bzip2sphinx3
nam
d
lbmperlbenchhm
m
erh264ref
cactusA
DM
w
rf
sjenggobm
kgrom
acszeusm
p
m
cf
gcc
astar
Speedup
Significant
Yes
No
Speedup of -O2 over -O1

90. −10%
0%
10%
20%
bzip2gobm
kzeusm
p
libquantum
w
rf
astar
m
cfhm
m
er
m
ilc
nam
d
gcc
lbmgrom
acsh264ref
cactusA
DMperlbenchsphinx3
sjeng
Speedup
Significant
Yes
No
Speedup of -O3 over -O2

91. Speedup of -O3 over -O2
0.0%
0.5%
1.0%
1.5%
bzip2gobm
kzeusm
p
libquantum
w
rf
astar
m
cfhm
m
er
m
ilc
nam
d
gcc
lbmgrom
acsh264ref
cactusA
DMperlbenchsphinx3
sjeng
Speedup
Significant
Yes
No

92. 0.0%
0.5%
1.0%
1.5%
bzip2gobm
kzeusm
p
libquantum
w
rf
astar
m
cfhm
m
er
m
ilc
nam
d
gcc
lbmgrom
acsh264ref
cactusA
DMperlbenchsphinx3
sjeng
Speedup
Significant
Yes
No
Speedup of -O3 over -O2

93. 0.0%
0.5%
1.0%
1.5%
bzip2gobm
kzeusm
p
libquantum
w
rf
astar
m
cfhm
m
er
m
ilc
nam
d
gcc
lbmgrom
acsh264ref
cactusA
DMperlbenchsphinx3
sjeng
Speedup
Significant
Yes
No
Speedup of -O3 over -O2

94. 0.0%
0.5%
1.0%
1.5%
bzip2gobm
kzeusm
p
libquantum
w
rf
astar
m
cfhm
m
er
m
ilc
nam
d
gcc
lbmgrom
acsh264ref
cactusA
DMperlbenchsphinx3
sjeng
Speedup
Significant
Yes
No
Speedup of -O3 over -O2

95. 0.0%
0.5%
1.0%
1.5%
bzip2gobm
kzeusm
p
libquantum
w
rf
astar
m
cfhm
m
er
m
ilc
nam
d
gcc
lbmgrom
acsh264ref
cactusA
DMperlbenchsphinx3
sjeng
Speedup
Significant
Yes
No
What do the results mean?

96. Comparing optimizations
-O2 -O3×30 ×30

97. -O2 -O3
×30 ×30lbm lbm
×30 ×30
Comparing optimizations

98. -O2 -O3
×30 ×30lbm lbm
×30 ×30astar astar
×30 ×30
Comparing optimizations

99. -O2 -O3
×30 ×30lbm lbm
×30 ×30astar astar
...
×30 ×30
Comparing optimizations

100. -O2 -O3×30 ×30
0%
1%
2%
3%
4%
0 25 50 75 100
Time (s)
PercentofObservedRuntimes
Version
−O2
−O3
Comparing optimizations

101. -O2-O3
0%
1%
2%
3%
4%
0 25 50 75 100
Time (s)
PercentofObservedRuntimes
Version
−O2
−O3
Is faster than ?

102. -O2-O3
0%
1%
2%
3%
4%
0 25 50 75 100
Time (s)
PercentofObservedRuntimes
Version
−O2
−O3
Is faster than ?

103. -O2-O3
0%
1%
2%
3%
4%
0 25 50 75 100
Time (s)
PercentofObservedRuntimes
Version
−O2
−O3
Is faster than ?

104. -O2-O3
0%
1%
2%
3%
4%
0 25 50 75 100
Time (s)
PercentofObservedRuntimes
Version
−O2
−O3
Is faster than ?

105. If = -O2-O3
0%
1%
2%
3%
4%
0 25 50 75 100
Time (s)
PercentofObservedRuntimes
Version
−O2
−O3

106. If = -O2-O3
1%
2%
3%
4%
PercentofObservedRuntimes
Version
−O2
−O3

107. If = -O2-O3
1%
2%
3%
4%
PercentofObservedRuntimes
Version
−O2
−O3
what is the probability of measuring
these differences?

108. If = -O2-O3
4%
imes
what is the probability of measuring
these differences?
Analysis ofVariance
aov(time~opt+Error(benchmark/opt), times)

109. If = -O2-O3
4%
imes
what is the probability of measuring
these differences?
Analysis ofVariance
If p-value

110. If = -O2-O3
4%
imes
what is the probability of measuring
these differences?
Analysis ofVariance
If p-value ≤ 5%

111. If = -O2-O3
4%
imes
what is the probability of measuring
these differences?
Analysis ofVariance
If p-value ≤ 5%
we reject the null hypothesis

112. Analysis ofVariance
If p-value ≤ 5%
we reject the null hypothesis
p-value = 26.4%
-O3 -O2vs
are we 73.6% confident?
one in four experiments will show an
effect that does not exist!

113. Analysis ofVariance
If p-value ≤ 5%
we reject the null hypothesis
p-value = 26.4%
fail to reject the null hypothesis
-O3 -O2vs

114. -O3
Analysis ofVariance
we reject the null hypothesis
-O2The effect of over is
indistinguishable from noise
If p-value ≤ 5%
Did STABILIZER hide the effect?

115. Runtime with -O2
Runtime with -O3
Runtime with -O1
Runtime with -O0
Execution Time
Did STABILIZER hide the effect?

116. STABILIZER
STABILIZER
STABILIZERSTABILIZER
Execution Time
Runtime with -O2
Runtime with -O3
Runtime with -O1
Runtime with -O0
speedups
Did STABILIZER hide the effect?

117. Runtime with -O2
Runtime with -O3
Runtime with -O1
Runtime with -O0
STABILIZER
STABILIZER
STABILIZERSTABILIZER
Execution Time
speedups
Did STABILIZER hide the effect?

118. SPECint SPECfp Summary
0%
10%
20%
30%
40%
perlbench
gcc
gobm
kh264ref
sjeng
astar
bzip2
libquantum
hm
m
er
m
cfcactusA
DM
zeusm
p
w
rfgrom
acssphinx3
lbm
m
ilc
nam
d
hm
eanm
edian
%IncreaseinExecutionTime
STABILIZER is fast enough

119. STABILIZER
Memory layout affects performance
STABILIZER eliminates the effect of layout
showed that -O3 does not have a statistically
significant effect across our benchmarks
Case Studies
random layout enables sound performance evaluation
makes performance evaluation difficult

120. available at
stabilizer-tool.org
STABILIZER
PhD Fellowship: Charlie Curtsinger

121. Backup Slides

122. LLVM’s Optimizations
• Function Inlining
• Unroll Loops
• GlobalValue
Numbering
• Dead Global
Elimination
• Merge Duplicate
Global Constants
• Argument Promotion
• Global CSE
• Scalar Replacement of
Aggregates
-O2 -O3

123. Mechanisms

124. STABILIZER randomizes
stack frame size
argv
argc
locals
arguments
main

125. STABILIZER randomizes
stack frame size
argv
argc
locals
arguments
main
return address
frame ptr
locals
arguments
foo

126. STABILIZER randomizes
stack frame size
argv
argc
locals
arguments
main
return address
frame ptr
locals
arguments
foo
return address
frame ptr bar

127. argv
argc
locals
arguments
main
return address
frame ptr
locals
arguments
foo
STABILIZER randomizes
stack frame size

128. return address
frame ptr
locals
locals
arguments
main
return address
frame ptr
locals
arguments
foo
bar
STABILIZER randomizes
stack frame size

129. return address
frame ptr
locals
locals
arguments
main
return address
frame ptr
locals
arguments
foo
bar
STABILIZER randomizes
stack frame size
Random
Padding

130. foo
bar
baz
STABILIZER randomizes
function placement

131. foo
bar
baz
trap
trap
trap
STABILIZER randomizes
function placement

132. foo
bar
baz
jmp
trap
trap
foo′
STABILIZER randomizes
function placement

133. foo
bar
baz
jmp
trap
trap
&bar
&baz
&g
STABILIZER randomizes
function placement
foo′

134. foo
bar
baz
jmp
trap
trap
STABILIZER randomizes
function placement
&bar
&baz
&g
foo′

135. foo
bar
baz
jmp
trap
baz′
&foo
&bar
jmp
STABILIZER randomizes
function placement
&bar
&baz
&g
foo′

136. and re-randomizes during execution
Randomizes placement of
heap objects
stack frames
functions

137. foo
bar
baz
jmp
trap
baz′
&foo
&bar
jmp
STABILIZER re-randomizes
function placement
&bar
&baz
&g
foo′

138. foo
bar
baz
jmp
trap
jmp
baz′
&foo
&bar
&bar
&baz
&g
foo′
STABILIZER re-randomizes
function placement

139. foo
bar
baz
trap
baz′
&foo
&bar
&bar
&baz
&g
foo′
trap
trap
STABILIZER re-randomizes
function placement

140. foo
bar
baz
trap
baz′
&foo
&bar
&bar
&baz
&g
foo′
trap
trap
STABILIZER re-randomizes
function placement

141. foo
bar
baz
trap
baz′
&foo
&bar
&bar
&baz
&g
foo′
trap
jmp
baz″
&foo
&bar
STABILIZER re-randomizes
function placement

142. foo
bar
baz
trap &bar
&baz
&g
foo′
trap
jmp
baz″
&foo
&bar
STABILIZER re-randomizes
function placement

143. 0.0
0.2
0.4
0.6
350 360 370 380 390 400
Time (s)
ProbabilityDensity

144. 0.0
0.2
0.4
0.6
350 360 370 380 390 400
Time (s)
ProbabilityDensity

145. 0.0
0.2
0.4
0.6
350 360 370 380 390 400
Time (s)
ProbabilityDensity

146. 0.0
0.2
0.4
0.6
350 360 370 380 390 400
Time (s)
ProbabilityDensity

147. 0.0
0.2
0.4
0.6
350 360 370 380 390 400
Time (s)
ProbabilityDensity

148. Programs with phases
Execution Time

149. Programs with phases
A B A B AA B A B A
Execution Time

150. A B A B A
Execution Time
A A A
B B
~ N(μA, σA)
Programs with phases
~ N(μB, σB)

151. A B A B A
Execution Time
A A A
B B
~ N(μA, σA)
Programs with phases
~ N(μB, σB)
~ A + B

152. A B A B A
Execution Time
Programs with phases
~ N(μA, σA) + N(μB, σB)
~ A + B
= N(μA+μB, σA+σB)