第1回カーネル/VM探検隊 Linux KVM のコードを追いかけてみよう のスライドです。

1. Linux KVM
/VM
1
by Tsuyoshi Ozawa, softlab@univ. of Tsukuba.
Twitter: oza_x86
2009 8 13 1

2. 2009 8 13 2

3. 1. VMM
2. Intel-VT
3. Linux KVM
KVM
2009 8 13 3

4. VMM
2009 8 13 4

5. VMM
2009 8 13 5

6. Virtual Machine
Monitor
2009 8 13 6

7. Virtual Machine
Monitor
1.
2. (OS)
2009 8 13 7

8. VMM
2009 8 13 8

9. 2009 8 13 9

10. 2009 8 13 10

11. Qemu
2009 8 13 11

12. 2009 8 13 12

13. 2009 8 13 13

14. Qemu
2009 8 13 14

15. 2009 8 13 15

16. CPU
2009 8 13 16

17. Main Memory
...
ALU movl %eax %ebx
...
2009 8 13 17

18. Main Memory
...
ALU movl %eax %ebx
...
fetch
2009 8 13 18

19. Qemu ...
2009 8 13 19

20. Qemu
Main Memory
...
ALU movl %eax %ebx
...
fetch
2009 8 13 20

21. opcode = mem[pc];
execute(opcode);
2009 8 13 21

22. mov
opcode = mem[pc];
execute(opcode);
jmp
2009 8 13 22

23. •
2009 8 13 23

24. 2009 8 13 24

25. opcode[0..N] = mem[pc];
mid_ops=compile(opcode);
execute(mid_ops);
2009 8 13 25

26. opcode[0..N] = mem[pc];
mid_ops=compile(opcode);
execute(mid_ops);
2009 8 13 26

27. opcode[0..N] = mem[pc];
mid_ops=compile(opcode);
execute(mid_ops);
2009 8 13 27

28. 2009 8 13 28

29. 2009 8 13 29

30. •
•
•
2009 8 13 30

31. 2009 8 13 31

32. 2009 8 13 32

33. VMM
2009 8 13 33

34. KVM
2009 8 13 34

35. •
2009 8 13 35

36. .section .data
message:
.ascii "hello,gas!n"
.section .text
.global _start
_start:
movl $4,%eax
movl $1,%ebx
movl $message,%ecx
movl $12,%edx
int $0x80
2009 8 13 36

37. ./a.out
hello,gas!
2009 8 13 37

38. Hello world
2009 8 13 38

39. printf(“HelloWorld¥n”)
2009 8 13 39

40. printf
2009 8 13 40

41. 2009 8 13 41

42. 2009 8 13 42

43. write
2009 8 13 43

44. write
out
2009 8 13 44

45. .section .data
message:
.ascii "hello,gas!n"
.section .text
.global _start
_start:
movl $4,%eax
movl $1,%ebx
movl $message,%ecx
out
movl $12,%edx
int $0x80
2009 8 13 45

46. ...
outb %ax
...
2009 8 13 46

47. Program
Display
CPU
outb
2009 8 13 47

48. hello,gas!
2009 8 13 48

49. •
2009 8 13 49

50. • VM
2009 8 13 50

51. .section .data
message:
.ascii "hello,gas!n"
.section .text
.global _start
_start:
movl $4,%eax
movl $1,%ebx
movl $message,%ecx
out
movl $12,%edx
int $0x80
2009 8 13 51

52. ...
outb $hogehoge
...
2009 8 13 52

53. hello,gas!
2009 8 13 53

54. OS .
2009 8 13 54

55. VM
Virtual Program
Display
VirtualCPU
outb
2009 8 13 55

56. VM
Program
Real
Display
VirtualCPU
outb
2009 8 13 56

57. hello,gas!
on host system?!
2009 8 13 57

58. 2009 8 13 58

59. 2009 8 13 59

60. 2009 8 13 60

61. 2009 8 13 61

62. VM out
Virtual Program
Display
CPU
2009 8 13 62

63. VM out
Virtual Program
Display
CPU
outb
2009 8 13 63

64. VM out
2009 8 13 64

65. Emu
Code
trap
CPU
2009 8 13 65

66. Emu
Code
CPU
handle
2009 8 13 66

67. VM out
Emu ret
Code
CPU
2009 8 13 67

68. VM out
ack
Virtual Program
Display
CPU
2009 8 13 68

69. VM out
out
2009 8 13 69

70. 2009 8 13 70

71. 2009 8 13 71

72. x86
2009 8 13 72

73. ?
2009 8 13 73

74. 2009 8 13 74

75. 2009 8 13 75

76. ?
2009 8 13 76

77. 2009 8 13 77

78. x86 ?
2009 8 13 78

79. ... ?
2009 8 13 79

80. 2009 8 13 80

81. 2009 8 13 81

82. x86
•
• Hypervisor Call
• Xen
2009 8 13 82

83. x86
•
• LilyVM
2009 8 13 83

84. x86
•
• (JIT )
• VMWare
2009 8 13 84

85. 2009 8 13 85

86. Intel
( AMD-V )
2009 8 13 86

87. Intel-VT
2009 8 13 87

88. CPU ?
2009 8 13 88

89. User
Privilege
2009 8 13 89

90. User
Privilege
2009 8 13 90

91. 2009 8 13 91

92. User User
warp
Privilege Privilege
2009 8 13 92

93. 2009 8 13 93

94. User User
Privilege Privilege
2009 8 13 94

95. User User
Privilege Privilege
2009 8 13 95

96. User User
Privilege Privilege
trap
2009 8 13 96

97. User User
Privilege Privilege
Intel-VT
2009 8 13 97

98. User
Privilege
2009 8 13 98

99. VMX-Root Mode
(VM )
2009 8 13 99

100. User
Privilege
2009 8 13 100

101. VMX-non root Mode
( )
2009 8 13 101

102. VMX non Root Mode
2009 8 13 102

103. User User
Privilege Privilege
2009 8 13 103

104. vmlaunch
vmresume
2009 8 13 104

105. KVM
2009 8 13 105

106. KVM
2009 8 13 106

107. Kernel-based
Virtul Machine
2009 8 13 107

108. KVM
2009 8 13 108

109. KVM
• •Intel-VT
• •Qemu
• •Linux
• •Linux
2009 8 13 109

110. KVM
• •Intel-VT
• •Qemu
• •Linux
• •Linux
KVM
2009 8 13 110

111. KVM
•
•VM
•
•
•
• OS
2009 8 13 111

112. 2009 8 13 112

113. 2009 8 13 113

114. Main loop
2009 8 13 114

115. KVM
2009 8 13 115

116. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 116

117. 2009 8 13 117

118. VMX
VMX
non Root
Root Mode
Mode
2009 8 13 118

119. 2009 8 13 119

120. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 120

121. VMX non Root Mode
2009 8 13 121

122. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 122

123. User
Qemu
KVM Program
Qemu
Linux Guest
KVM
Kernel Kernel
2009 8 13 123

124. ./qemu -hda disk1.img
(-hda )
2009 8 13 124

125. User
Qemu
Program
$ ./qemu -hda disk1
Linux Guest
KVM
Kernel Kernel
2009 8 13 125

126. User
Qemu
Program
Qemu
Linux Guest
KVM
Kernel Kernel
2009 8 13 126

127. Qemu
2009 8 13 127

128. User
Qemu
Program
IOCTL
Linux Guest
KVM
Kernel Kernel
2009 8 13 128

129. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 129

130. 2009 8 13 130

131. 2009 8 13 131

132. vmlaunch
vmresume
2009 8 13 132

133. VMX non Root
Mode
2009 8 13 133

134. User
Qemu
Program
Linux Guest
KVM VT Kernel
Kernel
2009 8 13 134

135. 2009 8 13 135

136. vmlaunch
vmresume
2009 8 13 136

137. User
Qemu
Program
vmlaunch / vmresume
Linux Guest
KVM
Kernel Kernel
2009 8 13 137

138. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 138

139. VMX non root Mode
2009 8 13 139

140. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 140

141. 2009 8 13 141

142. VMX non Root Mode
IO
2009 8 13 142

143. 2009 8 13 143

144. outb %ax
2009 8 13 144

145. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 145

146. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
outb %ax
2009 8 13 146

147. VMX Root Mode
2009 8 13 147

148. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 148

149. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 149

150. 2009 8 13 150

151. Qemu
2009 8 13 151

152. User
Qemu
Program
IOCTL return
Linux Guest
KVM
Kernel Kernel
2009 8 13 152

153. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 153

154. User
Qemu
Program
io
Linux Guest
KVM
Kernel Kernel
2009 8 13 154

155. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 155

156. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 156

157. 2009 8 13 157

158. IOCTL
CPU
2009 8 13 158

159. User
Qemu
Program
IOCTL
Linux Guest
KVM
Kernel Kernel
2009 8 13 159

160. 2009 8 13 160

161. ...
2009 8 13 161

162. User
Qemu
Program
IOCTL
Linux Guest
KVM
Kernel Kernel
vmlaunch
vmresume
2009 8 13 162

163. 2009 8 13 163

164. 2009 8 13 164

165. KVM
kvm-85
2009 8 13 165

166. 2009 8 13 166

167. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 167

168. Qemu
2009 8 13 168

169. $./qemu -hda disk1
...
2009 8 13 169

170. qemu/vl.c main.c
2009 8 13 170

171. 4439 int
4440 {
main (int argc, char **argv, char **envp)
// . Shadow page table .
5425 kvm_qemu_create_context()
//
5493 kvm_init(smp_cpus);
...
// qemu/hw/pc.c
5554 machine->init(ram_size, vga_ram_size, boot_devices,
5555 kernel_filename, kernel_cmdline,
5556 initrd_filename, cpu_model);
...
...
5742 main_loop();
...
5747 }
2009 8 13 171

172. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 172

173. User
Qemu
Program
Qemu
Linux Guest
KVM
Kernel Kernel
2009 8 13 173

174. -
•machine->init
•
• qemu/hw/pc.c
pc_init1
2009 8 13 174

175. 2009 8 13 175

176. 2009 8 13 176

177. pc_init1
pc_new_cpu
cpu_init = cpu_x86_init
kvm_init_cpu
pthread_create
ap_main_loop
kvm_main_loop_cpu
2009 8 13 177

178. 2009 8 13 178

179. 2009 8 13 179

180. User
Qemu
Program
IOCTL
Linux Guest
KVM
Kernel Kernel
2009 8 13 180

181. IOCTL
2009 8 13 181

182. ...
qemu/kvm-all.c
kvm_cpu_exec
2009 8 13 182

183. 378 static int kvm_main_loop_cpu (CPUState *env)
379 {
394 while (1) {
395 while (!has_work(env))
396 kvm_main_loop_wait(env, 1000);
397 if (env->interrupt_request &
(CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI))
398 env->halted = 0; // .
....
404 }
405 if (!env->halted && !env->kvm_cpu_state.init) {
406 kvm_cpu_exec(env); //
}
}
413 }
2009 8 13 183

184. int kvm_cpu_exec(){
...
461 ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
478 switch (run->exit_reason) {
479 case KVM_EXIT_IO:
481 ret = kvm_handle_io(env, run->io.port,
482 (uint8_t *)run + run>io.data_offset,
483 run->io.direction,
484 run->io.size,
485 run->io.count);
486 break;
case KVM_EXIT_MMIO:
.... ...
2009 8 13 184

185. 648 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
649 {
658 ret = ioctl(env->kvm_fd, type, arg);
659 if (ret == -1)
660 ret = -errno;
661
662 return ret;
663 }
2009 8 13 185

186. 2009 8 13 186

187. IOCTL
2009 8 13 187

188. User
Qemu
Program
IOCTL
Linux Guest
KVM
Kernel Kernel
2009 8 13 188

189. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 189

190. 2009 8 13 190

191. 2009 8 13 191

192. IOCTL
2009 8 13 192

193. IOCTL
2009 8 13 193

194. 2009 8 13 194

195. static struct file_operations kvm_vcpu_fops = {
.release = kvm_vcpu_release,
.unlocked_ioctl = kvm_vcpu_ioctl,
.compat_ioctl = kvm_vcpu_ioctl,
.mmap = kvm_vcpu_mmap,
};
2009 8 13 195

196. //
file->f_op = kvm_vcpu_fops;
2009 8 13 196

197. kvm_vcpu_ioctl
2009 8 13 197

198. static long kvm_vcpu_ioctl(struct file *filp,
unsigned int ioctl, unsigned long arg)
{
switch (ioctl) {
case KVM_RUN:
r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
break;
....
}
2009 8 13 198

199. VMX non Root
Mode
2009 8 13 199

200. User
Qemu
Program
Linux Guest
KVM VT Kernel
Kernel
2009 8 13 200

201. ...
2009 8 13 201

202. kvm_arch_vcpu_ioctl_run
__vcpu_run
vcpu_enter_guest
vmx_vcpu_run
2009 8 13 202

203. vmx_vcpu_run
kernel/x86/vmx.c
2009 8 13 203

204. User
Qemu
Program
vmlaunch / vmresume
Linux Guest
KVM
Kernel Kernel
2009 8 13 204

205. 2009 8 13 205

206. 2009 8 13 206

207. include/asm-x86/vmx.h
#define ASM_VMX_VMLAUNCH
".byte 0x0f, 0x01, 0xc2"
#define ASM_VMX_VMRESUME
".byte 0x0f, 0x01, 0xc3"
2009 8 13 207

208. 2009 8 13 208

209. vmlaunch/vmresume
2009 8 13 209

210. User
Qemu
Program
vmlaunch / vmresume
Linux Guest
KVM
Kernel Kernel
2009 8 13 210

211. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 211

212. VMX non-root mode
2009 8 13 212

213. 2009 8 13 213

214. VMX non Root Mode
IO
2009 8 13 214

215. 2009 8 13 215

216. 2009 8 13 216

217. outb %ax
2009 8 13 217

218. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 218

219. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
outb %ax
2009 8 13 219

220. 2009 8 13 220

221. "jne .Llaunched nt"
__ex(ASM_VMX_VMLAUNCH) "nt"
"jmp .Lkvm_vmx_return nt"
".Llaunched: " __ex(ASM_VMX_VMRESUME) "nt"
".Lkvm_vmx_return: "
vmlaunch
/* */ vmresume
...
...
2009 8 13 221

222. "jne .Llaunched nt"
__ex(ASM_VMX_VMLAUNCH) "nt"
"jmp .Lkvm_vmx_return nt"
".Llaunched: " __ex(ASM_VMX_VMRESUME) "nt"
".Lkvm_vmx_return: "
/* */
...
2009 8 13 222

223. 2009 8 13 223

224. 2009 8 13 224

225. out
2009 8 13 225

226. out
in .
2009 8 13 226

227. Intel-VT
2009 8 13 227

228. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 228

229. 2009 8 13 229

230. Qemu
2009 8 13 230

231. User
Qemu
Program
IOCTL return
Linux Guest
KVM
Kernel Kernel
2009 8 13 231

232. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 232

233. int kvm_cpu_exec(){
...
461 ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
478 switch (run->exit_reason) {
479 case KVM_EXIT_IO:
481 ret = kvm_handle_io(env, run->io.port,
482 (uint8_t *)run + run>io.data_offset,
483 run->io.direction,
484 run->io.size,
485 run->io.count);
486 break;
case KVM_EXIT_MMIO:
.... ...
2009 8 13 233

234. outb
2009 8 13 234

235. int kvm_cpu_exec(){
...
//
461 ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
478 switch (run->exit_reason) {
479 case KVM_EXIT_IO:
481 ret = kvm_handle_io(env, run->io.port,
482 (uint8_t *)run + run>io.data_offset,
483 run->io.direction,
484 run->io.size,
485 run->io.count);
486 break;
case KVM_EXIT_MMIO: outb
.... ...
2009 8 13 235

236. 2009 8 13 236

237. IO
2009 8 13 237

238. User
Qemu
Program
io
Linux Guest
KVM
Kernel Kernel
2009 8 13 238

239. 2009 8 13 239

240. out
2009 8 13 240

241. outb %al
2009 8 13 241

242. outb %al
2009 8 13 242

243. ( )
2009 8 13 243

244. int kvm_cpu_exec(){
...
//
461 ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
478 switch (run->exit_reason) {
479 case KVM_EXIT_IO:
481 ret = kvm_handle_io(env, run->io.port,
482 (uint8_t *)run + run>io.data_offset,
483 run->io.direction,
484 run->io.size,
485 run->io.count);
486 break;
case KVM_EXIT_MMIO:
.... ...
2009 8 13 244

245. kvm_handle_io
cpu_outb
ioport_write
2009 8 13 245

246. static void ioport_write(int index, uint32_t address, uint32_t data)
{
static IOPortWriteFunc *default_func[3] = {
default_ioport_writeb,
default_ioport_writew,
default_ioport_writel
};
IOPortWriteFunc *func = ioport_write_table[index][address];
if (!func)
func = default_func[index];
func(ioport_opaque[address], address, data);
}
ioport_write_table
2009 8 13 246

247. ( )
2009 8 13 247

248. User
Qemu
Program
io
Linux Guest
KVM
Kernel Kernel
2009 8 13 248

249. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 249

250. User
Qemu
Program
Linux Guest
KVM
Kernel Kernel
2009 8 13 250

251. 2009 8 13 251

252. IOCTL
CPU
2009 8 13 252

253. User
Qemu
Program
IOCTL
Linux Guest
KVM
Kernel Kernel
2009 8 13 253

254. 2009 8 13 254

255. ...
2009 8 13 255

256. User
Qemu
Program
IOCTL
Linux Guest
KVM
Kernel Kernel
vmlaunch
2009 8 13 vmresume 256

257. 2009 8 13 257

258. 378 static int kvm_main_loop_cpu(CPUState *env)
379 {
394 while (1) {
395 while (!has_work(env))
396 kvm_main_loop_wait(env, 1000);
397 if (env->interrupt_request &
(CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI))
398 env->halted = 0; // .
....
404 }
405 if (!env->halted && !env->kvm_cpu_state.init) {
406 kvm_cpu_exec(env); //
}
}
413 }
2009 8 13 258

259. 2009 8 13 259

260. Twitter
2009 8 13 260