1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/types.h>
31 #ifndef WITHOUT_CAPSICUM
32 #include <sys/capsicum.h>
33 #endif
34 #include <sys/mman.h>
35 #ifdef BHYVE_SNAPSHOT
36 #include <sys/socket.h>
37 #include <sys/stat.h>
38 #endif
39 #include <sys/time.h>
40 #ifdef BHYVE_SNAPSHOT
41 #include <sys/un.h>
42 #endif
43
44 #include <amd64/vmm/intel/vmcs.h>
45 #include <x86/apicreg.h>
46
47 #include <machine/atomic.h>
48 #include <machine/segments.h>
49
50 #ifndef WITHOUT_CAPSICUM
51 #include <capsicum_helpers.h>
52 #endif
53 #include <stdio.h>
54 #include <stdlib.h>
55 #include <string.h>
56 #include <err.h>
57 #include <errno.h>
58 #ifdef BHYVE_SNAPSHOT
59 #include <fcntl.h>
60 #endif
61 #include <libgen.h>
62 #include <unistd.h>
63 #include <assert.h>
64 #include <pthread.h>
65 #include <pthread_np.h>
66 #include <sysexits.h>
67 #include <stdbool.h>
68 #include <stdint.h>
69 #ifdef BHYVE_SNAPSHOT
70 #include <ucl.h>
71 #include <unistd.h>
72
73 #include <libxo/xo.h>
74 #endif
75
76 #include <machine/vmm.h>
77 #ifndef WITHOUT_CAPSICUM
78 #include <machine/vmm_dev.h>
79 #endif
80 #include <machine/vmm_instruction_emul.h>
81 #include <vmmapi.h>
82
83 #include "bhyverun.h"
84 #include "acpi.h"
85 #include "atkbdc.h"
86 #include "bootrom.h"
87 #include "config.h"
88 #include "inout.h"
89 #include "debug.h"
90 #include "e820.h"
91 #include "fwctl.h"
92 #include "gdb.h"
93 #include "ioapic.h"
94 #include "kernemu_dev.h"
95 #include "mem.h"
96 #include "mevent.h"
97 #include "mptbl.h"
98 #include "pci_emul.h"
99 #include "pci_irq.h"
100 #include "pci_lpc.h"
101 #include "qemu_fwcfg.h"
102 #include "smbiostbl.h"
103 #ifdef BHYVE_SNAPSHOT
104 #include "snapshot.h"
105 #endif
106 #include "xmsr.h"
107 #include "spinup_ap.h"
108 #include "rtc.h"
109 #include "vmgenc.h"
110
111 #define GUEST_NIO_PORT 0x488 /* guest upcalls via i/o port */
112
113 #define MB (1024UL * 1024)
114 #define GB (1024UL * MB)
115
116 static const char * const vmx_exit_reason_desc[] = {
117 [EXIT_REASON_EXCEPTION] = "Exception or non-maskable interrupt (NMI)",
118 [EXIT_REASON_EXT_INTR] = "External interrupt",
119 [EXIT_REASON_TRIPLE_FAULT] = "Triple fault",
120 [EXIT_REASON_INIT] = "INIT signal",
121 [EXIT_REASON_SIPI] = "Start-up IPI (SIPI)",
122 [EXIT_REASON_IO_SMI] = "I/O system-management interrupt (SMI)",
123 [EXIT_REASON_SMI] = "Other SMI",
124 [EXIT_REASON_INTR_WINDOW] = "Interrupt window",
125 [EXIT_REASON_NMI_WINDOW] = "NMI window",
126 [EXIT_REASON_TASK_SWITCH] = "Task switch",
127 [EXIT_REASON_CPUID] = "CPUID",
128 [EXIT_REASON_GETSEC] = "GETSEC",
129 [EXIT_REASON_HLT] = "HLT",
130 [EXIT_REASON_INVD] = "INVD",
131 [EXIT_REASON_INVLPG] = "INVLPG",
132 [EXIT_REASON_RDPMC] = "RDPMC",
133 [EXIT_REASON_RDTSC] = "RDTSC",
134 [EXIT_REASON_RSM] = "RSM",
135 [EXIT_REASON_VMCALL] = "VMCALL",
136 [EXIT_REASON_VMCLEAR] = "VMCLEAR",
137 [EXIT_REASON_VMLAUNCH] = "VMLAUNCH",
138 [EXIT_REASON_VMPTRLD] = "VMPTRLD",
139 [EXIT_REASON_VMPTRST] = "VMPTRST",
140 [EXIT_REASON_VMREAD] = "VMREAD",
141 [EXIT_REASON_VMRESUME] = "VMRESUME",
142 [EXIT_REASON_VMWRITE] = "VMWRITE",
143 [EXIT_REASON_VMXOFF] = "VMXOFF",
144 [EXIT_REASON_VMXON] = "VMXON",
145 [EXIT_REASON_CR_ACCESS] = "Control-register accesses",
146 [EXIT_REASON_DR_ACCESS] = "MOV DR",
147 [EXIT_REASON_INOUT] = "I/O instruction",
148 [EXIT_REASON_RDMSR] = "RDMSR",
149 [EXIT_REASON_WRMSR] = "WRMSR",
150 [EXIT_REASON_INVAL_VMCS] =
151 "VM-entry failure due to invalid guest state",
152 [EXIT_REASON_INVAL_MSR] = "VM-entry failure due to MSR loading",
153 [EXIT_REASON_MWAIT] = "MWAIT",
154 [EXIT_REASON_MTF] = "Monitor trap flag",
155 [EXIT_REASON_MONITOR] = "MONITOR",
156 [EXIT_REASON_PAUSE] = "PAUSE",
157 [EXIT_REASON_MCE_DURING_ENTRY] =
158 "VM-entry failure due to machine-check event",
159 [EXIT_REASON_TPR] = "TPR below threshold",
160 [EXIT_REASON_APIC_ACCESS] = "APIC access",
161 [EXIT_REASON_VIRTUALIZED_EOI] = "Virtualized EOI",
162 [EXIT_REASON_GDTR_IDTR] = "Access to GDTR or IDTR",
163 [EXIT_REASON_LDTR_TR] = "Access to LDTR or TR",
164 [EXIT_REASON_EPT_FAULT] = "EPT violation",
165 [EXIT_REASON_EPT_MISCONFIG] = "EPT misconfiguration",
166 [EXIT_REASON_INVEPT] = "INVEPT",
167 [EXIT_REASON_RDTSCP] = "RDTSCP",
168 [EXIT_REASON_VMX_PREEMPT] = "VMX-preemption timer expired",
169 [EXIT_REASON_INVVPID] = "INVVPID",
170 [EXIT_REASON_WBINVD] = "WBINVD",
171 [EXIT_REASON_XSETBV] = "XSETBV",
172 [EXIT_REASON_APIC_WRITE] = "APIC write",
173 [EXIT_REASON_RDRAND] = "RDRAND",
174 [EXIT_REASON_INVPCID] = "INVPCID",
175 [EXIT_REASON_VMFUNC] = "VMFUNC",
176 [EXIT_REASON_ENCLS] = "ENCLS",
177 [EXIT_REASON_RDSEED] = "RDSEED",
178 [EXIT_REASON_PM_LOG_FULL] = "Page-modification log full",
179 [EXIT_REASON_XSAVES] = "XSAVES",
180 [EXIT_REASON_XRSTORS] = "XRSTORS"
181 };
182
183 typedef int (*vmexit_handler_t)(struct vmctx *, struct vm_exit *, int *vcpu);
184
185 int guest_ncpus;
186 uint16_t cpu_cores, cpu_sockets, cpu_threads;
187
188 int raw_stdio = 0;
189
190 static char *progname;
191 static const int BSP = 0;
192
193 static cpuset_t cpumask;
194
195 static void vm_loop(struct vmctx *ctx, int vcpu);
196
197 static struct bhyvestats {
198 uint64_t vmexit_bogus;
199 uint64_t vmexit_reqidle;
200 uint64_t vmexit_hlt;
201 uint64_t vmexit_pause;
202 uint64_t vmexit_mtrap;
203 uint64_t vmexit_inst_emul;
204 uint64_t cpu_switch_rotate;
205 uint64_t cpu_switch_direct;
206 } stats;
207
208 static struct mt_vmm_info {
209 pthread_t mt_thr;
210 struct vmctx *mt_ctx;
211 int mt_vcpu;
212 } *mt_vmm_info;
213
214 static cpuset_t **vcpumap;
215
216 static void
usage(int code)217 usage(int code)
218 {
219
220 fprintf(stderr,
221 "Usage: %s [-AaCDeHhPSuWwxY]\n"
222 " %*s [-c [[cpus=]numcpus][,sockets=n][,cores=n][,threads=n]]\n"
223 " %*s [-G port] [-k config_file] [-l lpc] [-m mem] [-o var=value]\n"
224 " %*s [-p vcpu:hostcpu] [-r file] [-s pci] [-U uuid] vmname\n"
225 " -A: create ACPI tables\n"
226 " -a: local apic is in xAPIC mode (deprecated)\n"
227 " -C: include guest memory in core file\n"
228 " -c: number of CPUs and/or topology specification\n"
229 " -D: destroy on power-off\n"
230 " -e: exit on unhandled I/O access\n"
231 " -G: start a debug server\n"
232 " -H: vmexit from the guest on HLT\n"
233 " -h: help\n"
234 " -k: key=value flat config file\n"
235 " -K: PS2 keyboard layout\n"
236 " -l: LPC device configuration\n"
237 " -m: memory size\n"
238 " -o: set config 'var' to 'value'\n"
239 " -P: vmexit from the guest on pause\n"
240 " -p: pin 'vcpu' to 'hostcpu'\n"
241 #ifdef BHYVE_SNAPSHOT
242 " -r: path to checkpoint file\n"
243 #endif
244 " -S: guest memory cannot be swapped\n"
245 " -s: <slot,driver,configinfo> PCI slot config\n"
246 " -U: UUID\n"
247 " -u: RTC keeps UTC time\n"
248 " -W: force virtio to use single-vector MSI\n"
249 " -w: ignore unimplemented MSRs\n"
250 " -x: local APIC is in x2APIC mode\n"
251 " -Y: disable MPtable generation\n",
252 progname, (int)strlen(progname), "", (int)strlen(progname), "",
253 (int)strlen(progname), "");
254
255 exit(code);
256 }
257
258 /*
259 * XXX This parser is known to have the following issues:
260 * 1. It accepts null key=value tokens ",," as setting "cpus" to an
261 * empty string.
262 *
263 * The acceptance of a null specification ('-c ""') is by design to match the
264 * manual page syntax specification, this results in a topology of 1 vCPU.
265 */
266 static int
topology_parse(const char * opt)267 topology_parse(const char *opt)
268 {
269 char *cp, *str, *tofree;
270
271 if (*opt == '\0') {
272 set_config_value("sockets", "1");
273 set_config_value("cores", "1");
274 set_config_value("threads", "1");
275 set_config_value("cpus", "1");
276 return (0);
277 }
278
279 tofree = str = strdup(opt);
280 if (str == NULL)
281 errx(4, "Failed to allocate memory");
282
283 while ((cp = strsep(&str, ",")) != NULL) {
284 if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
285 set_config_value("cpus", cp + strlen("cpus="));
286 else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0)
287 set_config_value("sockets", cp + strlen("sockets="));
288 else if (strncmp(cp, "cores=", strlen("cores=")) == 0)
289 set_config_value("cores", cp + strlen("cores="));
290 else if (strncmp(cp, "threads=", strlen("threads=")) == 0)
291 set_config_value("threads", cp + strlen("threads="));
292 else if (strchr(cp, '=') != NULL)
293 goto out;
294 else
295 set_config_value("cpus", cp);
296 }
297 free(tofree);
298 return (0);
299
300 out:
301 free(tofree);
302 return (-1);
303 }
304
305 static int
parse_int_value(const char * key,const char * value,int minval,int maxval)306 parse_int_value(const char *key, const char *value, int minval, int maxval)
307 {
308 char *cp;
309 long lval;
310
311 errno = 0;
312 lval = strtol(value, &cp, 0);
313 if (errno != 0 || *cp != '\0' || cp == value || lval < minval ||
314 lval > maxval)
315 errx(4, "Invalid value for %s: '%s'", key, value);
316 return (lval);
317 }
318
319 /*
320 * Set the sockets, cores, threads, and guest_cpus variables based on
321 * the configured topology.
322 *
323 * The limits of UINT16_MAX are due to the types passed to
324 * vm_set_topology(). vmm.ko may enforce tighter limits.
325 */
326 static void
calc_topology(void)327 calc_topology(void)
328 {
329 const char *value;
330 bool explicit_cpus;
331 uint64_t ncpus;
332
333 value = get_config_value("cpus");
334 if (value != NULL) {
335 guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX);
336 explicit_cpus = true;
337 } else {
338 guest_ncpus = 1;
339 explicit_cpus = false;
340 }
341 value = get_config_value("cores");
342 if (value != NULL)
343 cpu_cores = parse_int_value("cores", value, 1, UINT16_MAX);
344 else
345 cpu_cores = 1;
346 value = get_config_value("threads");
347 if (value != NULL)
348 cpu_threads = parse_int_value("threads", value, 1, UINT16_MAX);
349 else
350 cpu_threads = 1;
351 value = get_config_value("sockets");
352 if (value != NULL)
353 cpu_sockets = parse_int_value("sockets", value, 1, UINT16_MAX);
354 else
355 cpu_sockets = guest_ncpus;
356
357 /*
358 * Compute sockets * cores * threads avoiding overflow. The
359 * range check above insures these are 16 bit values.
360 */
361 ncpus = (uint64_t)cpu_sockets * cpu_cores * cpu_threads;
362 if (ncpus > UINT16_MAX)
363 errx(4, "Computed number of vCPUs too high: %ju",
364 (uintmax_t)ncpus);
365
366 if (explicit_cpus) {
367 if (guest_ncpus != (int)ncpus)
368 errx(4, "Topology (%d sockets, %d cores, %d threads) "
369 "does not match %d vCPUs",
370 cpu_sockets, cpu_cores, cpu_threads,
371 guest_ncpus);
372 } else
373 guest_ncpus = ncpus;
374 }
375
376 static int
pincpu_parse(const char * opt)377 pincpu_parse(const char *opt)
378 {
379 const char *value;
380 char *newval;
381 char key[16];
382 int vcpu, pcpu;
383
384 if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) {
385 fprintf(stderr, "invalid format: %s\n", opt);
386 return (-1);
387 }
388
389 if (vcpu < 0) {
390 fprintf(stderr, "invalid vcpu '%d'\n", vcpu);
391 return (-1);
392 }
393
394 if (pcpu < 0 || pcpu >= CPU_SETSIZE) {
395 fprintf(stderr, "hostcpu '%d' outside valid range from "
396 "0 to %d\n", pcpu, CPU_SETSIZE - 1);
397 return (-1);
398 }
399
400 snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
401 value = get_config_value(key);
402
403 if (asprintf(&newval, "%s%s%d", value != NULL ? value : "",
404 value != NULL ? "," : "", pcpu) == -1) {
405 perror("failed to build new cpuset string");
406 return (-1);
407 }
408
409 set_config_value(key, newval);
410 free(newval);
411 return (0);
412 }
413
414 static void
parse_cpuset(int vcpu,const char * list,cpuset_t * set)415 parse_cpuset(int vcpu, const char *list, cpuset_t *set)
416 {
417 char *cp, *token;
418 int pcpu, start;
419
420 CPU_ZERO(set);
421 start = -1;
422 token = __DECONST(char *, list);
423 for (;;) {
424 pcpu = strtoul(token, &cp, 0);
425 if (cp == token)
426 errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
427 if (pcpu < 0 || pcpu >= CPU_SETSIZE)
428 errx(4, "hostcpu '%d' outside valid range from 0 to %d",
429 pcpu, CPU_SETSIZE - 1);
430 switch (*cp) {
431 case ',':
432 case '\0':
433 if (start >= 0) {
434 if (start > pcpu)
435 errx(4, "Invalid hostcpu range %d-%d",
436 start, pcpu);
437 while (start < pcpu) {
438 CPU_SET(start, set);
439 start++;
440 }
441 start = -1;
442 }
443 CPU_SET(pcpu, set);
444 break;
445 case '-':
446 if (start >= 0)
447 errx(4, "invalid cpuset for vcpu %d: '%s'",
448 vcpu, list);
449 start = pcpu;
450 break;
451 default:
452 errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
453 }
454 if (*cp == '\0')
455 break;
456 token = cp + 1;
457 }
458 }
459
460 static void
build_vcpumaps(void)461 build_vcpumaps(void)
462 {
463 char key[16];
464 const char *value;
465 int vcpu;
466
467 vcpumap = calloc(guest_ncpus, sizeof(*vcpumap));
468 for (vcpu = 0; vcpu < guest_ncpus; vcpu++) {
469 snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
470 value = get_config_value(key);
471 if (value == NULL)
472 continue;
473 vcpumap[vcpu] = malloc(sizeof(cpuset_t));
474 if (vcpumap[vcpu] == NULL)
475 err(4, "Failed to allocate cpuset for vcpu %d", vcpu);
476 parse_cpuset(vcpu, value, vcpumap[vcpu]);
477 }
478 }
479
480 void
vm_inject_fault(void * arg,int vcpu,int vector,int errcode_valid,int errcode)481 vm_inject_fault(void *arg, int vcpu, int vector, int errcode_valid,
482 int errcode)
483 {
484 struct vmctx *ctx;
485 int error, restart_instruction;
486
487 ctx = arg;
488 restart_instruction = 1;
489
490 error = vm_inject_exception(ctx, vcpu, vector, errcode_valid, errcode,
491 restart_instruction);
492 assert(error == 0);
493 }
494
495 void *
paddr_guest2host(struct vmctx * ctx,uintptr_t gaddr,size_t len)496 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
497 {
498
499 return (vm_map_gpa(ctx, gaddr, len));
500 }
501
502 #ifdef BHYVE_SNAPSHOT
503 uintptr_t
paddr_host2guest(struct vmctx * ctx,void * addr)504 paddr_host2guest(struct vmctx *ctx, void *addr)
505 {
506 return (vm_rev_map_gpa(ctx, addr));
507 }
508 #endif
509
510 int
fbsdrun_virtio_msix(void)511 fbsdrun_virtio_msix(void)
512 {
513
514 return (get_config_bool_default("virtio_msix", true));
515 }
516
517 static void *
fbsdrun_start_thread(void * param)518 fbsdrun_start_thread(void *param)
519 {
520 char tname[MAXCOMLEN + 1];
521 struct mt_vmm_info *mtp;
522 int error, vcpu;
523
524 mtp = param;
525 vcpu = mtp->mt_vcpu;
526
527 snprintf(tname, sizeof(tname), "vcpu %d", vcpu);
528 pthread_set_name_np(mtp->mt_thr, tname);
529
530 if (vcpumap[vcpu] != NULL) {
531 error = pthread_setaffinity_np(mtp->mt_thr, sizeof(cpuset_t),
532 vcpumap[vcpu]);
533 assert(error == 0);
534 }
535
536 #ifdef BHYVE_SNAPSHOT
537 checkpoint_cpu_add(vcpu);
538 #endif
539 gdb_cpu_add(vcpu);
540
541 vm_loop(mtp->mt_ctx, vcpu);
542
543 /* not reached */
544 exit(1);
545 return (NULL);
546 }
547
548 static void
fbsdrun_addcpu(struct vmctx * ctx,int newcpu)549 fbsdrun_addcpu(struct vmctx *ctx, int newcpu)
550 {
551 int error;
552
553 error = vm_activate_cpu(ctx, newcpu);
554 if (error != 0)
555 err(EX_OSERR, "could not activate CPU %d", newcpu);
556
557 CPU_SET_ATOMIC(newcpu, &cpumask);
558
559 vm_suspend_cpu(ctx, newcpu);
560
561 mt_vmm_info[newcpu].mt_ctx = ctx;
562 mt_vmm_info[newcpu].mt_vcpu = newcpu;
563
564 error = pthread_create(&mt_vmm_info[newcpu].mt_thr, NULL,
565 fbsdrun_start_thread, &mt_vmm_info[newcpu]);
566 assert(error == 0);
567 }
568
569 static int
fbsdrun_deletecpu(int vcpu)570 fbsdrun_deletecpu(int vcpu)
571 {
572
573 if (!CPU_ISSET(vcpu, &cpumask)) {
574 EPRINTLN("Attempting to delete unknown cpu %d", vcpu);
575 exit(4);
576 }
577
578 CPU_CLR_ATOMIC(vcpu, &cpumask);
579 return (CPU_EMPTY(&cpumask));
580 }
581
582 static int
vmexit_handle_notify(struct vmctx * ctx __unused,struct vm_exit * vme __unused,int * pvcpu __unused,uint32_t eax __unused)583 vmexit_handle_notify(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
584 int *pvcpu __unused, uint32_t eax __unused)
585 {
586 #if BHYVE_DEBUG
587 /*
588 * put guest-driven debug here
589 */
590 #endif
591 return (VMEXIT_CONTINUE);
592 }
593
594 static int
vmexit_inout(struct vmctx * ctx,struct vm_exit * vme,int * pvcpu)595 vmexit_inout(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
596 {
597 int error;
598 int bytes, port, in, out;
599 int vcpu;
600
601 vcpu = *pvcpu;
602
603 port = vme->u.inout.port;
604 bytes = vme->u.inout.bytes;
605 in = vme->u.inout.in;
606 out = !in;
607
608 /* Extra-special case of host notifications */
609 if (out && port == GUEST_NIO_PORT) {
610 error = vmexit_handle_notify(ctx, vme, pvcpu, vme->u.inout.eax);
611 return (error);
612 }
613
614 error = emulate_inout(ctx, vcpu, vme);
615 if (error) {
616 EPRINTLN("Unhandled %s%c 0x%04x at 0x%lx",
617 in ? "in" : "out",
618 bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'),
619 port, vme->rip);
620 return (VMEXIT_ABORT);
621 } else {
622 return (VMEXIT_CONTINUE);
623 }
624 }
625
626 static int
vmexit_rdmsr(struct vmctx * ctx,struct vm_exit * vme,int * pvcpu)627 vmexit_rdmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
628 {
629 uint64_t val;
630 uint32_t eax, edx;
631 int error;
632
633 val = 0;
634 error = emulate_rdmsr(ctx, *pvcpu, vme->u.msr.code, &val);
635 if (error != 0) {
636 EPRINTLN("rdmsr to register %#x on vcpu %d",
637 vme->u.msr.code, *pvcpu);
638 if (get_config_bool("x86.strictmsr")) {
639 vm_inject_gp(ctx, *pvcpu);
640 return (VMEXIT_CONTINUE);
641 }
642 }
643
644 eax = val;
645 error = vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RAX, eax);
646 assert(error == 0);
647
648 edx = val >> 32;
649 error = vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RDX, edx);
650 assert(error == 0);
651
652 return (VMEXIT_CONTINUE);
653 }
654
655 static int
vmexit_wrmsr(struct vmctx * ctx,struct vm_exit * vme,int * pvcpu)656 vmexit_wrmsr(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
657 {
658 int error;
659
660 error = emulate_wrmsr(ctx, *pvcpu, vme->u.msr.code, vme->u.msr.wval);
661 if (error != 0) {
662 EPRINTLN("wrmsr to register %#x(%#lx) on vcpu %d",
663 vme->u.msr.code, vme->u.msr.wval, *pvcpu);
664 if (get_config_bool("x86.strictmsr")) {
665 vm_inject_gp(ctx, *pvcpu);
666 return (VMEXIT_CONTINUE);
667 }
668 }
669 return (VMEXIT_CONTINUE);
670 }
671
672 #define DEBUG_EPT_MISCONFIG
673 #ifdef DEBUG_EPT_MISCONFIG
674 #define VMCS_GUEST_PHYSICAL_ADDRESS 0x00002400
675
676 static uint64_t ept_misconfig_gpa, ept_misconfig_pte[4];
677 static int ept_misconfig_ptenum;
678 #endif
679
680 static const char *
vmexit_vmx_desc(uint32_t exit_reason)681 vmexit_vmx_desc(uint32_t exit_reason)
682 {
683
684 if (exit_reason >= nitems(vmx_exit_reason_desc) ||
685 vmx_exit_reason_desc[exit_reason] == NULL)
686 return ("Unknown");
687 return (vmx_exit_reason_desc[exit_reason]);
688 }
689
690 static int
vmexit_vmx(struct vmctx * ctx,struct vm_exit * vme,int * pvcpu)691 vmexit_vmx(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
692 {
693
694 EPRINTLN("vm exit[%d]", *pvcpu);
695 EPRINTLN("\treason\t\tVMX");
696 EPRINTLN("\trip\t\t0x%016lx", vme->rip);
697 EPRINTLN("\tinst_length\t%d", vme->inst_length);
698 EPRINTLN("\tstatus\t\t%d", vme->u.vmx.status);
699 EPRINTLN("\texit_reason\t%u (%s)", vme->u.vmx.exit_reason,
700 vmexit_vmx_desc(vme->u.vmx.exit_reason));
701 EPRINTLN("\tqualification\t0x%016lx",
702 vme->u.vmx.exit_qualification);
703 EPRINTLN("\tinst_type\t\t%d", vme->u.vmx.inst_type);
704 EPRINTLN("\tinst_error\t\t%d", vme->u.vmx.inst_error);
705 #ifdef DEBUG_EPT_MISCONFIG
706 if (vme->u.vmx.exit_reason == EXIT_REASON_EPT_MISCONFIG) {
707 vm_get_register(ctx, *pvcpu,
708 VMCS_IDENT(VMCS_GUEST_PHYSICAL_ADDRESS),
709 &ept_misconfig_gpa);
710 vm_get_gpa_pmap(ctx, ept_misconfig_gpa, ept_misconfig_pte,
711 &ept_misconfig_ptenum);
712 EPRINTLN("\tEPT misconfiguration:");
713 EPRINTLN("\t\tGPA: %#lx", ept_misconfig_gpa);
714 EPRINTLN("\t\tPTE(%d): %#lx %#lx %#lx %#lx",
715 ept_misconfig_ptenum, ept_misconfig_pte[0],
716 ept_misconfig_pte[1], ept_misconfig_pte[2],
717 ept_misconfig_pte[3]);
718 }
719 #endif /* DEBUG_EPT_MISCONFIG */
720 return (VMEXIT_ABORT);
721 }
722
723 static int
vmexit_svm(struct vmctx * ctx __unused,struct vm_exit * vme,int * pvcpu)724 vmexit_svm(struct vmctx *ctx __unused, struct vm_exit *vme, int *pvcpu)
725 {
726
727 EPRINTLN("vm exit[%d]", *pvcpu);
728 EPRINTLN("\treason\t\tSVM");
729 EPRINTLN("\trip\t\t0x%016lx", vme->rip);
730 EPRINTLN("\tinst_length\t%d", vme->inst_length);
731 EPRINTLN("\texitcode\t%#lx", vme->u.svm.exitcode);
732 EPRINTLN("\texitinfo1\t%#lx", vme->u.svm.exitinfo1);
733 EPRINTLN("\texitinfo2\t%#lx", vme->u.svm.exitinfo2);
734 return (VMEXIT_ABORT);
735 }
736
737 static int
vmexit_bogus(struct vmctx * ctx __unused,struct vm_exit * vme,int * pvcpu __unused)738 vmexit_bogus(struct vmctx *ctx __unused, struct vm_exit *vme,
739 int *pvcpu __unused)
740 {
741
742 assert(vme->inst_length == 0);
743
744 stats.vmexit_bogus++;
745
746 return (VMEXIT_CONTINUE);
747 }
748
749 static int
vmexit_reqidle(struct vmctx * ctx __unused,struct vm_exit * vme,int * pvcpu __unused)750 vmexit_reqidle(struct vmctx *ctx __unused, struct vm_exit *vme,
751 int *pvcpu __unused)
752 {
753
754 assert(vme->inst_length == 0);
755
756 stats.vmexit_reqidle++;
757
758 return (VMEXIT_CONTINUE);
759 }
760
761 static int
vmexit_hlt(struct vmctx * ctx __unused,struct vm_exit * vme __unused,int * pvcpu __unused)762 vmexit_hlt(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
763 int *pvcpu __unused)
764 {
765
766 stats.vmexit_hlt++;
767
768 /*
769 * Just continue execution with the next instruction. We use
770 * the HLT VM exit as a way to be friendly with the host
771 * scheduler.
772 */
773 return (VMEXIT_CONTINUE);
774 }
775
776 static int
vmexit_pause(struct vmctx * ctx __unused,struct vm_exit * vme __unused,int * pvcpu __unused)777 vmexit_pause(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
778 int *pvcpu __unused)
779 {
780
781 stats.vmexit_pause++;
782
783 return (VMEXIT_CONTINUE);
784 }
785
786 static int
vmexit_mtrap(struct vmctx * ctx __unused,struct vm_exit * vme,int * pvcpu)787 vmexit_mtrap(struct vmctx *ctx __unused, struct vm_exit *vme, int *pvcpu)
788 {
789
790 assert(vme->inst_length == 0);
791
792 stats.vmexit_mtrap++;
793
794 #ifdef BHYVE_SNAPSHOT
795 checkpoint_cpu_suspend(*pvcpu);
796 #endif
797 gdb_cpu_mtrap(*pvcpu);
798 #ifdef BHYVE_SNAPSHOT
799 checkpoint_cpu_resume(*pvcpu);
800 #endif
801
802 return (VMEXIT_CONTINUE);
803 }
804
805 static int
vmexit_inst_emul(struct vmctx * ctx,struct vm_exit * vme,int * pvcpu)806 vmexit_inst_emul(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
807 {
808 int err, i, cs_d;
809 struct vie *vie;
810 enum vm_cpu_mode mode;
811
812 stats.vmexit_inst_emul++;
813
814 vie = &vme->u.inst_emul.vie;
815 if (!vie->decoded) {
816 /*
817 * Attempt to decode in userspace as a fallback. This allows
818 * updating instruction decode in bhyve without rebooting the
819 * kernel (rapid prototyping), albeit with much slower
820 * emulation.
821 */
822 vie_restart(vie);
823 mode = vme->u.inst_emul.paging.cpu_mode;
824 cs_d = vme->u.inst_emul.cs_d;
825 if (vmm_decode_instruction(mode, cs_d, vie) != 0)
826 goto fail;
827 if (vm_set_register(ctx, *pvcpu, VM_REG_GUEST_RIP,
828 vme->rip + vie->num_processed) != 0)
829 goto fail;
830 }
831
832 err = emulate_mem(ctx, *pvcpu, vme->u.inst_emul.gpa,
833 vie, &vme->u.inst_emul.paging);
834 if (err) {
835 if (err == ESRCH) {
836 EPRINTLN("Unhandled memory access to 0x%lx\n",
837 vme->u.inst_emul.gpa);
838 }
839 goto fail;
840 }
841
842 return (VMEXIT_CONTINUE);
843
844 fail:
845 fprintf(stderr, "Failed to emulate instruction sequence [ ");
846 for (i = 0; i < vie->num_valid; i++)
847 fprintf(stderr, "%02x", vie->inst[i]);
848 FPRINTLN(stderr, " ] at 0x%lx", vme->rip);
849 return (VMEXIT_ABORT);
850 }
851
852 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
853 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
854
855 static int
vmexit_suspend(struct vmctx * ctx,struct vm_exit * vme,int * pvcpu)856 vmexit_suspend(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
857 {
858 enum vm_suspend_how how;
859
860 how = vme->u.suspended.how;
861
862 fbsdrun_deletecpu(*pvcpu);
863
864 if (*pvcpu != BSP) {
865 pthread_mutex_lock(&resetcpu_mtx);
866 pthread_cond_signal(&resetcpu_cond);
867 pthread_mutex_unlock(&resetcpu_mtx);
868 pthread_exit(NULL);
869 }
870
871 pthread_mutex_lock(&resetcpu_mtx);
872 while (!CPU_EMPTY(&cpumask)) {
873 pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
874 }
875 pthread_mutex_unlock(&resetcpu_mtx);
876
877 switch (how) {
878 case VM_SUSPEND_RESET:
879 exit(0);
880 case VM_SUSPEND_POWEROFF:
881 if (get_config_bool_default("destroy_on_poweroff", false))
882 vm_destroy(ctx);
883 exit(1);
884 case VM_SUSPEND_HALT:
885 exit(2);
886 case VM_SUSPEND_TRIPLEFAULT:
887 exit(3);
888 default:
889 EPRINTLN("vmexit_suspend: invalid reason %d", how);
890 exit(100);
891 }
892 return (0); /* NOTREACHED */
893 }
894
895 static int
vmexit_debug(struct vmctx * ctx __unused,struct vm_exit * vme __unused,int * pvcpu)896 vmexit_debug(struct vmctx *ctx __unused, struct vm_exit *vme __unused,
897 int *pvcpu)
898 {
899
900 #ifdef BHYVE_SNAPSHOT
901 checkpoint_cpu_suspend(*pvcpu);
902 #endif
903 gdb_cpu_suspend(*pvcpu);
904 #ifdef BHYVE_SNAPSHOT
905 checkpoint_cpu_resume(*pvcpu);
906 #endif
907 /*
908 * XXX-MJ sleep for a short period to avoid chewing up the CPU in the
909 * window between activation of the vCPU thread and the STARTUP IPI.
910 */
911 usleep(1000);
912 return (VMEXIT_CONTINUE);
913 }
914
915 static int
vmexit_breakpoint(struct vmctx * ctx __unused,struct vm_exit * vme,int * pvcpu)916 vmexit_breakpoint(struct vmctx *ctx __unused, struct vm_exit *vme, int *pvcpu)
917 {
918
919 gdb_cpu_breakpoint(*pvcpu, vme);
920 return (VMEXIT_CONTINUE);
921 }
922
923 static int
vmexit_ipi(struct vmctx * ctx,struct vm_exit * vme,int * pvcpu __unused)924 vmexit_ipi(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu __unused)
925 {
926 int error = -1;
927 int i;
928 switch (vme->u.ipi.mode) {
929 case APIC_DELMODE_INIT:
930 CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
931 error = vm_suspend_cpu(ctx, i);
932 if (error) {
933 warnx("%s: failed to suspend cpu %d\n",
934 __func__, i);
935 break;
936 }
937 }
938 break;
939 case APIC_DELMODE_STARTUP:
940 CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
941 spinup_ap(ctx, i, vme->u.ipi.vector << PAGE_SHIFT);
942 }
943 error = 0;
944 break;
945 default:
946 break;
947 }
948
949 return (error);
950 }
951
952 static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
953 [VM_EXITCODE_INOUT] = vmexit_inout,
954 [VM_EXITCODE_INOUT_STR] = vmexit_inout,
955 [VM_EXITCODE_VMX] = vmexit_vmx,
956 [VM_EXITCODE_SVM] = vmexit_svm,
957 [VM_EXITCODE_BOGUS] = vmexit_bogus,
958 [VM_EXITCODE_REQIDLE] = vmexit_reqidle,
959 [VM_EXITCODE_RDMSR] = vmexit_rdmsr,
960 [VM_EXITCODE_WRMSR] = vmexit_wrmsr,
961 [VM_EXITCODE_MTRAP] = vmexit_mtrap,
962 [VM_EXITCODE_INST_EMUL] = vmexit_inst_emul,
963 [VM_EXITCODE_SUSPENDED] = vmexit_suspend,
964 [VM_EXITCODE_TASK_SWITCH] = vmexit_task_switch,
965 [VM_EXITCODE_DEBUG] = vmexit_debug,
966 [VM_EXITCODE_BPT] = vmexit_breakpoint,
967 [VM_EXITCODE_IPI] = vmexit_ipi,
968 };
969
970 static void
vm_loop(struct vmctx * ctx,int vcpu)971 vm_loop(struct vmctx *ctx, int vcpu)
972 {
973 struct vm_exit vme;
974 int error, rc;
975 enum vm_exitcode exitcode;
976 cpuset_t active_cpus;
977
978 error = vm_active_cpus(ctx, &active_cpus);
979 assert(CPU_ISSET(vcpu, &active_cpus));
980
981 while (1) {
982 error = vm_run(ctx, vcpu, &vme);
983 if (error != 0)
984 break;
985
986 exitcode = vme.exitcode;
987 if (exitcode >= VM_EXITCODE_MAX || handler[exitcode] == NULL) {
988 fprintf(stderr, "vm_loop: unexpected exitcode 0x%x\n",
989 exitcode);
990 exit(4);
991 }
992
993 rc = (*handler[exitcode])(ctx, &vme, &vcpu);
994
995 switch (rc) {
996 case VMEXIT_CONTINUE:
997 break;
998 case VMEXIT_ABORT:
999 abort();
1000 default:
1001 exit(4);
1002 }
1003 }
1004 EPRINTLN("vm_run error %d, errno %d", error, errno);
1005 }
1006
1007 static int
num_vcpus_allowed(struct vmctx * ctx)1008 num_vcpus_allowed(struct vmctx *ctx)
1009 {
1010 uint16_t sockets, cores, threads, maxcpus;
1011 int tmp, error;
1012
1013 /*
1014 * The guest is allowed to spinup more than one processor only if the
1015 * UNRESTRICTED_GUEST capability is available.
1016 */
1017 error = vm_get_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, &tmp);
1018 if (error != 0)
1019 return (1);
1020
1021 error = vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
1022 if (error == 0)
1023 return (maxcpus);
1024 else
1025 return (1);
1026 }
1027
1028 static void
fbsdrun_set_capabilities(struct vmctx * ctx,int cpu)1029 fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
1030 {
1031 int err, tmp;
1032
1033 if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
1034 err = vm_get_capability(ctx, cpu, VM_CAP_HALT_EXIT, &tmp);
1035 if (err < 0) {
1036 EPRINTLN("VM exit on HLT not supported");
1037 exit(4);
1038 }
1039 vm_set_capability(ctx, cpu, VM_CAP_HALT_EXIT, 1);
1040 if (cpu == BSP)
1041 handler[VM_EXITCODE_HLT] = vmexit_hlt;
1042 }
1043
1044 if (get_config_bool_default("x86.vmexit_on_pause", false)) {
1045 /*
1046 * pause exit support required for this mode
1047 */
1048 err = vm_get_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, &tmp);
1049 if (err < 0) {
1050 EPRINTLN("SMP mux requested, no pause support");
1051 exit(4);
1052 }
1053 vm_set_capability(ctx, cpu, VM_CAP_PAUSE_EXIT, 1);
1054 if (cpu == BSP)
1055 handler[VM_EXITCODE_PAUSE] = vmexit_pause;
1056 }
1057
1058 if (get_config_bool_default("x86.x2apic", false))
1059 err = vm_set_x2apic_state(ctx, cpu, X2APIC_ENABLED);
1060 else
1061 err = vm_set_x2apic_state(ctx, cpu, X2APIC_DISABLED);
1062
1063 if (err) {
1064 EPRINTLN("Unable to set x2apic state (%d)", err);
1065 exit(4);
1066 }
1067
1068 vm_set_capability(ctx, cpu, VM_CAP_ENABLE_INVPCID, 1);
1069
1070 err = vm_set_capability(ctx, cpu, VM_CAP_IPI_EXIT, 1);
1071 assert(err == 0);
1072 }
1073
1074 static struct vmctx *
do_open(const char * vmname)1075 do_open(const char *vmname)
1076 {
1077 struct vmctx *ctx;
1078 int error;
1079 bool reinit, romboot;
1080
1081 reinit = romboot = false;
1082
1083 if (lpc_bootrom())
1084 romboot = true;
1085
1086 error = vm_create(vmname);
1087 if (error) {
1088 if (errno == EEXIST) {
1089 if (romboot) {
1090 reinit = true;
1091 } else {
1092 /*
1093 * The virtual machine has been setup by the
1094 * userspace bootloader.
1095 */
1096 }
1097 } else {
1098 perror("vm_create");
1099 exit(4);
1100 }
1101 } else {
1102 if (!romboot) {
1103 /*
1104 * If the virtual machine was just created then a
1105 * bootrom must be configured to boot it.
1106 */
1107 fprintf(stderr, "virtual machine cannot be booted\n");
1108 exit(4);
1109 }
1110 }
1111
1112 ctx = vm_open(vmname);
1113 if (ctx == NULL) {
1114 perror("vm_open");
1115 exit(4);
1116 }
1117
1118 #ifndef WITHOUT_CAPSICUM
1119 if (vm_limit_rights(ctx) != 0)
1120 err(EX_OSERR, "vm_limit_rights");
1121 #endif
1122
1123 if (reinit) {
1124 error = vm_reinit(ctx);
1125 if (error) {
1126 perror("vm_reinit");
1127 exit(4);
1128 }
1129 }
1130 error = vm_set_topology(ctx, cpu_sockets, cpu_cores, cpu_threads, 0);
1131 if (error)
1132 errx(EX_OSERR, "vm_set_topology");
1133 return (ctx);
1134 }
1135
1136 static void
spinup_vcpu(struct vmctx * ctx,int vcpu)1137 spinup_vcpu(struct vmctx *ctx, int vcpu)
1138 {
1139 int error;
1140
1141 if (vcpu != BSP) {
1142 fbsdrun_set_capabilities(ctx, vcpu);
1143
1144 /*
1145 * Enable the 'unrestricted guest' mode for APs.
1146 *
1147 * APs startup in power-on 16-bit mode.
1148 */
1149 error = vm_set_capability(ctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
1150 assert(error == 0);
1151 }
1152
1153 fbsdrun_addcpu(ctx, vcpu);
1154 }
1155
1156 static bool
parse_config_option(const char * option)1157 parse_config_option(const char *option)
1158 {
1159 const char *value;
1160 char *path;
1161
1162 value = strchr(option, '=');
1163 if (value == NULL || value[1] == '\0')
1164 return (false);
1165 path = strndup(option, value - option);
1166 if (path == NULL)
1167 err(4, "Failed to allocate memory");
1168 set_config_value(path, value + 1);
1169 return (true);
1170 }
1171
1172 static void
parse_simple_config_file(const char * path)1173 parse_simple_config_file(const char *path)
1174 {
1175 FILE *fp;
1176 char *line, *cp;
1177 size_t linecap;
1178 unsigned int lineno;
1179
1180 fp = fopen(path, "r");
1181 if (fp == NULL)
1182 err(4, "Failed to open configuration file %s", path);
1183 line = NULL;
1184 linecap = 0;
1185 lineno = 1;
1186 for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) {
1187 if (*line == '#' || *line == '\n')
1188 continue;
1189 cp = strchr(line, '\n');
1190 if (cp != NULL)
1191 *cp = '\0';
1192 if (!parse_config_option(line))
1193 errx(4, "%s line %u: invalid config option '%s'", path,
1194 lineno, line);
1195 }
1196 free(line);
1197 fclose(fp);
1198 }
1199
1200 static void
parse_gdb_options(const char * opt)1201 parse_gdb_options(const char *opt)
1202 {
1203 const char *sport;
1204 char *colon;
1205
1206 if (opt[0] == 'w') {
1207 set_config_bool("gdb.wait", true);
1208 opt++;
1209 }
1210
1211 colon = strrchr(opt, ':');
1212 if (colon == NULL) {
1213 sport = opt;
1214 } else {
1215 *colon = '\0';
1216 colon++;
1217 sport = colon;
1218 set_config_value("gdb.address", opt);
1219 }
1220
1221 set_config_value("gdb.port", sport);
1222 }
1223
1224 static void
set_defaults(void)1225 set_defaults(void)
1226 {
1227
1228 set_config_bool("acpi_tables", false);
1229 set_config_value("memory.size", "256M");
1230 set_config_bool("x86.strictmsr", true);
1231 set_config_value("lpc.fwcfg", "bhyve");
1232 }
1233
1234 int
main(int argc,char * argv[])1235 main(int argc, char *argv[])
1236 {
1237 int c, error;
1238 int max_vcpus, memflags;
1239 struct vmctx *ctx;
1240 struct qemu_fwcfg_item *e820_fwcfg_item;
1241 uint64_t rip;
1242 size_t memsize;
1243 const char *optstr, *value, *vmname;
1244 #ifdef BHYVE_SNAPSHOT
1245 char *restore_file;
1246 struct restore_state rstate;
1247
1248 restore_file = NULL;
1249 #endif
1250
1251 init_config();
1252 set_defaults();
1253 progname = basename(argv[0]);
1254
1255 #ifdef BHYVE_SNAPSHOT
1256 optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:l:K:U:r:";
1257 #else
1258 optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:l:K:U:";
1259 #endif
1260 while ((c = getopt(argc, argv, optstr)) != -1) {
1261 switch (c) {
1262 case 'a':
1263 set_config_bool("x86.x2apic", false);
1264 break;
1265 case 'A':
1266 set_config_bool("acpi_tables", true);
1267 break;
1268 case 'D':
1269 set_config_bool("destroy_on_poweroff", true);
1270 break;
1271 case 'p':
1272 if (pincpu_parse(optarg) != 0) {
1273 errx(EX_USAGE, "invalid vcpu pinning "
1274 "configuration '%s'", optarg);
1275 }
1276 break;
1277 case 'c':
1278 if (topology_parse(optarg) != 0) {
1279 errx(EX_USAGE, "invalid cpu topology "
1280 "'%s'", optarg);
1281 }
1282 break;
1283 case 'C':
1284 set_config_bool("memory.guest_in_core", true);
1285 break;
1286 case 'f':
1287 if (qemu_fwcfg_parse_cmdline_arg(optarg) != 0) {
1288 errx(EX_USAGE, "invalid fwcfg item '%s'", optarg);
1289 }
1290 break;
1291 case 'G':
1292 parse_gdb_options(optarg);
1293 break;
1294 case 'k':
1295 parse_simple_config_file(optarg);
1296 break;
1297 case 'K':
1298 set_config_value("keyboard.layout", optarg);
1299 break;
1300 case 'l':
1301 if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1302 lpc_print_supported_devices();
1303 exit(0);
1304 } else if (lpc_device_parse(optarg) != 0) {
1305 errx(EX_USAGE, "invalid lpc device "
1306 "configuration '%s'", optarg);
1307 }
1308 break;
1309 #ifdef BHYVE_SNAPSHOT
1310 case 'r':
1311 restore_file = optarg;
1312 break;
1313 #endif
1314 case 's':
1315 if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1316 pci_print_supported_devices();
1317 exit(0);
1318 } else if (pci_parse_slot(optarg) != 0)
1319 exit(4);
1320 else
1321 break;
1322 case 'S':
1323 set_config_bool("memory.wired", true);
1324 break;
1325 case 'm':
1326 set_config_value("memory.size", optarg);
1327 break;
1328 case 'o':
1329 if (!parse_config_option(optarg))
1330 errx(EX_USAGE, "invalid configuration option '%s'", optarg);
1331 break;
1332 case 'H':
1333 set_config_bool("x86.vmexit_on_hlt", true);
1334 break;
1335 case 'I':
1336 /*
1337 * The "-I" option was used to add an ioapic to the
1338 * virtual machine.
1339 *
1340 * An ioapic is now provided unconditionally for each
1341 * virtual machine and this option is now deprecated.
1342 */
1343 break;
1344 case 'P':
1345 set_config_bool("x86.vmexit_on_pause", true);
1346 break;
1347 case 'e':
1348 set_config_bool("x86.strictio", true);
1349 break;
1350 case 'u':
1351 set_config_bool("rtc.use_localtime", false);
1352 break;
1353 case 'U':
1354 set_config_value("uuid", optarg);
1355 break;
1356 case 'w':
1357 set_config_bool("x86.strictmsr", false);
1358 break;
1359 case 'W':
1360 set_config_bool("virtio_msix", false);
1361 break;
1362 case 'x':
1363 set_config_bool("x86.x2apic", true);
1364 break;
1365 case 'Y':
1366 set_config_bool("x86.mptable", false);
1367 break;
1368 case 'h':
1369 usage(0);
1370 default:
1371 usage(1);
1372 }
1373 }
1374 argc -= optind;
1375 argv += optind;
1376
1377 if (argc > 1)
1378 usage(1);
1379
1380 #ifdef BHYVE_SNAPSHOT
1381 if (restore_file != NULL) {
1382 error = load_restore_file(restore_file, &rstate);
1383 if (error) {
1384 fprintf(stderr, "Failed to read checkpoint info from "
1385 "file: '%s'.\n", restore_file);
1386 exit(1);
1387 }
1388 vmname = lookup_vmname(&rstate);
1389 if (vmname != NULL)
1390 set_config_value("name", vmname);
1391 }
1392 #endif
1393
1394 if (argc == 1)
1395 set_config_value("name", argv[0]);
1396
1397 vmname = get_config_value("name");
1398 if (vmname == NULL)
1399 usage(1);
1400
1401 if (get_config_bool_default("config.dump", false)) {
1402 dump_config();
1403 exit(1);
1404 }
1405
1406 calc_topology();
1407 build_vcpumaps();
1408
1409 value = get_config_value("memory.size");
1410 error = vm_parse_memsize(value, &memsize);
1411 if (error)
1412 errx(EX_USAGE, "invalid memsize '%s'", value);
1413
1414 ctx = do_open(vmname);
1415
1416 #ifdef BHYVE_SNAPSHOT
1417 if (restore_file != NULL) {
1418 guest_ncpus = lookup_guest_ncpus(&rstate);
1419 memflags = lookup_memflags(&rstate);
1420 memsize = lookup_memsize(&rstate);
1421 }
1422
1423 if (guest_ncpus < 1) {
1424 fprintf(stderr, "Invalid guest vCPUs (%d)\n", guest_ncpus);
1425 exit(1);
1426 }
1427 #endif
1428
1429 max_vcpus = num_vcpus_allowed(ctx);
1430 if (guest_ncpus > max_vcpus) {
1431 fprintf(stderr, "%d vCPUs requested but only %d available\n",
1432 guest_ncpus, max_vcpus);
1433 exit(4);
1434 }
1435
1436 fbsdrun_set_capabilities(ctx, BSP);
1437
1438 memflags = 0;
1439 if (get_config_bool_default("memory.wired", false))
1440 memflags |= VM_MEM_F_WIRED;
1441 if (get_config_bool_default("memory.guest_in_core", false))
1442 memflags |= VM_MEM_F_INCORE;
1443 vm_set_memflags(ctx, memflags);
1444 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
1445 if (error) {
1446 fprintf(stderr, "Unable to setup memory (%d)\n", errno);
1447 exit(4);
1448 }
1449
1450 error = init_msr();
1451 if (error) {
1452 fprintf(stderr, "init_msr error %d", error);
1453 exit(4);
1454 }
1455
1456 init_mem(guest_ncpus);
1457 init_inout();
1458 kernemu_dev_init();
1459 init_bootrom(ctx);
1460 atkbdc_init(ctx);
1461 pci_irq_init(ctx);
1462 ioapic_init(ctx);
1463
1464 rtc_init(ctx);
1465 sci_init(ctx);
1466
1467 if (qemu_fwcfg_init(ctx) != 0) {
1468 fprintf(stderr, "qemu fwcfg initialization error\n");
1469 exit(4);
1470 }
1471
1472 if (qemu_fwcfg_add_file("opt/bhyve/hw.ncpu", sizeof(guest_ncpus),
1473 &guest_ncpus) != 0) {
1474 fprintf(stderr, "Could not add qemu fwcfg opt/bhyve/hw.ncpu\n");
1475 exit(4);
1476 }
1477
1478 if (e820_init(ctx) != 0) {
1479 fprintf(stderr, "Unable to setup E820");
1480 exit(4);
1481 }
1482
1483 /*
1484 * Exit if a device emulation finds an error in its initilization
1485 */
1486 if (init_pci(ctx) != 0) {
1487 EPRINTLN("Device emulation initialization error: %s",
1488 strerror(errno));
1489 exit(4);
1490 }
1491
1492 /*
1493 * Initialize after PCI, to allow a bootrom file to reserve the high
1494 * region.
1495 */
1496 if (get_config_bool("acpi_tables"))
1497 vmgenc_init(ctx);
1498
1499 init_gdb(ctx);
1500
1501 if (lpc_bootrom()) {
1502 if (vm_set_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, 1)) {
1503 EPRINTLN("ROM boot failed: unrestricted guest "
1504 "capability not available");
1505 exit(4);
1506 }
1507 error = vcpu_reset(ctx, BSP);
1508 assert(error == 0);
1509 }
1510
1511 /* Allocate per-VCPU resources. */
1512 mt_vmm_info = calloc(guest_ncpus, sizeof(*mt_vmm_info));
1513
1514 /*
1515 * Add all vCPUs.
1516 */
1517 for (int vcpu = 0; vcpu < guest_ncpus; vcpu++) {
1518 spinup_vcpu(ctx, vcpu);
1519 }
1520
1521 #ifdef BHYVE_SNAPSHOT
1522 if (restore_file != NULL) {
1523 FPRINTLN(stdout, "Pausing pci devs...");
1524 if (vm_pause_devices() != 0) {
1525 EPRINTLN("Failed to pause PCI device state.");
1526 exit(1);
1527 }
1528
1529 FPRINTLN(stdout, "Restoring vm mem...");
1530 if (restore_vm_mem(ctx, &rstate) != 0) {
1531 EPRINTLN("Failed to restore VM memory.");
1532 exit(1);
1533 }
1534
1535 FPRINTLN(stdout, "Restoring pci devs...");
1536 if (vm_restore_devices(ctx, &rstate) != 0) {
1537 EPRINTLN("Failed to restore PCI device state.");
1538 exit(1);
1539 }
1540
1541 FPRINTLN(stdout, "Restoring kernel structs...");
1542 if (vm_restore_kern_structs(ctx, &rstate) != 0) {
1543 EPRINTLN("Failed to restore kernel structs.");
1544 exit(1);
1545 }
1546
1547 FPRINTLN(stdout, "Resuming pci devs...");
1548 if (vm_resume_devices() != 0) {
1549 EPRINTLN("Failed to resume PCI device state.");
1550 exit(1);
1551 }
1552 }
1553 #endif
1554
1555 error = vm_get_register(ctx, BSP, VM_REG_GUEST_RIP, &rip);
1556 assert(error == 0);
1557
1558 /*
1559 * build the guest tables, MP etc.
1560 */
1561 if (get_config_bool_default("x86.mptable", true)) {
1562 error = mptable_build(ctx, guest_ncpus);
1563 if (error) {
1564 perror("error to build the guest tables");
1565 exit(4);
1566 }
1567 }
1568
1569 error = smbios_build(ctx);
1570 if (error != 0)
1571 exit(4);
1572
1573 if (get_config_bool("acpi_tables")) {
1574 error = acpi_build(ctx, guest_ncpus);
1575 assert(error == 0);
1576 }
1577
1578 e820_fwcfg_item = e820_get_fwcfg_item();
1579 if (e820_fwcfg_item == NULL) {
1580 EPRINTLN("invalid e820 table");
1581 exit(4);
1582 }
1583 if (qemu_fwcfg_add_file("etc/e820", e820_fwcfg_item->size,
1584 e820_fwcfg_item->data) != 0) {
1585 EPRINTLN("could not add qemu fwcfg etc/e820");
1586 exit(4);
1587 }
1588 free(e820_fwcfg_item);
1589
1590 if (lpc_bootrom() && strcmp(lpc_fwcfg(), "bhyve") == 0) {
1591 fwctl_init();
1592 }
1593
1594 /*
1595 * Change the proc title to include the VM name.
1596 */
1597 setproctitle("%s", vmname);
1598
1599 #ifdef BHYVE_SNAPSHOT
1600 /* initialize mutex/cond variables */
1601 init_snapshot();
1602
1603 /*
1604 * checkpointing thread for communication with bhyvectl
1605 */
1606 if (init_checkpoint_thread(ctx) != 0)
1607 errx(EX_OSERR, "Failed to start checkpoint thread");
1608 #endif
1609
1610 #ifndef WITHOUT_CAPSICUM
1611 caph_cache_catpages();
1612
1613 if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1)
1614 errx(EX_OSERR, "Unable to apply rights for sandbox");
1615
1616 if (caph_enter() == -1)
1617 errx(EX_OSERR, "cap_enter() failed");
1618 #endif
1619
1620 #ifdef BHYVE_SNAPSHOT
1621 if (restore_file != NULL) {
1622 destroy_restore_state(&rstate);
1623 if (vm_restore_time(ctx) < 0)
1624 err(EX_OSERR, "Unable to restore time");
1625
1626 for (int i = 0; i < guest_ncpus; i++) {
1627 if (i == BSP)
1628 continue;
1629 vm_resume_cpu(ctx, i);
1630 }
1631 }
1632 #endif
1633 vm_resume_cpu(ctx, BSP);
1634
1635 /*
1636 * Head off to the main event dispatch loop
1637 */
1638 mevent_dispatch();
1639
1640 exit(4);
1641 }
1642